def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/subscriptions/update'
data = json.dumps({
'updates': {
self.app.pargs.id: self.app.pargs.date,
},
})
resp, content = http_client.request(
base_url + path,
method='POST',
headers={'Content-Type': 'application/json'},
body=data,
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
else:
results = json.loads(content)
failed = results['results'].get('failed', [])
print '%d issues failed:\n%r' % (len(failed), failed)
added = results['results'].get('added', [])
print '%d issues added:\n%r' % (len(added), added)
updated = results['results'].get('updated', [])
print '%d issues updated:\n%r' % (len(updated), updated)
skipped = results['results'].get('skipped', [])
print '%d issues skipped:\n%r' % (len(skipped), skipped)
def list(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/volumes/%s/list' % self.app.pargs.identifier
if self.app.pargs.context:
path = path + '?context=1'
resp, content = http_client.request(base_url + path)
if self.app.pargs.raw:
print content
return
result = json.loads(content)
if result['status'] == 200:
for issue in result['results']:
print '%7s %s %s %s' % (
issue['identifier'],
issue['pubdate'],
result['volume']['name'],
issue['issue_number'],
)
else:
print '%d %s' % (
result['status'],
result['message'],
)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/issues/list'
params = {}
if self.app.pargs.queued:
params['queued'] = 1
if self.app.pargs.context:
params['context'] = 1
if self.app.pargs.sort_key:
params['sort_key'] = self.app.pargs.sort_key
if self.app.pargs.reverse:
params['sort_order'] = 'desc'
else:
params['sort_order'] = 'asc'
if params:
path = path + '?%s' % urlencode(params)
resp, content = http_client.request(base_url + path)
if resp.status != 200:
print resp, content
elif self.app.pargs.raw:
print content
else:
result = json.loads(content)
for issue in result['results']:
print issue.keys()
print "%7s %10s %s [%s]" % (
issue['issue']['identifier'], issue['issue']['pubdate'],
issue['issue']['name'], issue['issue']['key']
)
def commit_file_nomatch_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
self.app.redis.client = MockRedis()
self.app.google._http = HttpMockSequence([
({'status': 200}, open(datafile(
'pull_update_pull_1000.json')).read()),
({'status': 200}, open(
datafile('pull_fetch_1000.json')).read()),
])
self.app.longbox._http = HttpMockSequence([
({'status': 200}, open(datafile('storage.json')).read()),
({'status': 200}, open(datafile('storage_nomatch.json')).read()),
({'status': 200}, open(datafile('storage.json')).read()),
({'status': 200}, open(datafile('storage_1000.json')).read()),
])
candidate = (('test issue 1', 1), '.', 'Test Issue 1 (2014).cbr')
with open(datafile('pull_data_1000.json')) as pull_data:
best_match = json.load(pull_data)
# cases: unread pull, no file, transfer good
upload.subprocess.check_call = mock.Mock()
plugin.commit_file(best_match, candidate)
self.app.redis.client.sadd.assert_called_with('gs:seen', 1000)
self.assertNotIn('pull:1000', self.app.redis.client.additions)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/watches/add'
data = {}
if self.app.pargs.arcs:
data['arcs'] = self.app.pargs.arcs.split(',')
if self.app.pargs.volumes:
data['volumes'] = self.app.pargs.volumes.split(',')
resp, content = http_client.request(
base_url + path,
method='POST',
headers={'Content-Type': 'application/json'},
body=json.dumps(data),
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
elif self.app.pargs.raw:
print content
else:
results = json.loads(content)
failed = results['results'].get('failed', [])
print '%d issues failed:\n%r' % (len(failed), failed)
added = results['results'].get('added', [])
print '%d issues added:\n%r' % (len(added), added)
skipped = results['results'].get('skipped', [])
print '%d issues skipped:\n%r' % (len(skipped), skipped)
def test_detect_ip(self):
self.app.setup()
dnscontroller = handler.get('controller', 'base')()
dnscontroller._setup(self.app)
myip = dnscontroller.detect_ip()
self.assertRegexpMatches(myip, r'\d+(?:\.\d+){3}')
def test_clear_loggers(self):
self.app.setup()
han = handler.get('log', 'logging')
Log = han()
Log.clear_loggers(self.app._meta.label)
previous_logger = logging.getLogger('previous_logger')
MyLog = ext_logging.LoggingLogHandler(clear_loggers='previous_logger')
MyLog._setup(self.app)
def check_new_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
self.app.google._http = HttpMockSequence([
({'status': 200}, open(datafile('fetch_new.json')).read()),
])
results = list(plugin.identify_unseen(check_type='new'))
self.assertEqual(len(results), 7)
def reindex(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/volumes/%s/reindex' % self.app.pargs.identifier
resp, content = http_client.request(base_url + path)
results = json.loads(content)
print '%(status)d %(message)s' % results
def test_clear_loggers(self):
self.app.setup()
han = handler.get('log', 'logging')
Log = han()
Log.clear_loggers(self.app._meta.label)
previous_logger = logging.getLogger('previous_logger')
MyLog = ext_logging.LoggingLogHandler(clear_loggers='previous_logger')
MyLog._setup(self.app)
def shard(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'batch_url')
path = '/batch/issues/refresh?shard=%s' % (
self.app.pargs.issue,
)
resp, content = http_client.request(base_url + path)
print content
def fetch_weighted_pulls_test(self):
self.app.setup()
# setup httpmock
redis_mock = MockRedis()
redis_mock._load_pull_data(datafile('sync_unread.json'))
self.app.redis.client = redis_mock
sync_handler = handler.get('controller', 'todosync')()
sync_handler.app = self.app
for pull_tuple in sync_handler.weighted_pulls():
self.assertIsInstance(pull_tuple, tuple)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/volumes/%s/queue' % (
self.app.pargs.volume,
)
resp, content = http_client.request(base_url + path)
print content
def test_clear_loggers(self):
self.app.setup()
han = handler.get("log", "logging")
Log = han()
Log.clear_loggers(self.app._meta.label)
# previous_logger = logging.getLogger(name)
MyLog = ext_logging.LoggingLogHandler(clear_loggers="%s:%s" % (self.app._meta.label, self.app._meta.label))
MyLog._setup(self.app)
def test_clear_loggers(self):
self.app.setup()
han = handler.get('log', 'logging')
Log = han()
Log.clear_loggers(self.app._meta.label)
MyLog = ext_logging.LoggingLogHandler(clear_loggers="%s:%s" %
(self.app._meta.label,
self.app._meta.label))
MyLog._setup(self.app)
def scan_dir_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
data_files = [
'Test Issue 1 (2014).cbr',
'Test Issue 2 (2014).cbr',
'Test Issue 3 (2014).cbr',
'Test Issue 4 (2014).cbr',
'Test Issue 5 (2014).cbr',
]
for path, filename in plugin.scan_dir(datafile('scan_test')):
self.assertIn(filename, data_files)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'batch_url')
if self.app.pargs.active:
active = '/active'
else:
active = ''
path = '/batch/volumes/refresh%s?shard=%s' % (
active,
self.app.pargs.shard,
)
resp, content = http_client.request(base_url + path)
print content
def find_matches_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
candidates = [
('.', 'Test Issue 1 (2014).cbr'),
('.', 'Test Issue 2 (2014).cbr'),
('.', 'Test Issue 3 (2014).cbr'),
('.', 'Test Issue 4 (2014).cbr'),
('.', 'Test Issue 5 (2014).cbr'),
]
results = json.load(open(datafile('fetch_new.json')))
pulls = [entry['pull'] for entry in results['results']]
for good_match, best_match, candidate in plugin.find_matches(
candidates, pulls, 0.25):
self.assertFalse(good_match)
def stats(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/streams/stats'
resp, content = http_client.request(base_url + path)
if resp.status == 200:
if self.app.pargs.raw:
print content
else:
response = json.loads(content)
for result in response['results']:
stream = result['stream']
self.print_stats(stream)
else:
print resp, content
def stats(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/volumes/stats'
resp, content = http_client.request(base_url + path)
if resp.status == 200:
if self.app.pargs.raw:
print content
else:
issue_stats = json.loads(content)
counts = issue_stats['counts']
print 'Queued volumes: %d' % counts['queued']
print 'Unindexed volumes: %d' % counts['toindex']
else:
self.app.log.error(resp, content)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'batch_url')
path = '/batch/subscriptions/update?shard=%d' % (
self.app.pargs.shard,
)
resp, content = http_client.request(
base_url + path,
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
else:
print base_url+path
results = json.loads(content)
print 'Found %d updates' % results['updated']
def ignored(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/watches/%s/pulls/ignored' % self.app.pargs.watch
resp, content = http_client.request(
base_url + path,
method='GET',
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
elif self.app.pargs.raw:
print content
else:
results = json.loads(content)
for result in results['results']:
print '%(id)8s %(name)s' % result['pull']
def fetch_stream(self, path):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
resp, content = http_client.request(
base_url + path,
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
else:
if self.app.pargs.raw:
print content
else:
results = json.loads(content)
print '%(status)d %(message)s' % results
for result in results['results']:
print result
def local(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/arcs/search/local?%s' % urlencode({
'q': self.app.pargs.query,
})
resp, content = http_client.request(base_url + path)
if self.app.pargs.raw:
print content
else:
arcs = json.loads(content)
for arc in arcs['results']:
print '%7s %4s %4s' % (
arc['arc_id'],
arc.get('issue_count', ''),
arc.get('name', ''),
)
def update_stream(self, path):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'batch_url')
resp, content = http_client.request(
base_url + path,
)
if resp.status != 200:
self.app.log.error('%r %r' % (resp, content))
else:
try:
results = json.loads(content)
except ValueError as err:
self.app.log.error(
'Error fetching resource %r: got %r\n' % (path, content))
self.app.log.error(repr(err))
else:
print content
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/issues/get/%s' % self.app.pargs.identifier
resp, content = http_client.request(base_url + path)
if resp.status != 200:
print resp, content
elif self.app.pargs.raw:
print content
else:
result = json.loads(content)
for issue in result['results']:
print issue.keys()
print "%7s %10s %s [%s]" % (
issue['issue']['identifier'], issue['issue']['pubdate'],
issue['issue']['name'], issue['issue']['key']
)
def test_load_plugins_from_config(self):
tmpdir = mkdtemp()
f = open(os.path.join(tmpdir, 'myplugin.py'), 'w')
f.write(PLUGIN)
f.close()
defaults = init_defaults()
defaults['myplugin'] = dict()
defaults['myplugin']['enable_plugin'] = True
defaults['myplugin2'] = dict()
defaults['myplugin2']['enable_plugin'] = False
app = self.make_app(
'myapp',
config_defaults=defaults,
config_files=[],
plugin_config_dir=tmpdir,
plugin_dir=tmpdir,
plugin_bootstrap=None,
)
app.setup()
try:
han = handler.get('output', 'test_output_handler')()
self.eq(han._meta.label, 'test_output_handler')
finally:
shutil.rmtree(tmpdir)
# some more checks
res = 'myplugin' in app.plugin.get_enabled_plugins()
self.ok(res)
res = 'myplugin' in app.plugin.get_loaded_plugins()
self.ok(res)
res = 'myplugin2' in app.plugin.get_disabled_plugins()
self.ok(res)
res = 'myplugin2' not in app.plugin.get_enabled_plugins()
self.ok(res)
res = 'myplugin2' not in app.plugin.get_loaded_plugins()
self.ok(res)
def default(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/issues/search?%s' % urlencode({
'q': self.app.pargs.query,
})
resp, content = http_client.request(base_url + path)
issues = json.loads(content)
print "Found %s matches" % issues['count']
for issue in issues['results']:
if issue.get('pubdate'):
pubdate = parse_date(issue['pubdate']).strftime('%Y-%m-%d')
else:
pubdate = ''
print "%7d %10s %s" % (
int(float(issue['issue_id'])),
pubdate,
issue.get('name', 'key=%s' % issue['id']),
)
def commit_new_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
self.app.redis.client = MockRedis()
self.app.google._http = HttpMockSequence([
({'status': 200}, open(datafile('storage.json')).read()),
({'status': 200}, open(datafile('storage_1001.json')).read()),
({'status': 200}, open(
datafile('pull_update_pull_1001.json')).read()),
({'status': 200}, open(
datafile('pull_fetch_1001.json')).read()),
])
candidate = ('.', 'Test Issue 2 (2014).cbr')
with open(datafile('pull_data_1001.json')) as pull_data:
best_match = json.load(pull_data)
# cases: new pull, file exists
upload.subprocess.check_call = mock.Mock()
plugin.commit_file(best_match, candidate)
self.app.redis.client.sadd.assert_called_with('gs:seen', 1001)
self.assertIn('pull:1001', self.app.redis.client.additions)
def local(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/volumes/search?%s' % urlencode({
'q': self.app.pargs.query,
})
resp, content = http_client.request(base_url + path)
if self.app.pargs.raw:
print content
else:
volumes = json.loads(content)
for volume in volumes['results']:
print '%7s %4s %4s %s [%s]' % (
volume['volume_id'],
volume['start_year'],
volume.get('issue_count', ''),
volume['name'],
volume['id'],
)
def test_load_plugins_from_config(self):
tmpdir = mkdtemp()
f = open(os.path.join(tmpdir, 'myplugin.py'), 'w')
f.write(PLUGIN)
f.close()
defaults = init_defaults()
defaults['myplugin'] = dict()
defaults['myplugin']['enable_plugin'] = True
defaults['myplugin2'] = dict()
defaults['myplugin2']['enable_plugin'] = False
app = self.make_app(APP, config_defaults=defaults,
config_files=[],
plugin_config_dir=tmpdir,
plugin_dir=tmpdir,
plugin_bootstrap=None,
)
app.setup()
try:
han = handler.get('output', 'test_output_handler')()
self.eq(han._meta.label, 'test_output_handler')
finally:
shutil.rmtree(tmpdir)
# some more checks
res = 'myplugin' in app.plugin.get_enabled_plugins()
self.ok(res)
res = 'myplugin' in app.plugin.get_loaded_plugins()
self.ok(res)
res = 'myplugin2' in app.plugin.get_disabled_plugins()
self.ok(res)
res = 'myplugin2' not in app.plugin.get_enabled_plugins()
self.ok(res)
res = 'myplugin2' not in app.plugin.get_loaded_plugins()
self.ok(res)
def list(self):
auth_handler = handler.get('auth', 'oauth2')()
auth_handler._setup(self.app)
http_client = auth_handler.client()
base_url = self.app.config.get('base', 'base_url')
path = '/api/subscriptions/list'
if self.app.pargs.context:
path = path + '?context=1'
resp, content = http_client.request(base_url + path)
if self.app.pargs.raw:
print content
elif resp.status == 200:
subscriptions = json.loads(content)
for subscription in subscriptions['results']:
print "%6s %4s %s" % (
subscription['subscription']['volume_id'],
subscription['subscription']['start_date'],
subscription.get('volume', {}).get('name'),
)
else:
self.app.log.error('Unable to load content:\n%r\n\n%r' % (
resp, content))
def commit_file_nomatch_fail_test(self):
self.app.setup()
plugin = handler.get('controller', 'upload')()
plugin.app = self.app
self.app.redis.client = MockRedis()
self.app.longbox._http = HttpMockSequence([
({'status': 200}, open(datafile('storage.json')).read()),
({'status': 200}, open(datafile('storage_nomatch.json')).read()),
])
candidate = (('test issue 1', 1), '.', 'Test Issue 1 (2014).cbr')
with open(datafile('pull_data_1000.json')) as pull_data:
best_match = json.load(pull_data)
# cases: unread pull, no file, transfer fails
upload.subprocess.check_call = mock.Mock(side_effect=ProcessError)
self.app.redis.client.sadd.reset_mock()
self.app.redis.client.set.reset_mock()
with self.assertRaises(ProcessError):
plugin.commit_file(best_match, candidate)
assert not self.app.redis.client.sadd.called, (
'redis.client.sadd called unexpectedly')
assert not self.app.redis.client.set.called, (
'redis.client.set called unexpectedly')
def test_load_plugins_from_files(self):
tmpdir = mkdtemp()
f = open(os.path.join(tmpdir, 'myplugin.conf'), 'w')
f.write(CONF)
f.close()
f = open(os.path.join(tmpdir, 'myplugin.py'), 'w')
f.write(PLUGIN)
f.close()
app = self.make_app('myapp',
config_files=[],
plugin_config_dir=tmpdir,
plugin_dir=tmpdir,
plugin_bootstrap=None,
)
app.setup()
try:
han = handler.get('output', 'test_output_handler')()
self.eq(han._meta.label, 'test_output_handler')
finally:
shutil.rmtree(tmpdir)
def _dispatch(self):
"""
Takes the remaining arguments from self.app.argv and parses for a
command to dispatch, and if so... dispatches it.
"""
self._add_arguments_to_parser()
self._parse_args()
self._process_args()
if not self.command:
self.app.log.debug("no command to dispatch")
else:
func = self.exposed[self.command]
self.app.log.debug("dispatching command: %s.%s" % \
(func['controller'], func['label']))
if func['controller'] == self._meta.label:
getattr(self, func['label'])()
else:
controller = handler.get('controller', func['controller'])()
controller._setup(self.app)
getattr(controller, func['label'])()
def test_clear_loggers(self):
self.app.setup()
han = handler.get('log', 'logging')
Log = han()
Log.clear_loggers()
def test_get_invalid_handler(self):
handler.get('output', 'bogus_handler')
def test_get_bogus_handler(self):
handler.get('log', 'bogus')
def test_get_bogus_handler_type(self):
handler.get('bogus', 'bogus')
def test_handler_get_fallback(self):
self.app.setup()
self.eq(handler.get('log', 'foo', 'bar'), 'bar')
