def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/subscriptions/update' data = json.dumps({ 'updates': { self.app.pargs.id: self.app.pargs.date, }, }) resp, content = http_client.request( base_url + path, method='POST', headers={'Content-Type': 'application/json'}, body=data, ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) else: results = json.loads(content) failed = results['results'].get('failed', []) print '%d issues failed:\n%r' % (len(failed), failed) added = results['results'].get('added', []) print '%d issues added:\n%r' % (len(added), added) updated = results['results'].get('updated', []) print '%d issues updated:\n%r' % (len(updated), updated) skipped = results['results'].get('skipped', []) print '%d issues skipped:\n%r' % (len(skipped), skipped)
def list(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/volumes/%s/list' % self.app.pargs.identifier if self.app.pargs.context: path = path + '?context=1' resp, content = http_client.request(base_url + path) if self.app.pargs.raw: print content return result = json.loads(content) if result['status'] == 200: for issue in result['results']: print '%7s %s %s %s' % ( issue['identifier'], issue['pubdate'], result['volume']['name'], issue['issue_number'], ) else: print '%d %s' % ( result['status'], result['message'], )
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/issues/list' params = {} if self.app.pargs.queued: params['queued'] = 1 if self.app.pargs.context: params['context'] = 1 if self.app.pargs.sort_key: params['sort_key'] = self.app.pargs.sort_key if self.app.pargs.reverse: params['sort_order'] = 'desc' else: params['sort_order'] = 'asc' if params: path = path + '?%s' % urlencode(params) resp, content = http_client.request(base_url + path) if resp.status != 200: print resp, content elif self.app.pargs.raw: print content else: result = json.loads(content) for issue in result['results']: print issue.keys() print "%7s %10s %s [%s]" % ( issue['issue']['identifier'], issue['issue']['pubdate'], issue['issue']['name'], issue['issue']['key'] )
def commit_file_nomatch_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app self.app.redis.client = MockRedis() self.app.google._http = HttpMockSequence([ ({'status': 200}, open(datafile( 'pull_update_pull_1000.json')).read()), ({'status': 200}, open( datafile('pull_fetch_1000.json')).read()), ]) self.app.longbox._http = HttpMockSequence([ ({'status': 200}, open(datafile('storage.json')).read()), ({'status': 200}, open(datafile('storage_nomatch.json')).read()), ({'status': 200}, open(datafile('storage.json')).read()), ({'status': 200}, open(datafile('storage_1000.json')).read()), ]) candidate = (('test issue 1', 1), '.', 'Test Issue 1 (2014).cbr') with open(datafile('pull_data_1000.json')) as pull_data: best_match = json.load(pull_data) # cases: unread pull, no file, transfer good upload.subprocess.check_call = mock.Mock() plugin.commit_file(best_match, candidate) self.app.redis.client.sadd.assert_called_with('gs:seen', 1000) self.assertNotIn('pull:1000', self.app.redis.client.additions)
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/watches/add' data = {} if self.app.pargs.arcs: data['arcs'] = self.app.pargs.arcs.split(',') if self.app.pargs.volumes: data['volumes'] = self.app.pargs.volumes.split(',') resp, content = http_client.request( base_url + path, method='POST', headers={'Content-Type': 'application/json'}, body=json.dumps(data), ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) elif self.app.pargs.raw: print content else: results = json.loads(content) failed = results['results'].get('failed', []) print '%d issues failed:\n%r' % (len(failed), failed) added = results['results'].get('added', []) print '%d issues added:\n%r' % (len(added), added) skipped = results['results'].get('skipped', []) print '%d issues skipped:\n%r' % (len(skipped), skipped)
def test_detect_ip(self): self.app.setup() dnscontroller = handler.get('controller', 'base')() dnscontroller._setup(self.app) myip = dnscontroller.detect_ip() self.assertRegexpMatches(myip, r'\d+(?:\.\d+){3}')
def test_clear_loggers(self): self.app.setup() han = handler.get('log', 'logging') Log = han() Log.clear_loggers(self.app._meta.label) previous_logger = logging.getLogger('previous_logger') MyLog = ext_logging.LoggingLogHandler(clear_loggers='previous_logger') MyLog._setup(self.app)
def check_new_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app self.app.google._http = HttpMockSequence([ ({'status': 200}, open(datafile('fetch_new.json')).read()), ]) results = list(plugin.identify_unseen(check_type='new')) self.assertEqual(len(results), 7)
def reindex(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/volumes/%s/reindex' % self.app.pargs.identifier resp, content = http_client.request(base_url + path) results = json.loads(content) print '%(status)d %(message)s' % results
def shard(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'batch_url') path = '/batch/issues/refresh?shard=%s' % ( self.app.pargs.issue, ) resp, content = http_client.request(base_url + path) print content
def fetch_weighted_pulls_test(self): self.app.setup() # setup httpmock redis_mock = MockRedis() redis_mock._load_pull_data(datafile('sync_unread.json')) self.app.redis.client = redis_mock sync_handler = handler.get('controller', 'todosync')() sync_handler.app = self.app for pull_tuple in sync_handler.weighted_pulls(): self.assertIsInstance(pull_tuple, tuple)
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/volumes/%s/queue' % ( self.app.pargs.volume, ) resp, content = http_client.request(base_url + path) print content
def test_clear_loggers(self): self.app.setup() han = handler.get("log", "logging") Log = han() Log.clear_loggers(self.app._meta.label) # previous_logger = logging.getLogger(name) MyLog = ext_logging.LoggingLogHandler(clear_loggers="%s:%s" % (self.app._meta.label, self.app._meta.label)) MyLog._setup(self.app)
def test_clear_loggers(self): self.app.setup() han = handler.get('log', 'logging') Log = han() Log.clear_loggers(self.app._meta.label) MyLog = ext_logging.LoggingLogHandler(clear_loggers="%s:%s" % (self.app._meta.label, self.app._meta.label)) MyLog._setup(self.app)
def scan_dir_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app data_files = [ 'Test Issue 1 (2014).cbr', 'Test Issue 2 (2014).cbr', 'Test Issue 3 (2014).cbr', 'Test Issue 4 (2014).cbr', 'Test Issue 5 (2014).cbr', ] for path, filename in plugin.scan_dir(datafile('scan_test')): self.assertIn(filename, data_files)
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'batch_url') if self.app.pargs.active: active = '/active' else: active = '' path = '/batch/volumes/refresh%s?shard=%s' % ( active, self.app.pargs.shard, ) resp, content = http_client.request(base_url + path) print content
def find_matches_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app candidates = [ ('.', 'Test Issue 1 (2014).cbr'), ('.', 'Test Issue 2 (2014).cbr'), ('.', 'Test Issue 3 (2014).cbr'), ('.', 'Test Issue 4 (2014).cbr'), ('.', 'Test Issue 5 (2014).cbr'), ] results = json.load(open(datafile('fetch_new.json'))) pulls = [entry['pull'] for entry in results['results']] for good_match, best_match, candidate in plugin.find_matches( candidates, pulls, 0.25): self.assertFalse(good_match)
def stats(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/streams/stats' resp, content = http_client.request(base_url + path) if resp.status == 200: if self.app.pargs.raw: print content else: response = json.loads(content) for result in response['results']: stream = result['stream'] self.print_stats(stream) else: print resp, content
def stats(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/volumes/stats' resp, content = http_client.request(base_url + path) if resp.status == 200: if self.app.pargs.raw: print content else: issue_stats = json.loads(content) counts = issue_stats['counts'] print 'Queued volumes: %d' % counts['queued'] print 'Unindexed volumes: %d' % counts['toindex'] else: self.app.log.error(resp, content)
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'batch_url') path = '/batch/subscriptions/update?shard=%d' % ( self.app.pargs.shard, ) resp, content = http_client.request( base_url + path, ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) else: print base_url+path results = json.loads(content) print 'Found %d updates' % results['updated']
def ignored(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/watches/%s/pulls/ignored' % self.app.pargs.watch resp, content = http_client.request( base_url + path, method='GET', ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) elif self.app.pargs.raw: print content else: results = json.loads(content) for result in results['results']: print '%(id)8s %(name)s' % result['pull']
def fetch_stream(self, path): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') resp, content = http_client.request( base_url + path, ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) else: if self.app.pargs.raw: print content else: results = json.loads(content) print '%(status)d %(message)s' % results for result in results['results']: print result
def local(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/arcs/search/local?%s' % urlencode({ 'q': self.app.pargs.query, }) resp, content = http_client.request(base_url + path) if self.app.pargs.raw: print content else: arcs = json.loads(content) for arc in arcs['results']: print '%7s %4s %4s' % ( arc['arc_id'], arc.get('issue_count', ''), arc.get('name', ''), )
def update_stream(self, path): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'batch_url') resp, content = http_client.request( base_url + path, ) if resp.status != 200: self.app.log.error('%r %r' % (resp, content)) else: try: results = json.loads(content) except ValueError as err: self.app.log.error( 'Error fetching resource %r: got %r\n' % (path, content)) self.app.log.error(repr(err)) else: print content
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/issues/get/%s' % self.app.pargs.identifier resp, content = http_client.request(base_url + path) if resp.status != 200: print resp, content elif self.app.pargs.raw: print content else: result = json.loads(content) for issue in result['results']: print issue.keys() print "%7s %10s %s [%s]" % ( issue['issue']['identifier'], issue['issue']['pubdate'], issue['issue']['name'], issue['issue']['key'] )
def test_load_plugins_from_config(self): tmpdir = mkdtemp() f = open(os.path.join(tmpdir, 'myplugin.py'), 'w') f.write(PLUGIN) f.close() defaults = init_defaults() defaults['myplugin'] = dict() defaults['myplugin']['enable_plugin'] = True defaults['myplugin2'] = dict() defaults['myplugin2']['enable_plugin'] = False app = self.make_app( 'myapp', config_defaults=defaults, config_files=[], plugin_config_dir=tmpdir, plugin_dir=tmpdir, plugin_bootstrap=None, ) app.setup() try: han = handler.get('output', 'test_output_handler')() self.eq(han._meta.label, 'test_output_handler') finally: shutil.rmtree(tmpdir) # some more checks res = 'myplugin' in app.plugin.get_enabled_plugins() self.ok(res) res = 'myplugin' in app.plugin.get_loaded_plugins() self.ok(res) res = 'myplugin2' in app.plugin.get_disabled_plugins() self.ok(res) res = 'myplugin2' not in app.plugin.get_enabled_plugins() self.ok(res) res = 'myplugin2' not in app.plugin.get_loaded_plugins() self.ok(res)
def default(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/issues/search?%s' % urlencode({ 'q': self.app.pargs.query, }) resp, content = http_client.request(base_url + path) issues = json.loads(content) print "Found %s matches" % issues['count'] for issue in issues['results']: if issue.get('pubdate'): pubdate = parse_date(issue['pubdate']).strftime('%Y-%m-%d') else: pubdate = '' print "%7d %10s %s" % ( int(float(issue['issue_id'])), pubdate, issue.get('name', 'key=%s' % issue['id']), )
def commit_new_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app self.app.redis.client = MockRedis() self.app.google._http = HttpMockSequence([ ({'status': 200}, open(datafile('storage.json')).read()), ({'status': 200}, open(datafile('storage_1001.json')).read()), ({'status': 200}, open( datafile('pull_update_pull_1001.json')).read()), ({'status': 200}, open( datafile('pull_fetch_1001.json')).read()), ]) candidate = ('.', 'Test Issue 2 (2014).cbr') with open(datafile('pull_data_1001.json')) as pull_data: best_match = json.load(pull_data) # cases: new pull, file exists upload.subprocess.check_call = mock.Mock() plugin.commit_file(best_match, candidate) self.app.redis.client.sadd.assert_called_with('gs:seen', 1001) self.assertIn('pull:1001', self.app.redis.client.additions)
def local(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/volumes/search?%s' % urlencode({ 'q': self.app.pargs.query, }) resp, content = http_client.request(base_url + path) if self.app.pargs.raw: print content else: volumes = json.loads(content) for volume in volumes['results']: print '%7s %4s %4s %s [%s]' % ( volume['volume_id'], volume['start_year'], volume.get('issue_count', ''), volume['name'], volume['id'], )
def test_load_plugins_from_config(self): tmpdir = mkdtemp() f = open(os.path.join(tmpdir, 'myplugin.py'), 'w') f.write(PLUGIN) f.close() defaults = init_defaults() defaults['myplugin'] = dict() defaults['myplugin']['enable_plugin'] = True defaults['myplugin2'] = dict() defaults['myplugin2']['enable_plugin'] = False app = self.make_app(APP, config_defaults=defaults, config_files=[], plugin_config_dir=tmpdir, plugin_dir=tmpdir, plugin_bootstrap=None, ) app.setup() try: han = handler.get('output', 'test_output_handler')() self.eq(han._meta.label, 'test_output_handler') finally: shutil.rmtree(tmpdir) # some more checks res = 'myplugin' in app.plugin.get_enabled_plugins() self.ok(res) res = 'myplugin' in app.plugin.get_loaded_plugins() self.ok(res) res = 'myplugin2' in app.plugin.get_disabled_plugins() self.ok(res) res = 'myplugin2' not in app.plugin.get_enabled_plugins() self.ok(res) res = 'myplugin2' not in app.plugin.get_loaded_plugins() self.ok(res)
def list(self): auth_handler = handler.get('auth', 'oauth2')() auth_handler._setup(self.app) http_client = auth_handler.client() base_url = self.app.config.get('base', 'base_url') path = '/api/subscriptions/list' if self.app.pargs.context: path = path + '?context=1' resp, content = http_client.request(base_url + path) if self.app.pargs.raw: print content elif resp.status == 200: subscriptions = json.loads(content) for subscription in subscriptions['results']: print "%6s %4s %s" % ( subscription['subscription']['volume_id'], subscription['subscription']['start_date'], subscription.get('volume', {}).get('name'), ) else: self.app.log.error('Unable to load content:\n%r\n\n%r' % ( resp, content))
def commit_file_nomatch_fail_test(self): self.app.setup() plugin = handler.get('controller', 'upload')() plugin.app = self.app self.app.redis.client = MockRedis() self.app.longbox._http = HttpMockSequence([ ({'status': 200}, open(datafile('storage.json')).read()), ({'status': 200}, open(datafile('storage_nomatch.json')).read()), ]) candidate = (('test issue 1', 1), '.', 'Test Issue 1 (2014).cbr') with open(datafile('pull_data_1000.json')) as pull_data: best_match = json.load(pull_data) # cases: unread pull, no file, transfer fails upload.subprocess.check_call = mock.Mock(side_effect=ProcessError) self.app.redis.client.sadd.reset_mock() self.app.redis.client.set.reset_mock() with self.assertRaises(ProcessError): plugin.commit_file(best_match, candidate) assert not self.app.redis.client.sadd.called, ( 'redis.client.sadd called unexpectedly') assert not self.app.redis.client.set.called, ( 'redis.client.set called unexpectedly')
def test_load_plugins_from_files(self): tmpdir = mkdtemp() f = open(os.path.join(tmpdir, 'myplugin.conf'), 'w') f.write(CONF) f.close() f = open(os.path.join(tmpdir, 'myplugin.py'), 'w') f.write(PLUGIN) f.close() app = self.make_app('myapp', config_files=[], plugin_config_dir=tmpdir, plugin_dir=tmpdir, plugin_bootstrap=None, ) app.setup() try: han = handler.get('output', 'test_output_handler')() self.eq(han._meta.label, 'test_output_handler') finally: shutil.rmtree(tmpdir)
def _dispatch(self): """ Takes the remaining arguments from self.app.argv and parses for a command to dispatch, and if so... dispatches it. """ self._add_arguments_to_parser() self._parse_args() self._process_args() if not self.command: self.app.log.debug("no command to dispatch") else: func = self.exposed[self.command] self.app.log.debug("dispatching command: %s.%s" % \ (func['controller'], func['label'])) if func['controller'] == self._meta.label: getattr(self, func['label'])() else: controller = handler.get('controller', func['controller'])() controller._setup(self.app) getattr(controller, func['label'])()
def test_clear_loggers(self): self.app.setup() han = handler.get('log', 'logging') Log = han() Log.clear_loggers()
def test_get_invalid_handler(self): handler.get('output', 'bogus_handler')
def test_get_bogus_handler(self): handler.get('log', 'bogus')
def test_get_bogus_handler_type(self): handler.get('bogus', 'bogus')
def test_handler_get_fallback(self): self.app.setup() self.eq(handler.get('log', 'foo', 'bar'), 'bar')