def q_collection(collection_id, enrichment): timeout = 10000 print "ENRICH {} with {}".format(collection_id, enrichment) ENQ = CouchDBJobEnqueue() ENQ.queue_collection(collection_id, timeout, harvester.post_processing.enrich_existing_couch_doc.main, enrichment )
def main(user_email, cid, url_couchdb=None): enq = CouchDBJobEnqueue() timeout = 10000 enq.queue_collection(cid, timeout, harvest_image_for_doc, url_couchdb=url_couchdb, )
def main(user_email, cid, url_couchdb=None): enq = CouchDBJobEnqueue() timeout = 10000 enq.queue_collection( cid, timeout, harvest_image_for_doc, url_couchdb=url_couchdb, )
def main(args): parser = argparse.ArgumentParser( description='run an Akara enrichment chain on documents in a \ collection.') parser.add_argument('document_id', help='Registry id for the collection') parser.add_argument('enrichment', help='enrichment chain to run') args = parser.parse_args(args) print(args.collection_id) print(args.enrichment) enq = CouchDBJobEnqueue() timeout = 10000 enq.queue_collection(args.collection_id, timeout, harvester.post_processing.enrich_existing_couch_doc.main, args.enrichment )
class CouchDBJobEnqueueTestCase(TestCase): #@patch('redis.client.Redis', autospec=True) @patch('harvester.post_processing.couchdb_runner.Redis') @httpretty.activate def setUp(self, mock_redis): self.conf = config() self.url_couch_base = self.conf['couchdb_url'] self.cdb = self.conf['couchdb_dbname'] print "+++++++++++++confg:{0}".format(self.conf) url_head = os.path.join(self.url_couch_base, self.cdb) httpretty.register_uri(httpretty.HEAD, url_head, body='', content_length='0', content_type='text/plain; charset=utf-8', connection='close', server='CouchDB/1.5.0 (Erlang OTP/R16B03)', cache_control='must-revalidate', date='Mon, 24 Nov 2014 21:30:38 GMT') self._cdbrunner = CouchDBJobEnqueue(rq_queue='test-delete') def func_for_test(doc, *args, **kwargs): return doc, args, kwargs self.function = func_for_test @httpretty.activate def testCollectionSlice(self): '''Test that results are correct for a known couchdb result''' url_to_pretty = os.path.join(self.url_couch_base, self.cdb, '_design', COUCHDB_VIEW.split('/')[0], '_view', COUCHDB_VIEW.split('/')[1]) httpretty.register_uri( httpretty.GET, re.compile(url_to_pretty + ".*$"), body=open(DIR_FIXTURES + '/couchdb_by_provider_name-5112.json').read(), etag="2U5BW2TDDX9EHZJOO0DNE29D1", content_type='application/json', ) #transfer_encoding='chunked', #NOTE: doesn't work with httpretty results = self._cdbrunner.queue_collection('5112', 6000, self.function, 'arg1', 'arg2', kwarg1='1', kwarg2=2) self.assertEqual(len(results), 3) self.assertEqual(results[0].args, ('5112--http://ark.cdlib.org/ark:/13030/kt7580382j', 'arg1', 'arg2')) self.assertEqual(results[0].kwargs, {'kwarg1': '1', 'kwarg2': 2}) self.assertEqual(results[0].func_name, 'test.test_couchdb_runner.func_for_test')
def main(args): parser = argparse.ArgumentParser( description='run the enrichments stored for a collection.') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--collection_id', help='Registry id for the collection') group.add_argument('--cid_file', help='File with collection ids for running') parser.add_argument('--rq_queue', help='Override queue for jobs, normal-stage is default') args = parser.parse_args(args) Q = 'normal-stage' if args.rq_queue: Q = args.rq_queue enq = CouchDBJobEnqueue(Q) timeout = 10000 cids = [] if args.collection_id: cids = [ args.collection_id ] else: #cid file with open(args.cid_file) as foo: lines = foo.readlines() cids = [ l.strip() for l in lines] print "CIDS:{}".format(cids) for cid in cids: url_api = ''.join(('https://registry.cdlib.org/api/v1/collection/', cid, '/')) coll = Collection(url_api) print coll.id enrichments = coll.enrichments_item enq.queue_collection(cid, timeout, harvester.post_processing.enrich_existing_couch_doc.main, enrichments )
def main(args): parser = argparse.ArgumentParser( description='run an Akara enrichment chain on documents in a \ collection.') parser.add_argument('collection_id', help='Registry id for the collection') parser.add_argument('enrichment', help='File of enrichment chain to run') parser.add_argument('--rq_queue', help='Override queue for jobs, normal-stage is default') args = parser.parse_args(args) print "CID:{}".format(args.collection_id) print "ENRICH FILE:{}".format(args.enrichment) with open(args.enrichment) as enrichfoo: enrichments = enrichfoo.read() Q = 'normal-stage' if args.rq_queue: Q = args.rq_queue enq = CouchDBJobEnqueue(Q) timeout = 10000 enq.queue_collection(args.collection_id, timeout, harvester.post_processing.enrich_existing_couch_doc.main, enrichments )
class CouchDBJobEnqueueTestCase(TestCase): #@patch('redis.client.Redis', autospec=True) @patch('harvester.post_processing.couchdb_runner.Redis') @httpretty.activate def setUp(self, mock_redis): self.conf = config() self.url_couch_base = self.conf['couchdb_url'] self.cdb = self.conf['couchdb_dbname'] print "+++++++++++++confg:{0}".format(self.conf) url_head = os.path.join(self.url_couch_base, self.cdb) httpretty.register_uri(httpretty.HEAD, url_head, body='', content_length='0', content_type='text/plain; charset=utf-8', connection='close', server='CouchDB/1.5.0 (Erlang OTP/R16B03)', cache_control='must-revalidate', date='Mon, 24 Nov 2014 21:30:38 GMT' ) self._cdbrunner = CouchDBJobEnqueue(rq_queue='test-delete') def func_for_test(doc, *args, **kwargs): return doc, args, kwargs self.function = func_for_test @httpretty.activate def testCollectionSlice(self): '''Test that results are correct for a known couchdb result''' url_to_pretty = os.path.join(self.url_couch_base, self.cdb, '_design', COUCHDB_VIEW.split('/')[0], '_view', COUCHDB_VIEW.split('/')[1]) httpretty.register_uri(httpretty.GET, re.compile(url_to_pretty+".*$"), body=open(DIR_FIXTURES+'/couchdb_by_provider_name-5112.json').read(), etag="2U5BW2TDDX9EHZJOO0DNE29D1", content_type='application/json', ) #transfer_encoding='chunked', #NOTE: doesn't work with httpretty results = self._cdbrunner.queue_collection('5112', 6000, self.function, 'arg1', 'arg2', kwarg1='1', kwarg2=2) self.assertEqual(len(results), 3) self.assertEqual(results[0].args, ('5112--http://ark.cdlib.org/ark:/13030/kt7580382j', 'arg1', 'arg2')) self.assertEqual(results[0].kwargs, {'kwarg1': '1', 'kwarg2': 2}) self.assertEqual(results[0].func_name, 'test.test_couchdb_runner.func_for_test')