def setUp(self): self.config = None #placeholder self.TEST_PROVIDER_CONFIG = [ ("wikipedia", {}) ] # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() provider_queues = {} providers = ProviderFactory.get_providers(self.TEST_PROVIDER_CONFIG) for provider in providers: provider_queues[provider.provider_name] = backend.PythonQueue(provider.provider_name+"_queue") self.b = backend.Backend( backend.RedisQueue("alias-unittest", self.r), provider_queues, [backend.PythonQueue("couch_queue")], self.r) self.fake_item = { "_id": "1", "type": "item", "num_providers_still_updating":1, "aliases":{"pmid":["111"]}, "biblio": {}, "metrics": {} } self.fake_aliases_dict = {"pmid":["222"]} self.tiid = "abcd"
def setUp(self): self.d = None self.db = setup_postgres_for_unittests(db, app) # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://*****:*****@example.com' self.test_meta = { 'max_registered_items': 3, 'planned_use': 'individual CV', 'email': self.test_email, 'notes': '', 'api_key_owner': 'Julia Smith', "example_url":"", "organization":"NASA" } test_meta2 = copy.deepcopy(self.test_meta) test_meta2["email"] = '*****@*****.**' test_meta2["prefix"] = "SFU" self.existing_api_user = ApiUser(**test_meta2) self.existing_registered_item = RegisteredItem(self.test_alias_registered, self.existing_api_user) self.db.session.add(self.existing_api_user) self.db.session.add(self.existing_registered_item) self.db.session.commit()
def setUp(self): self.config = None #placeholder self.TEST_PROVIDER_CONFIG = [("wikipedia", {})] self.d = None # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb() provider_queues = {} providers = ProviderFactory.get_providers(self.TEST_PROVIDER_CONFIG) for provider in providers: provider_queues[provider.provider_name] = backend.PythonQueue( provider.provider_name + "_queue") self.b = backend.Backend(backend.RedisQueue("alias-unittest", self.r), provider_queues, [backend.PythonQueue("couch_queue")], self.r) self.fake_item = { "_id": "1", "type": "item", "num_providers_still_updating": 1, "aliases": { "pmid": ["111"] }, "biblio": {}, "metrics": {}, "last_modified": datetime.datetime(2013, 1, 1) } self.fake_aliases_dict = {"pmid": ["222"]} self.tiid = "abcd" self.db = setup_postgres_for_unittests(db, app)
def fix_github_year(): from totalimpact import item, tiredis myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) view_name = "queues/by_alias" view_rows = db.view(view_name, include_docs=True) row_count = 0 page_size = 500 start_key = ["url", "https://github.0000000"] end_key = ["url", "https://github.zzzzzzzz"] from couch_paginator import CouchPaginator page = CouchPaginator(db, view_name, page_size, include_docs=True, start_key=start_key, end_key=end_key) while page: for row in page: doc = row.doc print row.id try: doc["biblio"]["year"] = doc["biblio"]["create_date"][0:4] db.save(doc) except KeyError: pass row_count += 1 print "." logger.info("%i. getting new page, last id was %s" % (row_count, row.id)) if page.has_next: page = CouchPaginator(db, view_name, page_size, start_key=page.next, end_key=end_key, include_docs=True) else: page = None print "number items = ", row_count
def update_github(): from totalimpact import item, tiredis myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) view_name = "queues/by_alias" view_rows = db.view(view_name, include_docs=False) row_count = 0 page_size = 500 start_key = ["url", "https://github.0000000"] end_key = ["url", "https://github.zzzzzzzz"] from couch_paginator import CouchPaginator page = CouchPaginator(db, view_name, page_size, include_docs=False, start_key=start_key, end_key=end_key) while page: for row in page: tiid = row.id item.start_item_update([tiid], myredis, db, sleep_in_seconds=0.05) row_count += 1 print "." logger.info("%i. getting new page, last id was %s" % (row_count, row.id)) if page.has_next: page = CouchPaginator(db, view_name, page_size, start_key=page.next, end_key=end_key, include_docs=True) else: page = None print "number items = ", row_count
def setUp(self): self.config = None #placeholder self.TEST_PROVIDER_CONFIG = [ ("wikipedia", {}) ] self.d = None # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() provider_queues = {} providers = ProviderFactory.get_providers(self.TEST_PROVIDER_CONFIG) for provider in providers: provider_queues[provider.provider_name] = backend.PythonQueue(provider.provider_name+"_queue") self.b = backend.Backend( backend.RedisQueue("alias-unittest", self.r), provider_queues, [backend.PythonQueue("couch_queue")], self.r) self.fake_item = { "_id": "1", "type": "item", "num_providers_still_updating":1, "aliases":{"pmid":["111"]}, "biblio": {}, "metrics": {}, "last_modified": datetime.datetime(2013, 1, 1) } self.fake_aliases_dict = {"pmid":["222"]} self.tiid = "abcd" self.db = setup_postgres_for_unittests(db, app)
def fix_github_year(): from totalimpact import item, tiredis myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) view_name = "queues/by_alias" view_rows = db.view(view_name, include_docs=True) row_count = 0 page_size = 500 start_key = ["url", "https://github.0000000"] end_key = ["url", "https://github.zzzzzzzz"] from couch_paginator import CouchPaginator page = CouchPaginator(db, view_name, page_size, include_docs=True, start_key=start_key, end_key=end_key) while page: for row in page: doc = row.doc print row.id try: doc["biblio"]["year"] = doc["biblio"]["create_date"][0:4] db.save(doc) except KeyError: pass row_count += 1 print "." logger.info(u"%i. getting new page, last id was %s" %(row_count, row.id)) if page.has_next: page = CouchPaginator(db, view_name, page_size, start_key=page.next, end_key=end_key, include_docs=True) else: page = None print "number items = ", row_count
def update_github(): from totalimpact import item, tiredis myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) view_name = "queues/by_alias" view_rows = db.view(view_name, include_docs=False) row_count = 0 page_size = 500 start_key = ["url", "https://github.0000000"] end_key = ["url", "https://github.zzzzzzzz"] from couch_paginator import CouchPaginator page = CouchPaginator(db, view_name, page_size, include_docs=False, start_key=start_key, end_key=end_key) while page: for row in page: tiid = row.id item.start_item_update([tiid], myredis, db, sleep_in_seconds=0.05) row_count += 1 print "." logger.info(u"%i. getting new page, last id was %s" %(row_count, row.id)) if page.has_next: page = CouchPaginator(db, view_name, page_size, start_key=page.next, end_key=end_key, include_docs=True) else: page = None print "number items = ", row_count
def setUp(self): # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb()
def setUp(self): self.d = None # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb() self.db = setup_postgres_for_unittests(db, app) self.aliases = [["doi", "10.123"], ["doi", "10.124"], ["doi", "10.125"]]
def start_product_update(profile_id, tiids_to_update, priority): myredis = tiredis.from_url(os.getenv("REDIS_URL"), db=tiredis.REDIS_MAIN_DATABASE_NUMBER) # main app is on DB 0 # do all of this first and quickly for tiid in tiids_to_update: myredis.clear_provider_task_ids(tiid) myredis.set_provider_task_ids(tiid, ["STARTED"]) # set this right away # this import here to avoid circular dependancies from core_tasks import put_on_celery_queue put_on_celery_queue(profile_id, tiids_to_update, priority) return
def setUp(self): # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() self.myrefsets = {"nih": {"2011": { "facebook:comments": {0: [1, 99], 1: [91, 99]}, "mendeley:groups": {0: [1, 99], 3: [91, 99]} }}} # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb()
def setUp(self): self.d = None # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb() self.db = setup_postgres_for_unittests(db, app) self.aliases = [ ["doi", "10.123"], ["doi", "10.124"], ["doi", "10.125"] ]
def main(action_type, number_to_update=35, specific_publisher=None): #35 every 10 minutes is 35*6perhour*24hours=5040 per day redis_url = os.getenv("REDIS_URL") myredis = tiredis.from_url(redis_url) try: if action_type == "gold_update": print "running " + action_type tiids = gold_update(number_to_update, myredis) except (KeyboardInterrupt, SystemExit): # this approach is per http://stackoverflow.com/questions/2564137/python-how-to-terminate-a-thread-when-main-program-ends sys.exit()
def setUp(self): # setup a clean new redis database at our unittest redis DB location: Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() bibtex.Bibtex.paginate = lambda self, x: {"pages": [1,2,3,4], "number_entries":10} bibtex.Bibtex.member_items = lambda self, x: ("doi", str(x)) self.memberitems_resp = [ ["doi", "1"], ["doi", "2"], ["doi", "3"], ["doi", "4"], ] self.mi = models.MemberItems(bibtex.Bibtex(), self.r)
def setUp(self): # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() # do the same thing for the redis db. We're using DB 8 for unittests. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() #setup api test client self.app = app self.app.testing = True self.client = self.app.test_client()
def setUp(self): self.db = setup_postgres_for_unittests(db, app) # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() now = datetime.datetime.utcnow() self.before = now - datetime.timedelta(days=2) self.last_week = now - datetime.timedelta(days=7) self.last_year = now - datetime.timedelta(days=370) # save basic item beforehand, and some additional items self.fake_item_doc = { "_id": "tiid1", "type": "item", "last_modified": now.isoformat(), "last_update_run": now.isoformat(), "aliases":{"doi":["10.7554/elife.1"]}, "biblio": {"year":"2012"}, "metrics": {} } self.fake_item_obj = item_module.create_objects_from_item_doc(self.fake_item_doc) self.db.session.add(self.fake_item_obj) another_elife = copy.copy(self.fake_item_doc) another_elife["_id"] = "tiid2" another_elife["aliases"] = {"doi":["10.7554/ELIFE.2"]} another_elife["last_modified"] = self.before.isoformat() another_elife["last_update_run"] = self.before.isoformat() another_elife_obj = item_module.create_objects_from_item_doc(another_elife) self.db.session.add(another_elife_obj) different_journal = copy.copy(self.fake_item_doc) different_journal["_id"] = "tiid3" different_journal["aliases"] = {"doi":["10.3897/zookeys.3"], "biblio":[{"year":1999}]} different_journal["last_modified"] = now.isoformat() different_journal["last_update_run"] = self.last_week.isoformat() different_journal_obj = item_module.create_objects_from_item_doc(different_journal) self.db.session.add(different_journal_obj) different_journal2 = copy.copy(different_journal) different_journal2["_id"] = "tiid4" different_journal2["last_update_run"] = self.last_year.isoformat() different_journal_obj2 = item_module.create_objects_from_item_doc(different_journal2) self.db.session.add(different_journal_obj2) self.db.session.commit()
def setUp(self): self.db = setup_postgres_for_unittests(db, app) # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb() now = datetime.datetime.utcnow() self.before = now - datetime.timedelta(days=2) self.last_week = now - datetime.timedelta(days=7) self.last_year = now - datetime.timedelta(days=370) # save basic item beforehand, and some additional items self.fake_item_doc = { "_id": "tiid1", "type": "item", "last_modified": now.isoformat(), "last_update_run": now.isoformat(), "aliases":{"doi":["10.7554/elife.1"]}, "biblio": {"year":"2012"}, "metrics": {} } self.fake_item_obj = item_module.create_objects_from_item_doc(self.fake_item_doc) self.db.session.add(self.fake_item_obj) another_elife = copy.copy(self.fake_item_doc) another_elife["_id"] = "tiid2" another_elife["aliases"] = {"doi":["10.7554/ELIFE.2"]} another_elife["last_modified"] = self.before.isoformat() another_elife["last_update_run"] = self.before.isoformat() another_elife_obj = item_module.create_objects_from_item_doc(another_elife) self.db.session.add(another_elife_obj) different_journal = copy.copy(self.fake_item_doc) different_journal["_id"] = "tiid3" different_journal["aliases"] = {"doi":["10.3897/zookeys.3"], "biblio":[{"year":1999}]} different_journal["last_modified"] = now.isoformat() different_journal["last_update_run"] = self.last_week.isoformat() different_journal_obj = item_module.create_objects_from_item_doc(different_journal) self.db.session.add(different_journal_obj) different_journal2 = copy.copy(different_journal) different_journal2["_id"] = "tiid4" different_journal2["last_update_run"] = self.last_year.isoformat() different_journal_obj2 = item_module.create_objects_from_item_doc(different_journal2) self.db.session.add(different_journal_obj2) self.db.session.commit()
def setUp(self): # setup a clean new redis database at our unittest redis DB location: Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() bibtex.Bibtex.paginate = lambda self, x: { "pages": [1, 2, 3, 4], "number_entries": 10 } bibtex.Bibtex.member_items = lambda self, x: ("doi", str(x)) self.memberitems_resp = [ ["doi", "1"], ["doi", "2"], ["doi", "3"], ["doi", "4"], ] self.mi = models.MemberItems(bibtex.Bibtex(), self.r)
def main(): mydao = None myredis = tiredis.from_url(os.getenv("REDISTOGO_URL")) alias_queue = RedisQueue("aliasqueue", myredis) # to clear alias_queue: #import redis, os #myredis = redis.from_url(os.getenv("REDISTOGO_URL")) #myredis.delete(["aliasqueue"]) # these need to match the tiid alphabet defined in models: couch_queues = {} for i in "abcdefghijklmnopqrstuvwxyz1234567890": couch_queues[i] = PythonQueue(i+"_couch_queue") couch_worker = CouchWorker(couch_queues[i], myredis, mydao) couch_worker.spawn_and_loop() logger.info(u"launched backend couch worker with {i}_couch_queue".format( i=i)) polling_interval = 0.1 # how many seconds between polling to talk to provider provider_queues = {} providers = ProviderFactory.get_providers(default_settings.PROVIDERS) for provider in providers: provider_queues[provider.provider_name] = PythonQueue(provider.provider_name+"_queue") provider_worker = ProviderWorker( provider, polling_interval, alias_queue, provider_queues[provider.provider_name], couch_queues, ProviderWorker.wrapper, myredis) provider_worker.spawn_and_loop() backend = Backend(alias_queue, provider_queues, couch_queues, myredis) try: backend.run_in_loop() # don't need to spawn this one except (KeyboardInterrupt, SystemExit): # this approach is per http://stackoverflow.com/questions/2564137/python-how-to-terminate-a-thread-when-main-program-ends sys.exit()
def main(action_type, number_to_update=35): #35 every 10 minutes is 35*6perhour*24hours=5040 per day cloudant_db = os.getenv("CLOUDANT_DB") cloudant_url = os.getenv("CLOUDANT_URL") redis_url = os.getenv("REDIS_URL") mydao = dao.Dao(cloudant_url, cloudant_db) myredis = tiredis.from_url(redis_url) try: if action_type == "active_publishers": print "running " + action_type tiids = update_active_publisher_items(number_to_update, myredis, mydao) elif action_type == "least_recently_updated": print "running " + action_type tiids = update_least_recently_updated(number_to_update, myredis, mydao) except (KeyboardInterrupt, SystemExit): # this approach is per http://stackoverflow.com/questions/2564137/python-how-to-terminate-a-thread-when-main-program-ends sys.exit()
def setUp(self): from totalimpact import dao # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() self.test_alias = ("doi", "10.1371/journal.pcbi.1") self.test_alias_registered = ("doi", "10.1371/journal.pcbi.2") self.test_alias_registered_string = ":".join(self.test_alias_registered) self.sample_user_api_doc = {'key_history': {'2012-12-27T12:09:20.072080': 'SFUlqzam8'}, 'created': '2012-12-27T12:09:20.072080', 'current_key': 'SFUlqzam8', 'registered_items': {self.test_alias_registered_string: {"tiid":"tiid2", "registered_date":"2012etc"}}, 'max_registered_items':3, 'meta': {'usage': 'individual CV', 'api_limit': '', 'notes': '', 'api_key_owner': '', 'email': ''}, '_id': 'XeZhf8BWNgM5r9B9Xu3whT', 'type': 'api_user'} self.d.db.save(self.sample_user_api_doc)
def main(): mydao = None myredis = tiredis.from_url(os.getenv("REDISTOGO_URL")) alias_queue = RedisQueue("aliasqueue", myredis) # to clear alias_queue: #import redis, os #myredis = redis.from_url(os.getenv("REDISTOGO_URL")) #myredis.delete(["aliasqueue"]) # these need to match the tiid alphabet defined in models: couch_queues = {} for i in "abcdefghijklmnopqrstuvwxyz1234567890": couch_queues[i] = PythonQueue(i + "_couch_queue") couch_worker = CouchWorker(couch_queues[i], myredis, mydao) couch_worker.spawn_and_loop() logger.info( u"launched backend couch worker with {i}_couch_queue".format(i=i)) polling_interval = 0.1 # how many seconds between polling to talk to provider provider_queues = {} providers = ProviderFactory.get_providers(default_settings.PROVIDERS) for provider in providers: provider_queues[provider.provider_name] = PythonQueue( provider.provider_name + "_queue") provider_worker = ProviderWorker( provider, polling_interval, alias_queue, provider_queues[provider.provider_name], couch_queues, ProviderWorker.wrapper, myredis) provider_worker.spawn_and_loop() backend = Backend(alias_queue, provider_queues, couch_queues, myredis) try: backend.run_in_loop() # don't need to spawn this one except (KeyboardInterrupt, SystemExit): # this approach is per http://stackoverflow.com/questions/2564137/python-how-to-terminate-a-thread-when-main-program-ends sys.exit()
def setUp(self): # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) # need views to make sure to create them self.d.update_design_doc() # do the same thing for the redis db, set up the test redis database. We're using DB Number 8 self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() now = datetime.datetime.now() yesterday = now - datetime.timedelta(days=1) last_week = now - datetime.timedelta(days=7) # save basic item beforehand, and some additional items self.fake_item = { "_id": "tiid1", "type": "item", "last_modified": now.isoformat(), "aliases":{"doi":["10.7554/elife.1"]}, "biblio": {"year":2012}, "metrics": {} } self.d.save(self.fake_item) another_elife = copy.copy(self.fake_item) another_elife["_id"] = "tiid2" another_elife["aliases"] = {"doi":["10.7554/ELIFE.2"]} another_elife["last_modified"] = yesterday.isoformat() self.d.save(another_elife) different_journal = copy.copy(self.fake_item) different_journal["_id"] = "tiid3" different_journal["aliases"] = {"doi":["10.3897/zookeys.3"]} different_journal["last_modified"] = now.isoformat() different_journal["last_update_run"] = last_week.isoformat() self.d.save(different_journal)
def setUp(self): self.d = None self.db = setup_postgres_for_unittests(db, app) # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://*****:*****@example.com' self.test_meta = { 'max_registered_items': 3, 'planned_use': 'individual CV', 'email': self.test_email, 'notes': '', 'api_key_owner': 'Julia Smith', "example_url": "", "organization": "NASA" } test_meta2 = copy.deepcopy(self.test_meta) test_meta2["email"] = '*****@*****.**' test_meta2["prefix"] = "SFU" self.existing_api_user = ApiUser(**test_meta2) self.existing_registered_item = RegisteredItem( self.test_alias_registered, self.existing_api_user) self.db.session.add(self.existing_api_user) self.db.session.add(self.existing_registered_item) self.db.session.commit()
def set_redis(url, db): """useful for unit testing, where you want to use a local database """ global myredis myredis = tiredis.from_url(url, db) return myredis
import redis import uuid from totalimpact import dao, app, tiredis, collection, api_user, mixpanel from totalimpact import item as item_module from totalimpact.models import MemberItems, UserFactory, NotAuthenticatedError from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError, ProviderServerError, ProviderTimeout from totalimpact import default_settings import logging logger = logging.getLogger("ti.views") logger.setLevel(logging.DEBUG) mydao = dao.Dao(os.environ["CLOUDANT_URL"], os.getenv("CLOUDANT_DB")) myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) #main app is on DB 0 logger.debug("Building reference sets") myrefsets = None myrefsets_histograms = None try: (myrefsets, myrefsets_histograms) = collection.build_all_reference_lookups(myredis, mydao) logger.debug("Reference sets dict has %i keys" %len(myrefsets.keys())) except (couchdb.ResourceNotFound, LookupError, AttributeError), e: logger.error("Exception %s: Unable to load reference sets" % (e.__repr__())) def set_db(url, db): """useful for unit testing, where you want to use a local database """ global mydao mydao = dao.Dao(url, db)
import analytics import requests from totalimpact import app, tiredis, collection, api_user, incoming_email from totalimpact import item as item_module from totalimpact.models import MemberItems, NotAuthenticatedError from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError, ProviderServerError, ProviderTimeout from totalimpact import unicode_helpers from totalimpact import default_settings import logging logger = logging.getLogger("ti.views") logger.setLevel(logging.DEBUG) mydao = None myredis = tiredis.from_url(os.getenv("REDISTOGO_URL"), db=0) # main app is on DB 0 logger.debug(u"Building reference sets") myrefsets = None myrefsets_histograms = None try: (myrefsets, myrefsets_histograms) = collection.build_all_reference_lookups( myredis, mydao) logger.debug(u"Reference sets dict has %i keys" % len(myrefsets.keys())) except (LookupError, AttributeError), e: logger.error(u"Exception %s: Unable to load reference sets" % (e.__repr__())) def set_db(url, db): """useful for unit testing, where you want to use a local database
def setUp(self): ALIAS_DATA = { "title":["Why Most Published Research Findings Are False"], "url":["http://www.plosmedicine.org/article/info:doi/10.1371/journal.pmed.0020124"], "doi": ["10.1371/journal.pmed.0020124"] } STATIC_META = { "display_name": "readers", "provider": "Mendeley", "provider_url": "http://www.mendeley.com/", "description": "Mendeley readers: the number of readers of the article", "icon": "http://www.mendeley.com/favicon.ico", "category": "bookmark", "can_use_commercially": "0", "can_embed": "1", "can_aggregate": "1", "other_terms_of_use": "Must show logo and say 'Powered by Santa'", } self.KEY1 = "8888888888.8" self.KEY2 = "9999999999.9" self.VAL1 = 1 self.VAL2 = 2 METRICS_DATA = { "ignore": False, "static_meta": STATIC_META, "provenance_url": ["http://api.mendeley.com/research/public-chemical-compound-databases/"], "values":{ "raw": self.VAL1, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA2 = { "ignore": False, "latest_value": 21, "static_meta": STATIC_META, "provenance_url": ["http://api.mendeley.com/research/public-chemical-compound-databases/"], "values":{ "raw": self.VAL1, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA3 = { "ignore": False, "latest_value": 31, "static_meta": STATIC_META, "provenance_url": ["http://api.mendeley.com/research/public-chemical-compound-databases/"], "values":{ "raw": self.VAL1, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } BIBLIO_DATA = { "title": "An extension of de Finetti's theorem", "journal": "Advances in Applied Probability", "author": [ "Pitman, J" ], "collection": "pitnoid", "volume": "10", "id": "p78", "year": "1978", "pages": "268 to 270" } self.ITEM_DATA = { "_id": "test", "created": 1330260456.916, "last_modified": 12414214.234, "aliases": ALIAS_DATA, "metrics": { "wikipedia:mentions": METRICS_DATA, "bar:views": METRICS_DATA2 }, "biblio": BIBLIO_DATA, "type": "item" } self.TEST_PROVIDER_CONFIG = [ ("wikipedia", {}) ] # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() self.myrefsets = {"nih": {"2011": { "facebook:comments": {0: [1, 99], 1: [91, 99]}, "mendeley:groups": {0: [1, 99], 3: [91, 99]} }}} # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb()
def setUp(self): self.BIBLIO_DATA = { "title": "An extension of de Finetti's theorem", "journal": "Advances in Applied Probability", "author": [ "Pitman, J" ], "authors": "Pitman", "collection": "pitnoid", "volume": "10", "id": "p78", "year": "1978", "pages": "268 to 270" } self.ALIAS_DATA = { "title":["Why Most Published Research Findings Are False"], "url":["http://www.plosmedicine.org/article/info:doi/10.1371/journal.pmed.0020124"], "doi": ["10.1371/journal.pmed.0020124"], "biblio": [self.BIBLIO_DATA] } self.KEY1 = '2012-08-23T14:40:16.888888' self.KEY2 = '2012-08-23T14:40:16.999999' self.VAL1 = 1 self.VAL2 = 2 METRICS_DATA = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA2 = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA3 = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } self.ITEM_DATA = { "_id": "test", "created": '2012-08-23T14:40:16.399932', "last_modified": '2012-08-23T14:40:16.399932', "last_update_run": '2012-08-23T14:40:16.399932', "aliases": self.ALIAS_DATA, "metrics": { "wikipedia:mentions": METRICS_DATA, "altmetric_com:tweets": METRICS_DATA2 }, "biblio": self.BIBLIO_DATA, "type": "item" } self.TEST_PROVIDER_CONFIG = [ ("wikipedia", {}) ] self.d = None self.myrefsets = {"nih": {"2011": { "facebook:comments": {0: [1, 99], 1: [91, 99]}, "mendeley:groups": {0: [1, 99], 3: [91, 99]} }}} # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb() self.db = setup_postgres_for_unittests(db, app)
from celery.signals import task_postrun, task_prerun, task_failure, worker_process_init from celery import group, chain, chord from celery import current_app as celery_app from celery.signals import task_sent from celery.utils import uuid from eventlet import timeout from totalimpact import item as item_module from totalimpact import db from totalimpact import REDIS_MAIN_DATABASE_NUMBER from totalimpact import tiredis, default_settings from totalimpact.providers.provider import ProviderFactory, ProviderError, ProviderTimeout import rate_limit logger = logging.getLogger("core.tasks") myredis = tiredis.from_url(os.getenv("REDIS_URL"), db=REDIS_MAIN_DATABASE_NUMBER) rate = rate_limit.RateLimiter(redis_url=os.getenv("REDIS_URL"), redis_db=REDIS_MAIN_DATABASE_NUMBER) rate.add_condition({'requests':25, 'seconds':1}) # from https://github.com/celery/celery/issues/1671#issuecomment-47247074 # pending this being fixed in useful celery version """ Monkey patch for celery.chord.type property """ def _type(self): if self._type: return self._type if self._app: app = self._app
def setUp(self): # we're putting unittests for redis in their own db (number 8) so they can be deleted with abandon self.r = tiredis.from_url("redis://localhost:6379", db=REDIS_UNITTEST_DATABASE_NUMBER) self.r.flushdb()
def setUp(self): """ This test item is a lightly-modified version of a real doc from our demo collection; it's available at http://total-impact-core.herokuapp.com/collection/kn5auf """ test_item = ''' { "_id": "1aff9dfebea711e1bdf912313d1a5e63", "_rev": "968-c7891982fca2ea41346a20b80c2b888d", "aliases": { "doi": [ "10.5061/dryad.j1fd7" ], "title": [ "Data from: Data archiving is a good use of research funds", "data from: data archiving is a good investment" ], "url": [ "http://datadryad.org/handle/10255/dryad.33537", "http://hdl.handle.net/10255/dryad.33537" ] }, "biblio": { "authors": "Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock", "genre": "dataset", "h1": "Data from: Data archiving is a good investment", "repository": "Dryad Digital Repository", "title": "Data from: Data archiving is a good investment", "year": "2011" }, "created": "2012-06-25T09:21:11.960271", "currently_updating": false, "last_modified": "2012-11-18T04:57:40.539053", "metrics": { "delicious:bookmarks": { "provenance_url": "http://www.delicious.com/url/4794ddb7a3e934ba23165af65fcfa9cd", "static_meta": { "description": "The number of bookmarks to this artifact (maximum=100).", "display_name": "bookmarks", "icon": "http://www.delicious.com/favicon.ico", "provider": "Delicious", "provider_url": "http://www.delicious.com/" }, "values": { "raw": 1 } }, "dryad:total_downloads": { "provenance_url": "http://dx.doi.org/10.5061/dryad.j1fd7", "static_meta": { "description": "Dryad total downloads: combined number of downloads of the data package and data files", "display_name": "total downloads", "icon": "http:\\/\\/datadryad.org\\/favicon.ico", "provider": "Dryad", "provider_url": "http:\\/\\/www.datadryad.org\\/" }, "values": { "dryad": { "CI95_lower": 91, "CI95_upper": 98, "estimate_lower": 96, "estimate_upper": 96 }, "raw": 207, "raw_history": { "2012-06-25T09:21:16.027149": 132, "2012-06-26T18:05:19.598432": 132, "2012-06-26T20:10:16.858294": 132 } } } }, "type": "item" } ''' test_api_user = """ { "_id": "yDnhDa3fdFxxEsQnzYnA96", "created": "2012-11-19T16:11:17.713812", "current_key": "validkey", "registered_items": { "doi:10.1371/journal.pcbi.1000355": { "tiid": "b229e24abec811e1887612313d1a5e63", "registered_date": "2012-12-29T18:11:20.870026" } }, "max_registered_items": 1000, "key_history": { "2012-11-19T16:11:17.713812": "validkey" }, "meta": { "planned_use": "individual CV", "example_url": "", "api_key_owner": "Superman", "organization": "individual", "email": "*****@*****.**" }, "type": "api_user" } """ # hacky way to delete the "ti" db, then make it fresh again for each test. temp_dao = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) temp_dao.delete_db(os.getenv("CLOUDANT_DB")) self.d = dao.Dao("http://localhost:5984", os.getenv("CLOUDANT_DB")) self.d.update_design_doc() self.d.save(json.loads(test_item)) self.d.save(json.loads(test_api_user)) # do the same thing for the redis db. We're using DB 8 for unittests. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() #setup api test client self.app = app self.app.testing = True self.client = self.app.test_client()
def setUp(self): self.BIBLIO_DATA = { "title": "An extension of de Finetti's theorem", "journal": "Advances in Applied Probability", "author": [ "Pitman, J" ], "authors": "Pitman", "collection": "pitnoid", "volume": "10", "id": "p78", "year": "1978", "pages": "268 to 270" } self.ALIAS_DATA = { "title":["Why Most Published Research Findings Are False"], "url":["http://www.plosmedicine.org/article/info:doi/10.1371/journal.pmed.0020124"], "doi": ["10.1371/journal.pmed.0020124"], "biblio": [self.BIBLIO_DATA] } self.KEY1 = '2012-08-23T14:40:16.888888' self.KEY2 = '2012-08-23T14:40:16.999999' self.VAL1 = 1 self.VAL2 = 2 METRICS_DATA = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA2 = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } METRICS_DATA3 = { "provenance_url": "http://api.mendeley.com/research/public-chemical-compound-databases/", "values":{ "raw": self.VAL2, "raw_history": { self.KEY1: self.VAL1, self.KEY2: self.VAL2 } } } self.ITEM_DATA = { "_id": "test", "created": '2012-08-23T14:40:16.399932', "last_modified": '2012-08-23T14:40:16.399932', "aliases": self.ALIAS_DATA, "metrics": { "wikipedia:mentions": METRICS_DATA, "topsy:tweets": METRICS_DATA2 }, "biblio": self.BIBLIO_DATA, "type": "item" } self.TEST_PROVIDER_CONFIG = [ ("wikipedia", {}) ] self.d = None self.myrefsets = {"nih": {"2011": { "facebook:comments": {0: [1, 99], 1: [91, 99]}, "mendeley:groups": {0: [1, 99], 3: [91, 99]} }}} # setup a clean new redis test database. We're putting unittest redis at DB Number 8. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() self.db = setup_postgres_for_unittests(db, app)
from totalimpact import app, tiredis, collection, incoming_email, db from totalimpact import item as item_module from totalimpact.models import MemberItems, NotAuthenticatedError from totalimpact.providers import provider as provider_module from totalimpact.providers.provider import ProviderFactory, ProviderItemNotFoundError, ProviderError, ProviderServerError, ProviderTimeout from totalimpact import unicode_helpers from totalimpact import default_settings from totalimpact import REDIS_MAIN_DATABASE_NUMBER import logging logger = logging.getLogger("ti.views") logger.setLevel(logging.DEBUG) mydao = None myredis = tiredis.from_url(os.getenv("REDIS_URL"), db=REDIS_MAIN_DATABASE_NUMBER) # main app is on DB 0 # logger.debug(u"Building reference sets") myrefsets = None myrefsets_histograms = None # try: # (myrefsets, myrefsets_histograms) = collection.build_all_reference_lookups(myredis, mydao) # logger.debug(u"Reference sets dict has %i keys" %len(myrefsets.keys())) # except (LookupError, AttributeError), e: # logger.error(u"Exception %s: Unable to load reference sets" % (e.__repr__())) def set_db(url, db): """useful for unit testing, where you want to use a local database """ global mydao mydao = None
def setUp(self): # we're putting unittests for redis in their own db (number 8) so they can be deleted with abandon self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb()
def setUp(self): """ This test item is a lightly-modified version of a real doc from our demo collection; it's available at http://total-impact-core.herokuapp.com/collection/kn5auf """ test_item = ''' { "_id": "1aff9dfebea711e1bdf912313d1a5e63", "_rev": "968-c7891982fca2ea41346a20b80c2b888d", "aliases": { "doi": [ "10.5061/dryad.j1fd7" ], "title": [ "Data from: Data archiving is a good use of research funds", "data from: data archiving is a good investment" ], "url": [ "http://datadryad.org/handle/10255/dryad.33537", "http://hdl.handle.net/10255/dryad.33537" ] }, "biblio": { "authors": "Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock", "genre": "dataset", "h1": "Data from: Data archiving is a good investment", "repository": "Dryad Digital Repository", "title": "Data from: Data archiving is a good investment", "year": "2011" }, "created": "2012-06-25T09:21:11.960271", "currently_updating": false, "last_modified": "2012-11-18T04:57:40.539053", "metrics": { "delicious:bookmarks": { "provenance_url": "http://www.delicious.com/url/4794ddb7a3e934ba23165af65fcfa9cd", "static_meta": { "description": "The number of bookmarks to this artifact (maximum=100).", "display_name": "bookmarks", "icon": "http://www.delicious.com/favicon.ico", "provider": "Delicious", "provider_url": "http://www.delicious.com/" }, "values": { "raw": 1, "raw_history": { "2012-06-23T09:21:16.027149": 1 } } }, "dryad:total_downloads": { "provenance_url": "http://dx.doi.org/10.5061/dryad.j1fd7", "static_meta": { "description": "Dryad total downloads: combined number of downloads of the data package and data files", "display_name": "total downloads", "icon": "http:\\/\\/datadryad.org\\/favicon.ico", "provider": "Dryad", "provider_url": "http:\\/\\/www.datadryad.org\\/" }, "values": { "dryad": { "CI95_lower": 91, "CI95_upper": 98, "estimate_lower": 96, "estimate_upper": 96 }, "raw": 207, "raw_history": { "2012-06-25T09:21:16.027149": 132, "2012-06-26T18:05:19.598432": 132, "2012-06-26T20:10:16.858294": 132 } } } }, "type": "item" } ''' self.test_api_user_meta = { 'max_registered_items': 3, 'planned_use': 'individual CV', 'email': "*****@*****.**", 'notes': '', 'api_key_owner': 'Julia Smith', "example_url": "", "organization": "NASA", "prefix": "NASA", } self.db = setup_postgres_for_unittests(db, app) item = item_module.create_objects_from_item_doc(json.loads(test_item)) self.db.session.add(item) self.existing_api_user = api_user.ApiUser(**self.test_api_user_meta) self.existing_api_user.api_key = "validkey" #override randomly assigned key self.db.session.add(self.existing_api_user) self.db.session.commit() # do the same thing for the redis db. We're using DB 8 for unittests. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() #setup api test client self.app = app self.app.testing = True self.client = self.app.test_client() # Mock out relevant methods of the Dryad provider self.orig_Dryad_member_items = Dryad.member_items Dryad.member_items = MOCK_member_items self.aliases = [ ["doi", "10.123"], ["doi", "10.124"], ["doi", "10.125"] ]
def setUp(self): """ This test item is a lightly-modified version of a real doc from our demo collection; it's available at http://total-impact-core.herokuapp.com/collection/kn5auf """ test_item = ''' { "_id": "1aff9dfebea711e1bdf912313d1a5e63", "_rev": "968-c7891982fca2ea41346a20b80c2b888d", "aliases": { "doi": [ "10.5061/dryad.j1fd7" ], "title": [ "Data from: Data archiving is a good use of research funds", "data from: data archiving is a good investment" ], "url": [ "http://datadryad.org/handle/10255/dryad.33537", "http://hdl.handle.net/10255/dryad.33537" ] }, "biblio": { "authors": "Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock, Piwowar, Vision, Whitlock", "genre": "dataset", "h1": "Data from: Data archiving is a good investment", "repository": "Dryad Digital Repository", "title": "Data from: Data archiving is a good investment", "year": "2011" }, "created": "2012-06-25T09:21:11.960271", "currently_updating": false, "last_modified": "2012-11-18T04:57:40.539053", "metrics": { "delicious:bookmarks": { "provenance_url": "http://www.delicious.com/url/4794ddb7a3e934ba23165af65fcfa9cd", "static_meta": { "description": "The number of bookmarks to this artifact (maximum=100).", "display_name": "bookmarks", "icon": "http://www.delicious.com/favicon.ico", "provider": "Delicious", "provider_url": "http://www.delicious.com/" }, "values": { "raw": 1, "raw_history": { "2012-06-23T09:21:16.027149": 1 } } }, "dryad:total_downloads": { "provenance_url": "http://dx.doi.org/10.5061/dryad.j1fd7", "static_meta": { "description": "Dryad total downloads: combined number of downloads of the data package and data files", "display_name": "total downloads", "icon": "http:\\/\\/datadryad.org\\/favicon.ico", "provider": "Dryad", "provider_url": "http:\\/\\/www.datadryad.org\\/" }, "values": { "dryad": { "CI95_lower": 91, "CI95_upper": 98, "estimate_lower": 96, "estimate_upper": 96 }, "raw": 207, "raw_history": { "2012-06-25T09:21:16.027149": 132, "2012-06-26T18:05:19.598432": 132, "2012-06-26T20:10:16.858294": 132 } } } }, "type": "item" } ''' self.test_api_user_meta = { 'max_registered_items': 3, 'planned_use': 'individual CV', 'email': "*****@*****.**", 'notes': '', 'api_key_owner': 'Julia Smith', "example_url": "", "organization": "NASA", "prefix": "NASA", } self.db = setup_postgres_for_unittests(db, app) item = item_module.create_objects_from_item_doc(json.loads(test_item)) self.db.session.add(item) self.existing_api_user = api_user.ApiUser(**self.test_api_user_meta) self.existing_api_user.api_key = "validkey" #override randomly assigned key self.db.session.add(self.existing_api_user) self.db.session.commit() # do the same thing for the redis db. We're using DB 8 for unittests. self.r = tiredis.from_url("redis://localhost:6379", db=8) self.r.flushdb() #setup api test client self.app = app self.app.testing = True self.client = self.app.test_client() # Mock out relevant methods of the Dryad provider self.orig_Dryad_member_items = Dryad.member_items Dryad.member_items = MOCK_member_items self.aliases = [["doi", "10.123"], ["doi", "10.124"], ["doi", "10.125"]]