def import_products(provider_name, import_input): if provider_name in ["bibtex", "product_id_strings"]: logger.debug(u"in import_products with provider_name {provider_name}".format( provider_name=provider_name)) else: logger.debug(u"in import_products with provider_name {provider_name}: {import_input}".format( provider_name=provider_name, import_input=import_input)) aliases = [] # pull in standard items, if we were passed any of these if provider_name=="product_id_strings": aliases = get_aliases_from_product_id_strings(import_input) elif provider_name=="bibtex": provider_module = ProviderFactory.get_provider("bibtex") aliases = provider_module.member_items(import_input) else: try: provider_module = ProviderFactory.get_provider(provider_name) aliases = provider_module.member_items(import_input) except ImportError: logger.debug(u"in import_products, got ImportError") pass # logger.debug(u"returning from import_products with aliases {aliases}".format( # aliases=aliases)) return(aliases)
def importer_post(provider_name): """ Gets aliases associated with a query from a given provider. """ input_string = request.json["input"] if provider_name == "pmids": provider_name = "pubmed" elif provider_name == "dois": provider_name = "crossref" elif provider_name == "urls": provider_name = "webpage" try: provider = ProviderFactory.get_provider(provider_name) except ImportError: abort_custom(404, "an importer for provider '{provider_name}' is not found".format( provider_name=provider_name)) try: aliases = provider.member_items(input_string) except ProviderItemNotFoundError: abort_custom(404, "item not found") except (ProviderTimeout, ProviderServerError): abort_custom(503, "timeout error, might be transient") except ProviderError: abort(500, "internal error from provider") tiids_aliases_map = item_module.create_tiids_from_aliases(aliases, myredis) logger.debug(u"in provider_importer_get with {tiids_aliases_map}".format( tiids_aliases_map=tiids_aliases_map)) products_dict = format_into_products_dict(tiids_aliases_map) resp = make_response(json.dumps({"products": products_dict}, sort_keys=True, indent=4), 200) return resp
def provider_method_wrapper(tiid, provider, method_name): # logger.info(u"{:20}: in provider_method_wrapper with {tiid} {provider_name} {method_name} with {aliases}".format( # "wrapper", tiid=tiid, provider_name=provider.provider_name, method_name=method_name, aliases=input_aliases_dict)) product = Product.query.get(tiid) if not product: logger.warning(u"Empty product in provider_run for tiid {tiid}".format( tiid=tiid)) return None input_alias_tuples = product.aliases_for_providers try: method = getattr(provider, method_name) except AttributeError: provider = ProviderFactory.get_provider(provider) method = getattr(provider, method_name) provider_name = provider.provider_name worker_name = provider_name+"_worker" try: method_response = method(input_alias_tuples) except ProviderError, e: method_response = None logger.info(u"{:20}: **ProviderError {tiid} {method_name} {provider_name}, Exception type {exception_type} {exception_arguments}".format( worker_name, tiid=tiid, provider_name=provider_name.upper(), method_name=method_name.upper(), exception_type=type(e).__name__, exception_arguments=e.args))
def provider_run(aliases_dict, tiid, method_name, provider_name): provider = ProviderFactory.get_provider(provider_name) # logger.info(u"in provider_run for {provider}".format( # provider=provider.provider_name)) (success, estimated_wait_seconds) = rate.acquire(provider_name, block=False) # add up to random 2 seconds to spread it out estimated_wait_seconds += random.random() * 3 if not success: logger.warning(u"RATE LIMIT HIT in provider_run for {provider} {method_name} {tiid}, retrying".format( provider=provider.provider_name, method_name=method_name, tiid=tiid)) provider_run.retry(args=[aliases_dict, tiid, method_name, provider_name], countdown=estimated_wait_seconds, max_retries=10) timeout_seconds = 30 try: with timeout.Timeout(timeout_seconds): response = provider_method_wrapper(tiid, aliases_dict, provider, method_name) except timeout.Timeout: msg = u"TIMEOUT in provider_run for {provider} {method_name} {tiid} after {timeout_seconds} seconds".format( provider=provider.provider_name, method_name=method_name, tiid=tiid, timeout_seconds=timeout_seconds) # logger.warning(msg) # message is written elsewhere raise ProviderTimeout(msg) return response
def provider_memberitems(provider_name): """ Starts a memberitems update for a specified provider, using a supplied file. Returns a hash of the file's contents, which is needed to get memberitems' output. To get output, poll GET /provider/<provider_name>/memberitems/<hash>?method=async """ # logger.debug("Query POSTed to {provider_name}/memberitems with request headers '{headers}'".format( # provider_name=provider_name, # headers=request.headers # )) file = request.files["file"] logger.debug("In provider_memberitems got file") logger.debug("filename = " + file.filename) query = file.read().decode("utf-8") provider = ProviderFactory.get_provider(provider_name) memberitems = MemberItems(provider, myredis) query_hash = memberitems.start_update(query) response_dict = {"query_hash": query_hash} resp = make_response(json.dumps(response_dict), 201) # created resp.mimetype = "application/json" resp.headers["Access-Control-Allow-Origin"] = "*" return resp
def provider_memberitems(provider_name): """ Make a descr string (like bibtex) into a dict strings describing items. """ provider = ProviderFactory.get_provider(provider_name) items_dict = provider.parse(request.json["descr"]) resp = make_response( json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200) return resp
def provider_memberitems(provider_name): """ Make a descr string (like bibtex) into a dict strings describing items. """ provider = ProviderFactory.get_provider(provider_name) items_dict = provider.parse(request.json["descr"]) resp = make_response( json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200 ) return resp
def provider_memberitems(provider_name): query = request.values.get('query','') logger.debug("In provider_memberitems with " + query) provider = ProviderFactory.get_provider(provider_name) logger.debug("provider: " + provider.provider_name) memberitems = provider.member_items(query, cache_enabled=False) resp = make_response( json.dumps(memberitems, sort_keys=True, indent=4), 200 ) resp.mimetype = "application/json" return resp
def provider_biblio(provider_name, id): provider = ProviderFactory.get_provider(provider_name) if id=="example": id = provider.example_id[1] url = "http://localhost:8080/" + provider_name + "/biblio?%s" else: url = None biblio = provider.get_biblio_for_id(id, url, cache_enabled=False) resp = make_response( json.dumps(biblio, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp
def provider_memberitems(provider_name): """ Make a file into a dict strings describing items. """ mixpanel.track("Trigger:Import", {"Provider":provider_name}, request) file = request.files['file'] logger.debug("In"+provider_name+"/memberitems, got file: filename="+file.filename) entries_str = file.read().decode("utf-8") provider = ProviderFactory.get_provider(provider_name) items_dict = provider.parse(entries_str) resp = make_response(json.dumps(items_dict), 200) # created resp.mimetype = "application/json" resp.headers['Access-Control-Allow-Origin'] = "*" return resp
def provider_aliases(provider_name, id): provider = ProviderFactory.get_provider(provider_name) if id=="example": id = provider.example_id[1] url = "http://localhost:8080/" + provider_name + "/aliases?%s" else: url = None try: new_aliases = provider._get_aliases_for_id(id, url, cache_enabled=False) except NotImplementedError: new_aliases = [] all_aliases = [(provider.example_id[0], id)] + new_aliases resp = make_response( json.dumps(all_aliases, sort_keys=True, indent=4) ) resp.mimetype = "application/json" return resp
def provider_memberitems_get(provider_name, query): provider = ProviderFactory.get_provider(provider_name) memberitems = MemberItems(provider, myredis) method = request.args.get("method", "sync") try: ret = getattr(memberitems, "get_" + method)(query) except ProviderItemNotFoundError: abort(404) except ProviderError: abort(500) if ret: if ret["error"]: abort(503) # crossref lookup error, might be transient resp = make_response(json.dumps(ret, sort_keys=True, indent=4), 200) resp.mimetype = "application/json" resp.headers["Access-Control-Allow-Origin"] = "*" return resp
def provider_memberitems_get(provider_name, query): """ Gets aliases associated with a query from a given provider. """ query = unicode_helpers.remove_nonprinting_characters(query) provider = ProviderFactory.get_provider(provider_name) try: items_dict = provider.member_items(query) except ProviderItemNotFoundError: abort_custom(404, "item not found") except (ProviderTimeout, ProviderServerError): abort_custom(503, "crossref lookup error, might be transient") except ProviderError: abort(500, "internal error from provider") resp = make_response( json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200) return resp
def importer_post(provider_name): """ Gets aliases associated with a query from a given provider. """ input_string = request.json["input"] if provider_name == "pmids": provider_name = "pubmed" elif provider_name == "dois": provider_name = "crossref" elif provider_name == "urls": provider_name = "webpage" try: provider = ProviderFactory.get_provider(provider_name) except ImportError: abort_custom( 404, "an importer for provider '{provider_name}' is not found".format( provider_name=provider_name)) try: aliases = provider.member_items(input_string) except ProviderItemNotFoundError: abort_custom(404, "item not found") except (ProviderTimeout, ProviderServerError): abort_custom(503, "timeout error, might be transient") except ProviderError: abort(500, "internal error from provider") tiids_aliases_map = item_module.create_tiids_from_aliases(aliases, myredis) logger.debug(u"in provider_importer_get with {tiids_aliases_map}".format( tiids_aliases_map=tiids_aliases_map)) products_dict = format_into_products_dict(tiids_aliases_map) resp = make_response( json.dumps({"products": products_dict}, sort_keys=True, indent=4), 200) return resp
def provider_memberitems_get(provider_name, query): """ Gets aliases associated with a query from a given provider. """ mixpanel.track("Trigger:Import", {"Provider":provider_name}, request) try: provider = ProviderFactory.get_provider(provider_name) ret = provider.member_items(query) except ProviderItemNotFoundError: abort(404) except (ProviderTimeout, ProviderServerError): abort(503) # crossref lookup error, might be transient except ProviderError: abort(500) resp = make_response( json.dumps({"memberitems":ret}, sort_keys=True, indent=4), 200 ) resp.mimetype = "application/json" resp.headers['Access-Control-Allow-Origin'] = "*" return resp
def provider_memberitems_get(provider_name, query): """ Gets aliases associated with a query from a given provider. """ query = unicode_helpers.remove_nonprinting_characters(query) provider = ProviderFactory.get_provider(provider_name) try: items_dict = provider.member_items(query) except ProviderItemNotFoundError: abort_custom(404, "item not found") except (ProviderTimeout, ProviderServerError): abort_custom(503, "crossref lookup error, might be transient") except ProviderError: abort(500, "internal error from provider") resp = make_response( json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200 ) return resp
def setUp(self): self.provider = ProviderFactory.get_provider(self.provider_name) self.old_http_get = Provider.http_get
def test_get_provider(self): provider = ProviderFactory.get_provider("wikipedia") assert_equals(provider.__class__.__name__, "Wikipedia")
def test_08_get_provider(self): provider = ProviderFactory.get_provider("wikipedia") assert_equals(provider.__class__.__name__, "Wikipedia")
def setUp(self): self.provider = ProviderFactory.get_provider(self.provider_name) self.old_http_get = Provider.http_get