def import_products(provider_name, import_input):
if provider_name in ["bibtex", "product_id_strings"]:
logger.debug(u"in import_products with provider_name {provider_name}".format(
provider_name=provider_name))
else:
logger.debug(u"in import_products with provider_name {provider_name}: {import_input}".format(
provider_name=provider_name, import_input=import_input))
aliases = []
# pull in standard items, if we were passed any of these
if provider_name=="product_id_strings":
aliases = get_aliases_from_product_id_strings(import_input)
elif provider_name=="bibtex":
provider_module = ProviderFactory.get_provider("bibtex")
aliases = provider_module.member_items(import_input)
else:
try:
provider_module = ProviderFactory.get_provider(provider_name)
aliases = provider_module.member_items(import_input)
except ImportError:
logger.debug(u"in import_products, got ImportError")
pass
# logger.debug(u"returning from import_products with aliases {aliases}".format(
# aliases=aliases))
return(aliases)
def importer_post(provider_name):
"""
Gets aliases associated with a query from a given provider.
"""
input_string = request.json["input"]
if provider_name == "pmids":
provider_name = "pubmed"
elif provider_name == "dois":
provider_name = "crossref"
elif provider_name == "urls":
provider_name = "webpage"
try:
provider = ProviderFactory.get_provider(provider_name)
except ImportError:
abort_custom(404, "an importer for provider '{provider_name}' is not found".format(
provider_name=provider_name))
try:
aliases = provider.member_items(input_string)
except ProviderItemNotFoundError:
abort_custom(404, "item not found")
except (ProviderTimeout, ProviderServerError):
abort_custom(503, "timeout error, might be transient")
except ProviderError:
abort(500, "internal error from provider")
tiids_aliases_map = item_module.create_tiids_from_aliases(aliases, myredis)
logger.debug(u"in provider_importer_get with {tiids_aliases_map}".format(
tiids_aliases_map=tiids_aliases_map))
products_dict = format_into_products_dict(tiids_aliases_map)
resp = make_response(json.dumps({"products": products_dict}, sort_keys=True, indent=4), 200)
return resp
def provider_method_wrapper(tiid, provider, method_name):
# logger.info(u"{:20}: in provider_method_wrapper with {tiid} {provider_name} {method_name} with {aliases}".format(
# "wrapper", tiid=tiid, provider_name=provider.provider_name, method_name=method_name, aliases=input_aliases_dict))
product = Product.query.get(tiid)
if not product:
logger.warning(u"Empty product in provider_run for tiid {tiid}".format(
tiid=tiid))
return None
input_alias_tuples = product.aliases_for_providers
try:
method = getattr(provider, method_name)
except AttributeError:
provider = ProviderFactory.get_provider(provider)
method = getattr(provider, method_name)
provider_name = provider.provider_name
worker_name = provider_name+"_worker"
try:
method_response = method(input_alias_tuples)
except ProviderError, e:
method_response = None
logger.info(u"{:20}: **ProviderError {tiid} {method_name} {provider_name}, Exception type {exception_type} {exception_arguments}".format(
worker_name,
tiid=tiid,
provider_name=provider_name.upper(),
method_name=method_name.upper(),
exception_type=type(e).__name__,
exception_arguments=e.args))
def provider_run(aliases_dict, tiid, method_name, provider_name):
provider = ProviderFactory.get_provider(provider_name)
# logger.info(u"in provider_run for {provider}".format(
# provider=provider.provider_name))
(success, estimated_wait_seconds) = rate.acquire(provider_name, block=False)
# add up to random 2 seconds to spread it out
estimated_wait_seconds += random.random() * 3
if not success:
logger.warning(u"RATE LIMIT HIT in provider_run for {provider} {method_name} {tiid}, retrying".format(
provider=provider.provider_name, method_name=method_name, tiid=tiid))
provider_run.retry(args=[aliases_dict, tiid, method_name, provider_name],
countdown=estimated_wait_seconds,
max_retries=10)
timeout_seconds = 30
try:
with timeout.Timeout(timeout_seconds):
response = provider_method_wrapper(tiid, aliases_dict, provider, method_name)
except timeout.Timeout:
msg = u"TIMEOUT in provider_run for {provider} {method_name} {tiid} after {timeout_seconds} seconds".format(
provider=provider.provider_name, method_name=method_name, tiid=tiid, timeout_seconds=timeout_seconds)
# logger.warning(msg) # message is written elsewhere
raise ProviderTimeout(msg)
return response
def provider_memberitems(provider_name):
"""
Starts a memberitems update for a specified provider, using a supplied file.
Returns a hash of the file's contents, which is needed to get memberitems'
output. To get output, poll GET /provider/<provider_name>/memberitems/<hash>?method=async
"""
# logger.debug("Query POSTed to {provider_name}/memberitems with request headers '{headers}'".format(
# provider_name=provider_name,
# headers=request.headers
# ))
file = request.files["file"]
logger.debug("In provider_memberitems got file")
logger.debug("filename = " + file.filename)
query = file.read().decode("utf-8")
provider = ProviderFactory.get_provider(provider_name)
memberitems = MemberItems(provider, myredis)
query_hash = memberitems.start_update(query)
response_dict = {"query_hash": query_hash}
resp = make_response(json.dumps(response_dict), 201) # created
resp.mimetype = "application/json"
resp.headers["Access-Control-Allow-Origin"] = "*"
return resp
def provider_memberitems(provider_name):
"""
Make a descr string (like bibtex) into a dict strings describing items.
"""
provider = ProviderFactory.get_provider(provider_name)
items_dict = provider.parse(request.json["descr"])
resp = make_response(
json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200)
return resp
def provider_memberitems(provider_name):
"""
Make a descr string (like bibtex) into a dict strings describing items.
"""
provider = ProviderFactory.get_provider(provider_name)
items_dict = provider.parse(request.json["descr"])
resp = make_response(
json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4),
200
)
return resp
def provider_memberitems(provider_name):
query = request.values.get('query','')
logger.debug("In provider_memberitems with " + query)
provider = ProviderFactory.get_provider(provider_name)
logger.debug("provider: " + provider.provider_name)
memberitems = provider.member_items(query, cache_enabled=False)
resp = make_response( json.dumps(memberitems, sort_keys=True, indent=4), 200 )
resp.mimetype = "application/json"
return resp
def provider_biblio(provider_name, id):
provider = ProviderFactory.get_provider(provider_name)
if id=="example":
id = provider.example_id[1]
url = "http://localhost:8080/" + provider_name + "/biblio?%s"
else:
url = None
biblio = provider.get_biblio_for_id(id, url, cache_enabled=False)
resp = make_response( json.dumps(biblio, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
def provider_memberitems(provider_name):
"""
Make a file into a dict strings describing items.
"""
mixpanel.track("Trigger:Import", {"Provider":provider_name}, request)
file = request.files['file']
logger.debug("In"+provider_name+"/memberitems, got file: filename="+file.filename)
entries_str = file.read().decode("utf-8")
provider = ProviderFactory.get_provider(provider_name)
items_dict = provider.parse(entries_str)
resp = make_response(json.dumps(items_dict), 200) # created
resp.mimetype = "application/json"
resp.headers['Access-Control-Allow-Origin'] = "*"
return resp
def provider_aliases(provider_name, id):
provider = ProviderFactory.get_provider(provider_name)
if id=="example":
id = provider.example_id[1]
url = "http://localhost:8080/" + provider_name + "/aliases?%s"
else:
url = None
try:
new_aliases = provider._get_aliases_for_id(id, url, cache_enabled=False)
except NotImplementedError:
new_aliases = []
all_aliases = [(provider.example_id[0], id)] + new_aliases
resp = make_response( json.dumps(all_aliases, sort_keys=True, indent=4) )
resp.mimetype = "application/json"
return resp
def provider_memberitems_get(provider_name, query):
provider = ProviderFactory.get_provider(provider_name)
memberitems = MemberItems(provider, myredis)
method = request.args.get("method", "sync")
try:
ret = getattr(memberitems, "get_" + method)(query)
except ProviderItemNotFoundError:
abort(404)
except ProviderError:
abort(500)
if ret:
if ret["error"]:
abort(503) # crossref lookup error, might be transient
resp = make_response(json.dumps(ret, sort_keys=True, indent=4), 200)
resp.mimetype = "application/json"
resp.headers["Access-Control-Allow-Origin"] = "*"
return resp
def provider_memberitems_get(provider_name, query):
"""
Gets aliases associated with a query from a given provider.
"""
query = unicode_helpers.remove_nonprinting_characters(query)
provider = ProviderFactory.get_provider(provider_name)
try:
items_dict = provider.member_items(query)
except ProviderItemNotFoundError:
abort_custom(404, "item not found")
except (ProviderTimeout, ProviderServerError):
abort_custom(503, "crossref lookup error, might be transient")
except ProviderError:
abort(500, "internal error from provider")
resp = make_response(
json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4), 200)
return resp
def importer_post(provider_name):
"""
Gets aliases associated with a query from a given provider.
"""
input_string = request.json["input"]
if provider_name == "pmids":
provider_name = "pubmed"
elif provider_name == "dois":
provider_name = "crossref"
elif provider_name == "urls":
provider_name = "webpage"
try:
provider = ProviderFactory.get_provider(provider_name)
except ImportError:
abort_custom(
404,
"an importer for provider '{provider_name}' is not found".format(
provider_name=provider_name))
try:
aliases = provider.member_items(input_string)
except ProviderItemNotFoundError:
abort_custom(404, "item not found")
except (ProviderTimeout, ProviderServerError):
abort_custom(503, "timeout error, might be transient")
except ProviderError:
abort(500, "internal error from provider")
tiids_aliases_map = item_module.create_tiids_from_aliases(aliases, myredis)
logger.debug(u"in provider_importer_get with {tiids_aliases_map}".format(
tiids_aliases_map=tiids_aliases_map))
products_dict = format_into_products_dict(tiids_aliases_map)
resp = make_response(
json.dumps({"products": products_dict}, sort_keys=True, indent=4), 200)
return resp
def provider_memberitems_get(provider_name, query):
"""
Gets aliases associated with a query from a given provider.
"""
mixpanel.track("Trigger:Import", {"Provider":provider_name}, request)
try:
provider = ProviderFactory.get_provider(provider_name)
ret = provider.member_items(query)
except ProviderItemNotFoundError:
abort(404)
except (ProviderTimeout, ProviderServerError):
abort(503) # crossref lookup error, might be transient
except ProviderError:
abort(500)
resp = make_response(
json.dumps({"memberitems":ret}, sort_keys=True, indent=4),
200
)
resp.mimetype = "application/json"
resp.headers['Access-Control-Allow-Origin'] = "*"
return resp
def provider_memberitems_get(provider_name, query):
"""
Gets aliases associated with a query from a given provider.
"""
query = unicode_helpers.remove_nonprinting_characters(query)
provider = ProviderFactory.get_provider(provider_name)
try:
items_dict = provider.member_items(query)
except ProviderItemNotFoundError:
abort_custom(404, "item not found")
except (ProviderTimeout, ProviderServerError):
abort_custom(503, "crossref lookup error, might be transient")
except ProviderError:
abort(500, "internal error from provider")
resp = make_response(
json.dumps({"memberitems": items_dict}, sort_keys=True, indent=4),
200
)
return resp
def setUp(self):
self.provider = ProviderFactory.get_provider(self.provider_name)
self.old_http_get = Provider.http_get
def test_get_provider(self):
provider = ProviderFactory.get_provider("wikipedia")
assert_equals(provider.__class__.__name__, "Wikipedia")
def test_08_get_provider(self):
provider = ProviderFactory.get_provider("wikipedia")
assert_equals(provider.__class__.__name__, "Wikipedia")
def setUp(self):
self.provider = ProviderFactory.get_provider(self.provider_name)
self.old_http_get = Provider.http_get
