def item_namespace_post_with_tiid(namespace, nid):
mixpanel.track("Create:Item", {"Namespace":namespace, "Source":"/item post"}, request)
tiid = item_module.create_item_from_namespace_nid(namespace, nid, myredis, mydao)
response_code = 201 # Created
resp = make_response(json.dumps(tiid), response_code)
resp.mimetype = "application/json"
return resp
def update_least_recently_updated(number_to_update, myredis, mydao):
(tiids_to_update, docs) = get_least_recently_updated_tiids_in_db(number_to_update, mydao)
update_docs_with_updater_timestamp(docs, mydao)
QUEUE_DELAY_IN_SECONDS = 0.25
mixpanel.track("Trigger:Update", {"Number Items":len(tiids_to_update), "Update Type":"Scheduled Least Recently"})
item.start_item_update(tiids_to_update, myredis, mydao, sleep_in_seconds=QUEUE_DELAY_IN_SECONDS)
return tiids_to_update
def create_item(namespace, nid, myredis, mydao):
logger.debug("In create_item with alias" + str((namespace, nid)))
item = make()
namespace = clean_id(namespace)
nid = clean_id(nid)
item["aliases"][namespace] = [nid]
item["aliases"] = canonical_aliases(item["aliases"])
# set this so we know when it's still updating later on
myredis.set_num_providers_left(
item["_id"],
ProviderFactory.num_providers_with_metrics(default_settings.PROVIDERS)
)
mydao.save(item)
myredis.add_to_alias_queue(item["_id"], item["aliases"])
logger.info("Created new item '{id}' with alias '{alias}'".format(
id=item["_id"],
alias=str((namespace, nid))
))
mixpanel.track("Create:Item", {"Namespace":namespace})
try:
return item["_id"]
except AttributeError:
abort(500)
def collection_create():
"""
POST /collection
creates new collection
"""
response_code = None
coll, key = collection.make(request.json.get("owner", None))
refset_metadata = request.json.get("refset_metadata", None)
if refset_metadata:
coll["refset_metadata"] = refset_metadata
coll["ip_address"] = request.remote_addr
try:
coll["title"] = request.json["title"]
aliases = request.json["aliases"]
(tiids, new_items) = item_module.create_or_update_items_from_aliases(aliases, myredis, mydao)
for item in new_items:
namespaces = item["aliases"].keys()
mixpanel.track("Create:Item", {"Namespace":namespaces[0], "Source":"collection"}, request)
if not tiids:
abort(404, "POST /collection requires a list of [namespace, id] pairs.")
except (AttributeError, TypeError):
# we got missing or improperly formated data.
logger.error(
"we got missing or improperly formated data: '{id}' with {json}.".format(
id=coll["_id"],
json=str(request.json)))
abort(404, "Missing arguments.")
aliases_strings = [namespace+":"+nid for (namespace, nid) in aliases]
# save dict of alias:tiid
coll["alias_tiids"] = dict(zip(aliases_strings, tiids))
logger.info(json.dumps(coll, sort_keys=True, indent=4))
mydao.save(coll)
response_code = 201 # Created
logger.info(
"saved new collection '{id}' with {num_items} items.".format(
id=coll["_id"],
num_items=len(coll["alias_tiids"])
))
mixpanel.track("Create:Report", {"Report Id":coll["_id"]}, request)
resp = make_response(json.dumps({"collection":coll, "key":key},
sort_keys=True, indent=4), response_code)
resp.mimetype = "application/json"
return resp
def provider_memberitems(provider_name):
"""
Make a file into a dict strings describing items.
"""
mixpanel.track("Trigger:Import", {"Provider":provider_name}, request)
file = request.files['file']
logger.debug("In"+provider_name+"/memberitems, got file: filename="+file.filename)
entries_str = file.read().decode("utf-8")
provider = ProviderFactory.get_provider(provider_name)
items_dict = provider.parse(entries_str)
resp = make_response(json.dumps(items_dict), 200) # created
resp.mimetype = "application/json"
resp.headers['Access-Control-Allow-Origin'] = "*"
return resp
def register_item(alias, api_key, myredis, mydao):
if not is_valid_key(api_key, mydao):
raise InvalidApiKeyException
if is_registered(alias, api_key, mydao):
raise ItemAlreadyRegisteredToThisKey
(namespace, nid) = alias
tiid = item.get_tiid_by_alias(namespace, nid, mydao)
if not tiid:
if is_over_quota(api_key, mydao):
raise ApiLimitExceededException
else:
tiid = item.create_item(namespace, nid, myredis, mydao)
registered = add_registration_data(alias, tiid, api_key, mydao)
if registered:
mixpanel.track("Create:Register", {"Namespace":namespace,
"API Key":api_key})
return tiid
def collection_update(cid=""):
# first, get the tiids in this collection:
try:
collection = mydao.get(cid)
tiids = collection["alias_tiids"].values()
except Exception:
logger.exception("couldn't get tiids in POST collection '{cid}'".format(
cid=cid
))
abort(404, "couldn't get tiids for this collection...maybe doesn't exist?")
mixpanel.track("Trigger:Update",
{"Number Items":len(tiids), "Report Id":cid, "Update Type":"webapp"},
request)
item_module.start_item_update(tiids, myredis, mydao)
resp = make_response("true", 200)
resp.mimetype = "application/json"
return resp
def update_user(userid=''):
"""
PUT /user
creates new user
"""
new_stuff = request.json
try:
key = new_stuff["key"]
except KeyError:
abort(400, "the submitted user object is missing required properties.")
try:
res = UserFactory.put(new_stuff, key, mydao)
mixpanel.track("Create:User", {}, request)
except NotAuthenticatedError:
abort(403, "You've got the wrong password.")
except AttributeError:
abort(400, "the submitted user object is missing required properties.")
resp = make_response(json.dumps(res, indent=4), 200)
resp.mimetype = "application/json"
return resp
def update_active_publisher_items(number_to_update, myredis, mydao):
all_tiids = []
all_docs = []
for publisher in active_publishers:
for journal_dict in active_publishers[publisher]["journals"]:
min_year = 2011 #only update 2012-2013 right now
(tiids_from_doi_prefix, docs_from_doi_prefix) = get_least_recently_updated_items_from_doi_prefix(min_year, journal_dict["doi_prefix"], myredis, mydao)
logger.info("doi prefix {prefix} has {num_tiids} items published since {min_year} last updated more than 24 hours ago".format(
num_tiids=len(tiids_from_doi_prefix), prefix=journal_dict["doi_prefix"], min_year=min_year))
all_tiids += tiids_from_doi_prefix
all_docs += docs_from_doi_prefix
print "recent items for active publishers that were last updated more than a day ago, n=", len(all_tiids)
tiids_to_update = all_tiids[0:min(number_to_update, len(all_tiids))]
docs_to_update = all_docs[0:min(number_to_update, len(all_docs))]
response = update_docs_with_updater_timestamp(docs_to_update, mydao)
print "updating {number_to_update} of them now".format(number_to_update=number_to_update)
QUEUE_DELAY_IN_SECONDS = 0.25
mixpanel.track("Trigger:Update", {"Number Items":len(tiids_to_update), "Update Type":"Scheduled Registered"})
item.start_item_update(tiids_to_update, myredis, mydao, sleep_in_seconds=QUEUE_DELAY_IN_SECONDS)
return tiids_to_update
def provider_memberitems_get(provider_name, query):
"""
Gets aliases associated with a query from a given provider.
"""
mixpanel.track("Trigger:Import", {"Provider":provider_name}, request)
try:
provider = ProviderFactory.get_provider(provider_name)
ret = provider.member_items(query)
except ProviderItemNotFoundError:
abort(404)
except (ProviderTimeout, ProviderServerError):
abort(503) # crossref lookup error, might be transient
except ProviderError:
abort(500)
resp = make_response(
json.dumps({"memberitems":ret}, sort_keys=True, indent=4),
200
)
resp.mimetype = "application/json"
resp.headers['Access-Control-Allow-Origin'] = "*"
return resp
