def item_namespace_post_with_tiid(namespace, nid): mixpanel.track("Create:Item", {"Namespace":namespace, "Source":"/item post"}, request) tiid = item_module.create_item_from_namespace_nid(namespace, nid, myredis, mydao) response_code = 201 # Created resp = make_response(json.dumps(tiid), response_code) resp.mimetype = "application/json" return resp
def update_least_recently_updated(number_to_update, myredis, mydao): (tiids_to_update, docs) = get_least_recently_updated_tiids_in_db(number_to_update, mydao) update_docs_with_updater_timestamp(docs, mydao) QUEUE_DELAY_IN_SECONDS = 0.25 mixpanel.track("Trigger:Update", {"Number Items":len(tiids_to_update), "Update Type":"Scheduled Least Recently"}) item.start_item_update(tiids_to_update, myredis, mydao, sleep_in_seconds=QUEUE_DELAY_IN_SECONDS) return tiids_to_update
def create_item(namespace, nid, myredis, mydao): logger.debug("In create_item with alias" + str((namespace, nid))) item = make() namespace = clean_id(namespace) nid = clean_id(nid) item["aliases"][namespace] = [nid] item["aliases"] = canonical_aliases(item["aliases"]) # set this so we know when it's still updating later on myredis.set_num_providers_left( item["_id"], ProviderFactory.num_providers_with_metrics(default_settings.PROVIDERS) ) mydao.save(item) myredis.add_to_alias_queue(item["_id"], item["aliases"]) logger.info("Created new item '{id}' with alias '{alias}'".format( id=item["_id"], alias=str((namespace, nid)) )) mixpanel.track("Create:Item", {"Namespace":namespace}) try: return item["_id"] except AttributeError: abort(500)
def collection_create(): """ POST /collection creates new collection """ response_code = None coll, key = collection.make(request.json.get("owner", None)) refset_metadata = request.json.get("refset_metadata", None) if refset_metadata: coll["refset_metadata"] = refset_metadata coll["ip_address"] = request.remote_addr try: coll["title"] = request.json["title"] aliases = request.json["aliases"] (tiids, new_items) = item_module.create_or_update_items_from_aliases(aliases, myredis, mydao) for item in new_items: namespaces = item["aliases"].keys() mixpanel.track("Create:Item", {"Namespace":namespaces[0], "Source":"collection"}, request) if not tiids: abort(404, "POST /collection requires a list of [namespace, id] pairs.") except (AttributeError, TypeError): # we got missing or improperly formated data. logger.error( "we got missing or improperly formated data: '{id}' with {json}.".format( id=coll["_id"], json=str(request.json))) abort(404, "Missing arguments.") aliases_strings = [namespace+":"+nid for (namespace, nid) in aliases] # save dict of alias:tiid coll["alias_tiids"] = dict(zip(aliases_strings, tiids)) logger.info(json.dumps(coll, sort_keys=True, indent=4)) mydao.save(coll) response_code = 201 # Created logger.info( "saved new collection '{id}' with {num_items} items.".format( id=coll["_id"], num_items=len(coll["alias_tiids"]) )) mixpanel.track("Create:Report", {"Report Id":coll["_id"]}, request) resp = make_response(json.dumps({"collection":coll, "key":key}, sort_keys=True, indent=4), response_code) resp.mimetype = "application/json" return resp
def provider_memberitems(provider_name): """ Make a file into a dict strings describing items. """ mixpanel.track("Trigger:Import", {"Provider":provider_name}, request) file = request.files['file'] logger.debug("In"+provider_name+"/memberitems, got file: filename="+file.filename) entries_str = file.read().decode("utf-8") provider = ProviderFactory.get_provider(provider_name) items_dict = provider.parse(entries_str) resp = make_response(json.dumps(items_dict), 200) # created resp.mimetype = "application/json" resp.headers['Access-Control-Allow-Origin'] = "*" return resp
def register_item(alias, api_key, myredis, mydao): if not is_valid_key(api_key, mydao): raise InvalidApiKeyException if is_registered(alias, api_key, mydao): raise ItemAlreadyRegisteredToThisKey (namespace, nid) = alias tiid = item.get_tiid_by_alias(namespace, nid, mydao) if not tiid: if is_over_quota(api_key, mydao): raise ApiLimitExceededException else: tiid = item.create_item(namespace, nid, myredis, mydao) registered = add_registration_data(alias, tiid, api_key, mydao) if registered: mixpanel.track("Create:Register", {"Namespace":namespace, "API Key":api_key}) return tiid
def collection_update(cid=""): # first, get the tiids in this collection: try: collection = mydao.get(cid) tiids = collection["alias_tiids"].values() except Exception: logger.exception("couldn't get tiids in POST collection '{cid}'".format( cid=cid )) abort(404, "couldn't get tiids for this collection...maybe doesn't exist?") mixpanel.track("Trigger:Update", {"Number Items":len(tiids), "Report Id":cid, "Update Type":"webapp"}, request) item_module.start_item_update(tiids, myredis, mydao) resp = make_response("true", 200) resp.mimetype = "application/json" return resp
def update_user(userid=''): """ PUT /user creates new user """ new_stuff = request.json try: key = new_stuff["key"] except KeyError: abort(400, "the submitted user object is missing required properties.") try: res = UserFactory.put(new_stuff, key, mydao) mixpanel.track("Create:User", {}, request) except NotAuthenticatedError: abort(403, "You've got the wrong password.") except AttributeError: abort(400, "the submitted user object is missing required properties.") resp = make_response(json.dumps(res, indent=4), 200) resp.mimetype = "application/json" return resp
def update_active_publisher_items(number_to_update, myredis, mydao): all_tiids = [] all_docs = [] for publisher in active_publishers: for journal_dict in active_publishers[publisher]["journals"]: min_year = 2011 #only update 2012-2013 right now (tiids_from_doi_prefix, docs_from_doi_prefix) = get_least_recently_updated_items_from_doi_prefix(min_year, journal_dict["doi_prefix"], myredis, mydao) logger.info("doi prefix {prefix} has {num_tiids} items published since {min_year} last updated more than 24 hours ago".format( num_tiids=len(tiids_from_doi_prefix), prefix=journal_dict["doi_prefix"], min_year=min_year)) all_tiids += tiids_from_doi_prefix all_docs += docs_from_doi_prefix print "recent items for active publishers that were last updated more than a day ago, n=", len(all_tiids) tiids_to_update = all_tiids[0:min(number_to_update, len(all_tiids))] docs_to_update = all_docs[0:min(number_to_update, len(all_docs))] response = update_docs_with_updater_timestamp(docs_to_update, mydao) print "updating {number_to_update} of them now".format(number_to_update=number_to_update) QUEUE_DELAY_IN_SECONDS = 0.25 mixpanel.track("Trigger:Update", {"Number Items":len(tiids_to_update), "Update Type":"Scheduled Registered"}) item.start_item_update(tiids_to_update, myredis, mydao, sleep_in_seconds=QUEUE_DELAY_IN_SECONDS) return tiids_to_update
def provider_memberitems_get(provider_name, query): """ Gets aliases associated with a query from a given provider. """ mixpanel.track("Trigger:Import", {"Provider":provider_name}, request) try: provider = ProviderFactory.get_provider(provider_name) ret = provider.member_items(query) except ProviderItemNotFoundError: abort(404) except (ProviderTimeout, ProviderServerError): abort(503) # crossref lookup error, might be transient except ProviderError: abort(500) resp = make_response( json.dumps({"memberitems":ret}, sort_keys=True, indent=4), 200 ) resp.mimetype = "application/json" resp.headers['Access-Control-Allow-Origin'] = "*" return resp