def linked_accounts(account_type, url_slug=None, min_url_slug=None): column_name = account_type+"_id" if url_slug: q = db.session.query(Profile).filter(getattr(Profile, column_name) != None).filter(Profile.url_slug==url_slug) else: if min_url_slug: q = db.session.query(Profile).filter(getattr(Profile, column_name) != None).filter(Profile.url_slug>=min_url_slug) else: q = db.session.query(Profile).filter(getattr(Profile, column_name) != None) number_considered = 0.0 for profile in windowed_query(q, Profile.url_slug, 25): number_considered += 1 logger.info(u"{url_slug} previous number of account products: {num}".format( url_slug=profile.url_slug, num=len(profile.account_products))) existing_account_product_list = [p for p in profile.account_products if p.index_name==account_type] if existing_account_product_list: existing_account_product = existing_account_product_list[0] if existing_account_product.followers: logger.info(u"{url_slug} already has an account_product for {account_type}, so skipping".format( url_slug=profile.url_slug, account_type=account_type)) else: logger.info(u"{url_slug} already has an account_product for {account_type}, but no followers, so refreshing".format( url_slug=profile.url_slug, account_type=account_type)) refresh_products_from_tiids(existing_account_product.profile_id, [existing_account_product.tiid], source="scheduled") else: logger.info(u"{url_slug} had no account_product for {account_type}, so adding".format( url_slug=profile.url_slug, account_type=account_type)) tiids = profile.update_products_from_linked_account(account_type, update_even_removed_products=False) if tiids: logger.info(u"{url_slug} added {num} products for {account_type}".format( url_slug=profile.url_slug, num=len(tiids), account_type=account_type))
def collect_new_mendeley(url_slug=None, min_url_slug=None): if url_slug: q = db.session.query(Profile).filter(Profile.url_slug==url_slug) else: q = db.session.query(Profile).filter(or_(Profile.is_advisor!=None, Profile.stripe_id!=None)) if min_url_slug: q = q.filter(Profile.url_slug>=min_url_slug) start_time = datetime.datetime.utcnow() number_profiles = 0.0 total_refreshes = 0 for profile in windowed_query(q, Profile.url_slug, 25): number_profiles += 1 number_refreshes = 0.0 print profile.url_slug, profile.id for product in profile.display_products: if product.get_metric_by_name("mendeley", "readers"): number_refreshes += 1 refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled") if number_refreshes: total_refreshes += number_refreshes pause_length = min(number_refreshes * 2, 60) print "pausing", pause_length, "seconds after refreshing", number_refreshes, "products" time.sleep(pause_length) print total_refreshes, "total refreshes across", number_profiles, "profiles" elapsed_seconds = (datetime.datetime.utcnow() - start_time).seconds print "elapsed seconds=", elapsed_seconds, "; number profiles per second=", number_profiles/(0.1+elapsed_seconds)
def refresh_twitter(min_tiid=None): if min_tiid: q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid>min_tiid) else: q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid.in_(tiids_that_need_twitter)) start_time = datetime.datetime.utcnow() number_considered = 0.0 number_refreshed = 0 for product in windowed_query(q, Product.tiid, 25): number_considered += 1 try: if product.biblio.repository=="Twitter" and len(product.metrics)==0: print "refreshing", product.tiid, number_refreshed refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled") number_refreshed += 1 if number_refreshed >= 15: #api limit print "refreshed 15, so breaking now" break except AttributeError: pass
def refresh_tweeted_products(min_tiid=None): if min_tiid: q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid>min_tiid) else: q = db.session.query(Product).filter(Product.profile_id != None) start_time = datetime.datetime.utcnow() number_considered = 0.0 number_refreshed = 0 for product in windowed_query(q, Product.tiid, 25): number_considered += 1 try: if product.get_metric_by_name("altmetric_com", "tweets"): print number_refreshed, ". refreshing: ", product.tiid refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled") number_refreshed += 1 time.sleep(0.5) elapsed_seconds = (datetime.datetime.utcnow() - start_time).seconds print "elapsed seconds=", elapsed_seconds, "; number per second=", number_considered/(0.1+elapsed_seconds) except AttributeError: pass
def refresh_tiid(tiid): tiids = refresh_products_from_tiids(None, [tiid]) print tiids return tiids
def refresh_products(self, source="webapp"): analytics_credentials = self.get_analytics_credentials() resp = refresh_products_from_tiids(self.id, self.tiids, analytics_credentials, source) save_profile_last_refreshed_timestamp(self.id) return resp