def linked_accounts(account_type, url_slug=None, min_url_slug=None):
column_name = account_type+"_id"
if url_slug:
q = db.session.query(Profile).filter(getattr(Profile, column_name) != None).filter(Profile.url_slug==url_slug)
else:
if min_url_slug:
q = db.session.query(Profile).filter(getattr(Profile, column_name) != None).filter(Profile.url_slug>=min_url_slug)
else:
q = db.session.query(Profile).filter(getattr(Profile, column_name) != None)
number_considered = 0.0
for profile in windowed_query(q, Profile.url_slug, 25):
number_considered += 1
logger.info(u"{url_slug} previous number of account products: {num}".format(
url_slug=profile.url_slug, num=len(profile.account_products)))
existing_account_product_list = [p for p in profile.account_products if p.index_name==account_type]
if existing_account_product_list:
existing_account_product = existing_account_product_list[0]
if existing_account_product.followers:
logger.info(u"{url_slug} already has an account_product for {account_type}, so skipping".format(
url_slug=profile.url_slug, account_type=account_type))
else:
logger.info(u"{url_slug} already has an account_product for {account_type}, but no followers, so refreshing".format(
url_slug=profile.url_slug, account_type=account_type))
refresh_products_from_tiids(existing_account_product.profile_id, [existing_account_product.tiid], source="scheduled")
else:
logger.info(u"{url_slug} had no account_product for {account_type}, so adding".format(
url_slug=profile.url_slug, account_type=account_type))
tiids = profile.update_products_from_linked_account(account_type, update_even_removed_products=False)
if tiids:
logger.info(u"{url_slug} added {num} products for {account_type}".format(
url_slug=profile.url_slug, num=len(tiids), account_type=account_type))
def collect_new_mendeley(url_slug=None, min_url_slug=None):
if url_slug:
q = db.session.query(Profile).filter(Profile.url_slug==url_slug)
else:
q = db.session.query(Profile).filter(or_(Profile.is_advisor!=None, Profile.stripe_id!=None))
if min_url_slug:
q = q.filter(Profile.url_slug>=min_url_slug)
start_time = datetime.datetime.utcnow()
number_profiles = 0.0
total_refreshes = 0
for profile in windowed_query(q, Profile.url_slug, 25):
number_profiles += 1
number_refreshes = 0.0
print profile.url_slug, profile.id
for product in profile.display_products:
if product.get_metric_by_name("mendeley", "readers"):
number_refreshes += 1
refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled")
if number_refreshes:
total_refreshes += number_refreshes
pause_length = min(number_refreshes * 2, 60)
print "pausing", pause_length, "seconds after refreshing", number_refreshes, "products"
time.sleep(pause_length)
print total_refreshes, "total refreshes across", number_profiles, "profiles"
elapsed_seconds = (datetime.datetime.utcnow() - start_time).seconds
print "elapsed seconds=", elapsed_seconds, "; number profiles per second=", number_profiles/(0.1+elapsed_seconds)
def refresh_twitter(min_tiid=None):
if min_tiid:
q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid>min_tiid)
else:
q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid.in_(tiids_that_need_twitter))
start_time = datetime.datetime.utcnow()
number_considered = 0.0
number_refreshed = 0
for product in windowed_query(q, Product.tiid, 25):
number_considered += 1
try:
if product.biblio.repository=="Twitter" and len(product.metrics)==0:
print "refreshing", product.tiid, number_refreshed
refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled")
number_refreshed += 1
if number_refreshed >= 15:
#api limit
print "refreshed 15, so breaking now"
break
except AttributeError:
pass
def refresh_tweeted_products(min_tiid=None):
if min_tiid:
q = db.session.query(Product).filter(Product.profile_id != None).filter(Product.tiid>min_tiid)
else:
q = db.session.query(Product).filter(Product.profile_id != None)
start_time = datetime.datetime.utcnow()
number_considered = 0.0
number_refreshed = 0
for product in windowed_query(q, Product.tiid, 25):
number_considered += 1
try:
if product.get_metric_by_name("altmetric_com", "tweets"):
print number_refreshed, ". refreshing: ", product.tiid
refresh_products_from_tiids(product.profile_id, [product.tiid], source="scheduled")
number_refreshed += 1
time.sleep(0.5)
elapsed_seconds = (datetime.datetime.utcnow() - start_time).seconds
print "elapsed seconds=", elapsed_seconds, "; number per second=", number_considered/(0.1+elapsed_seconds)
except AttributeError:
pass
def refresh_tiid(tiid):
tiids = refresh_products_from_tiids(None, [tiid])
print tiids
return tiids
def refresh_products(self, source="webapp"):
analytics_credentials = self.get_analytics_credentials()
resp = refresh_products_from_tiids(self.id, self.tiids, analytics_credentials, source)
save_profile_last_refreshed_timestamp(self.id)
return resp
