def after_refresh_complete(tiid, failure_message=None):
# logger.info(u"here in after_refresh_complete with {tiid}".format(
# tiid=tiid))
product = Product.query.get(tiid)
if not product:
logger.warning(u"Empty product in after_refresh_complete for tiid {tiid}".format(
tiid=tiid))
return None
product.embed_markup = product.get_embed_markup()
product.set_refresh_status(myredis, failure_message) #need commit after this
db.session.merge(product)
commit(db)
sleep_seconds = random.random() * 10
logger.info(u"Sleeping {sleep_seconds}s in after_refresh_complete for {tiid} before checking done refreshes".format(
sleep_seconds=sleep_seconds, tiid=tiid))
time.sleep(sleep_seconds)
profile = Profile.query.get(product.profile_id)
if not profile:
print "\n\n-------> no profile after done all refreshes?!?", product.profile_id, "\n\n\n---------------\n\n\n"
return None
refresh_status = profile.get_refresh_status()
if refresh_status.is_done_refreshing and refresh_status.refresh_state == "progress bar":
print "\n\n-------> done all refreshes", product.profile_id, "\n\n\n---------------\n\n\n"
logger.info(u"just_finished_profile_refresh for {tiid}, now deduping etc".format(
tiid=tiid))
save_profile_refresh_status(profile, RefreshStatus.states["CRUNCHING"])
logger.info(u"deduplicating for {url_slug}".format(
url_slug=profile.url_slug))
deleted_tiids = profile.remove_duplicates()
logger.info(u"parse_and_save_tweets for {url_slug}".format(
url_slug=profile.url_slug))
profile.parse_and_save_tweets()
save_profile_refresh_status(profile, RefreshStatus.states["ALL_DONE"])
def get_new_products(self, provider_name, product_seeds, analytics_credentials={}, add_even_if_removed=False):
save_profile_refresh_status(self, RefreshStatus.states["PROGRESS_BAR"])
if add_even_if_removed:
tiids_to_exclude = self.tiids
else:
tiids_to_exclude = self.tiids_including_removed # don't re-import dup or removed products
try:
new_products = import_and_create_products(
self.id,
provider_name,
product_seeds,
analytics_credentials,
tiids_to_exclude)
except (ImportError, ProviderError):
new_products = []
return new_products
def done_all_refreshes(profile_id):
print "\n\n-------> done all refreshes", profile_id, "\n\n\n---------------\n\n\n"
profile = Profile.query.get(profile_id)
save_profile_refresh_status(profile, RefreshStatus.states["CRUNCHING"])
logger.info(u"deduplicating for {url_slug}".format(
url_slug=profile.url_slug))
deleted_tiids = profile.remove_duplicates()
logger.info(u"parse_and_save_tweets for {url_slug}".format(
url_slug=profile.url_slug))
profile.parse_and_save_tweets()
save_profile_refresh_status(profile, RefreshStatus.states["ALL_DONE"])
return
def refresh_products_from_tiids(profile_id, tiids, analytics_credentials={}, source="webapp"):
# assume the profile is the same one as the first product
if not profile_id:
temp_profile = Product.query.get(tiids[0])
profile_id = temp_profile.profile_id
from totalimpactwebapp.profile import Profile
profile = Profile.query.get(profile_id)
save_profile_refresh_status(profile, RefreshStatus.states["PROGRESS_BAR"])
if not tiids:
return None
priority = "high"
if source=="scheduled":
priority = "low"
products = Product.query.filter(Product.tiid.in_(tiids)).all()
tiids_to_update = []
for product in products:
try:
tiid = product.tiid
product.set_last_refresh_start()
db.session.merge(product)
tiids_to_update += [tiid]
except AttributeError:
logger.debug(u"couldn't find tiid {tiid} so not refreshing its metrics".format(
tiid=tiid))
db.session.commit()
start_product_update(profile_id, tiids_to_update, priority)
return tiids
