def profile_deets_live(args): url_slug = args.get("url_slug", None) min_url_slug = args.get("min_url_slug", None) q = profile_query(url_slug, min_url_slug) number_considered = 0.0 start_time = datetime.datetime.utcnow() profile_deets = [] for profile in windowed_query(q, Profile.url_slug, 25): logger.info(u"profile_deets: {url_slug}".format( url_slug=profile.url_slug)) profile_deets += [get_profile_summary_dict(profile)] db.session.expunge(profile) # print csv_of_dict(profile_deets) # with open("profile_deets.pickle", "wb") as handle: # pickle.dump(profile_deets, handle) # print json.dumps(profile_deets, sort_keys=True, indent=4) print "****" csv_contents = csv_of_dict(profile_deets) print csv_contents import tempfile temp_csv_file = tempfile.NamedTemporaryFile(delete=False) temp_csv_file.write(csv_contents) temp_csv_file.close() upload_to_s3(temp_csv_file.name, "exploring/profile_deets.csv") time.sleep(30)
def profile_deets(url_slug=None, min_url_slug=None, start_days_ago=44, end_days_ago=30): start_date = datetime.datetime.utcnow() - datetime.timedelta(days=start_days_ago) end_date = datetime.datetime.utcnow() - datetime.timedelta(days=end_days_ago) q = db.session.query(Profile) \ .filter(Profile.created.between(start_date, end_date)) if url_slug: q = db.session.query(Profile).filter(Profile.url_slug==url_slug) elif min_url_slug: q = q.filter(Profile.url_slug>=min_url_slug) profile_deets = [] for profile in windowed_query(q, Profile.url_slug, 25): logger.info(u"profile_deets: {url_slug}".format( url_slug=profile.url_slug)) profile_deets += [get_profile_summary_dict(profile)] db.session.expunge(profile) # print csv_of_dict(profile_deets) # with open("profile_deets.pickle", "wb") as handle: # pickle.dump(profile_deets, handle) # print json.dumps(profile_deets, sort_keys=True, indent=4) print "****" csv_contents = csv_of_dict(profile_deets) print csv_contents import tempfile temp_csv_file = tempfile.NamedTemporaryFile(delete=False) temp_csv_file.write(csv_contents) temp_csv_file.close() upload_to_s3(temp_csv_file.name, "exploring/profile_deets.csv") time.sleep(30)