def analyze_user(intake_user_key):
#logging.debug("Analyzing User Key: %s" % intake_user_key)
intake_user = intake_user_key.get()
counter_key = "analysis::run_count::%s" % intake_user.org.urlsafe()
count = CombinedUser.query(CombinedUser.intake_users == intake_user.key).count(1)
if not count:
# Generate a legit report
legit_report, intake_users = get_legit_report(intake_user, api_type="api")
# Create a combined user
cu = CombinedUser.create(intake_users, legit_report)
cu.put()
# Check one off the list
#memcache.decr(counter_key)
return "Great Success"
def analyze_user(intake_user_key):
#logging.debug("Analyzing User Key: %s" % intake_user_key)
intake_user = intake_user_key.get()
counter_key = "analysis::run_count::%s" % intake_user.org.urlsafe()
count = CombinedUser.query(
CombinedUser.intake_users == intake_user.key).count(1)
if not count:
# Generate a legit report
legit_report, intake_users = get_legit_report(intake_user,
api_type="api")
# Create a combined user
cu = CombinedUser.create(intake_users, legit_report)
cu.put()
# Check one off the list
#memcache.decr(counter_key)
return "Great Success"
def run_analysis(org_key=None):
"""
Task that deletes all existing CombinedUsers and regenerates them from the
underlying IntakeUsers. This task can take a long time to run, so it should
be run on a dedicated instance.
"""
DELETE_BATCH = 500
ANALYZE_BATCH = 50
# Clear out the existing combined users
cu_query = CombinedUser.query()
if org_key:
org_key = ndb.Key(urlsafe=org_key)
cu_query = cu_query.filter(CombinedUser.orgs == org_key)
while True:
results, cursor, more = cu_query.fetch_page(DELETE_BATCH,
keys_only=True)
ndb.delete_multi(results)
if not more:
break
# Analyze all the intake users, on a per-organization basis
if org_key:
org_keys = [org_key]
else:
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
counter_key = "analysis::run_count::%s" % org_key.urlsafe()
memcache.set(key=counter_key, value=0)
iu_query = IntakeUser.query(IntakeUser.org == org_key)
for iu_key in iu_query.iter(keys_only=True):
memcache.incr(counter_key)
#deferred.defer(analyze_user, intake_user_key=iu_key)
analyze_user(iu_key)
generate_csv(org_key.get().name)
return "Great Success"
def run_analysis(org_key=None):
"""
Task that deletes all existing CombinedUsers and regenerates them from the
underlying IntakeUsers. This task can take a long time to run, so it should
be run on a dedicated instance.
"""
DELETE_BATCH = 500
ANALYZE_BATCH = 50
# Clear out the existing combined users
cu_query = CombinedUser.query()
if org_key:
org_key = ndb.Key(urlsafe=org_key)
cu_query = cu_query.filter(CombinedUser.orgs == org_key)
while True:
results, cursor, more = cu_query.fetch_page(DELETE_BATCH, keys_only=True)
ndb.delete_multi(results)
if not more:
break
# Analyze all the intake users, on a per-organization basis
if org_key:
org_keys = [org_key]
else:
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
counter_key = "analysis::run_count::%s" % org_key.urlsafe()
memcache.set(key=counter_key, value=0)
iu_query = IntakeUser.query(IntakeUser.org == org_key)
for iu_key in iu_query.iter(keys_only=True):
memcache.incr(counter_key)
#deferred.defer(analyze_user, intake_user_key=iu_key)
analyze_user(iu_key)
generate_csv(org_key.get().name)
return "Great Success"
def generate_csv(org_name):
org = Organization.query(Organization.name == org_name).get()
combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
for cu in combined_users:
org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
has_facebook = 1 if cu.facebook_id else 0
has_twitter = 1 if cu.twitter_id else 0
has_linkedin = 1 if cu.linkedin_id else 0
rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" %
(org_user_id, cu.legit_report["legitscore"],
cu.legit_report["lrg_reputation"]["marketplace_memberships"],
cu.legit_report["lrg_reputation"]["total_transactions"],
cu.legit_report["lrg_reputation"]["total_reviews"],
cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
has_facebook, has_twitter, has_linkedin))
#logging.info("CSV FILE")
#logging.info(rv)
file_name = files.blobstore.create(
mime_type='text/plain',
_blobinfo_uploaded_filename="%s-analysis" % org_name)
with files.open(file_name, 'a') as f:
f.write(rv)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
ar = AnalysisReport.query(
AnalysisReport.org_name == org_name).get() or AnalysisReport(
org_name=org_name)
ar.blob_key = blob_key
ar.put()
def generate_csv(org_name):
org = Organization.query(Organization.name == org_name).get()
combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
for cu in combined_users:
org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
has_facebook = 1 if cu.facebook_id else 0
has_twitter = 1 if cu.twitter_id else 0
has_linkedin = 1 if cu.linkedin_id else 0
rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" %
(org_user_id,
cu.legit_report["legitscore"],
cu.legit_report["lrg_reputation"]["marketplace_memberships"],
cu.legit_report["lrg_reputation"]["total_transactions"],
cu.legit_report["lrg_reputation"]["total_reviews"],
cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
has_facebook,
has_twitter,
has_linkedin))
#logging.info("CSV FILE")
#logging.info(rv)
file_name = files.blobstore.create(mime_type='text/plain',
_blobinfo_uploaded_filename="%s-analysis" % org_name)
with files.open(file_name, 'a') as f:
f.write(rv)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
ar = AnalysisReport.query(AnalysisReport.org_name == org_name).get() or AnalysisReport(org_name=org_name)
ar.blob_key = blob_key
ar.put()