def kickoff_compute_value():
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
url = url_for("compute_org_value", organization_key=org_key.urlsafe())
if DEVELOPMENT:
taskqueue.add(url=url)
else:
taskqueue.add(url=url, target="1.analyzer")
return redirect(url_for("value"))
def kickoff_compute_value():
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
url = url_for("compute_org_value", organization_key=org_key.urlsafe())
if DEVELOPMENT:
taskqueue.add(url=url)
else:
taskqueue.add(url=url, target="1.analyzer")
return redirect(url_for("value"))
def organizations_management():
if request.method == "POST":
form = CreateOrganizationForm(request.form)
if form.validate():
org = Organization(name=form.short_name.data, full_name=form.full_name.data)
org.put()
else:
form = CreateOrganizationForm()
existing_orgs = Organization.query().fetch(100)
return render_template("admin/organizations.html",
existing_orgs=existing_orgs,
form=form)
def organizations_management():
if request.method == "POST":
form = CreateOrganizationForm(request.form)
if form.validate():
org = Organization(name=form.short_name.data,
full_name=form.full_name.data)
org.put()
else:
form = CreateOrganizationForm()
existing_orgs = Organization.query().fetch(100)
return render_template("admin/organizations.html",
existing_orgs=existing_orgs,
form=form)
def get(self):
self.response.headers['Content-Type'] = 'application/json'
key = ndb.Key(urlsafe = self.request.get('entity'))
if key.kind() == 'Organization':
entity = Organization.get_by_key(key.urlsafe())
elif key.kind() == 'OU':
entity = OU.get_by_key(key.urlsafe())
elif key.kind() == 'CN':
entity = CN.get_by_key(key.urlsafe())
logging.warning('DEBUG OrganizationTree - key.kind(): %s' % key.kind())
if key.kind() in ['Organization', 'OU']:
name = entity.name
else:
name = entity.first_name + ' ' + entity.last_name
treedict = {'key': entity.key.urlsafe(), 'name': name, 'kind': key.kind()}
level = treedict
# Ascending tree
while key.kind() != 'Organization':
parent = entity.parent
if parent.kind() == 'Organization':
entity = Organization.query(Organization.key == parent).get()
else:
entity = OU.query(OU.key == parent).get()
key = entity.key
logging.warning('DEBUG OrganizationTree - parent: %s - entity: %s' % (parent, str(entity)))
entitydict = {'key': entity.key.urlsafe(), 'name': entity.name, 'kind': parent.kind()}
level['parent'] = entitydict
level = level['parent']
children = []
ouchildren = OU.query(OU.parent == key).fetch()
for ouchild in ouchildren:
oudict = {'key': ouchild.key.urlsafe(), 'name': ouchild.name, 'kind': ouchild.key.kind()}
children.append(oudict)
cnchildren = CN.query(CN.parent == key).fetch()
for cnchild in cnchildren:
cndict = {'key': cnchild.key.urlsafe(), 'name': cnchild.first_name + ' ' + cnchild.last_name, 'kind': cnchild.key.kind()}
children.append(cndict)
treedict['child'] = pop_list(treedict)
self.response.write(json.dumps(treedict))
def run_analysis(org_key=None):
"""
Task that deletes all existing CombinedUsers and regenerates them from the
underlying IntakeUsers. This task can take a long time to run, so it should
be run on a dedicated instance.
"""
DELETE_BATCH = 500
ANALYZE_BATCH = 50
# Clear out the existing combined users
cu_query = CombinedUser.query()
if org_key:
org_key = ndb.Key(urlsafe=org_key)
cu_query = cu_query.filter(CombinedUser.orgs == org_key)
while True:
results, cursor, more = cu_query.fetch_page(DELETE_BATCH,
keys_only=True)
ndb.delete_multi(results)
if not more:
break
# Analyze all the intake users, on a per-organization basis
if org_key:
org_keys = [org_key]
else:
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
counter_key = "analysis::run_count::%s" % org_key.urlsafe()
memcache.set(key=counter_key, value=0)
iu_query = IntakeUser.query(IntakeUser.org == org_key)
for iu_key in iu_query.iter(keys_only=True):
memcache.incr(counter_key)
#deferred.defer(analyze_user, intake_user_key=iu_key)
analyze_user(iu_key)
generate_csv(org_key.get().name)
return "Great Success"
def analysis():
orgs = Organization.query().fetch(50)
orgs_info = {}
for org in orgs:
total_users = IntakeUser.query(IntakeUser.org == org.key).count()
if total_users == 0:
continue
orgs_info[org.name] = {}
counter_key = "analysis::run_count::%s" % org.key.urlsafe()
orgs_info[org.name]["analyzed_users"] = memcache.get(counter_key)
orgs_info[org.name]["total_users"] = total_users
orgs_info[org.name]["org_key"] = org.key.urlsafe()
if request.method == "POST":
org_key = request.form.get("org_key", None)
taskqueue.add(url=url_for("run_analysis", org_key=org_key),
target="analyzer")
return redirect(url_for("analysis"))
return render_template("admin/analysis.html", orgs_info=orgs_info)
def run_analysis(org_key=None):
"""
Task that deletes all existing CombinedUsers and regenerates them from the
underlying IntakeUsers. This task can take a long time to run, so it should
be run on a dedicated instance.
"""
DELETE_BATCH = 500
ANALYZE_BATCH = 50
# Clear out the existing combined users
cu_query = CombinedUser.query()
if org_key:
org_key = ndb.Key(urlsafe=org_key)
cu_query = cu_query.filter(CombinedUser.orgs == org_key)
while True:
results, cursor, more = cu_query.fetch_page(DELETE_BATCH, keys_only=True)
ndb.delete_multi(results)
if not more:
break
# Analyze all the intake users, on a per-organization basis
if org_key:
org_keys = [org_key]
else:
org_keys = Organization.query().iter(keys_only=True)
for org_key in org_keys:
counter_key = "analysis::run_count::%s" % org_key.urlsafe()
memcache.set(key=counter_key, value=0)
iu_query = IntakeUser.query(IntakeUser.org == org_key)
for iu_key in iu_query.iter(keys_only=True):
memcache.incr(counter_key)
#deferred.defer(analyze_user, intake_user_key=iu_key)
analyze_user(iu_key)
generate_csv(org_key.get().name)
return "Great Success"
def analysis():
orgs = Organization.query().fetch(50)
orgs_info = {}
for org in orgs:
total_users = IntakeUser.query(IntakeUser.org == org.key).count()
if total_users == 0:
continue
orgs_info[org.name] = {}
counter_key = "analysis::run_count::%s" % org.key.urlsafe()
orgs_info[org.name]["analyzed_users"] = memcache.get(counter_key)
orgs_info[org.name]["total_users"] = total_users
orgs_info[org.name]["org_key"] = org.key.urlsafe()
if request.method == "POST":
org_key = request.form.get("org_key", None)
taskqueue.add(url=url_for("run_analysis", org_key=org_key),
target="analyzer")
return redirect(url_for("analysis"))
return render_template("admin/analysis.html",
orgs_info=orgs_info)
def generate_csv(org_name):
org = Organization.query(Organization.name == org_name).get()
combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
for cu in combined_users:
org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
has_facebook = 1 if cu.facebook_id else 0
has_twitter = 1 if cu.twitter_id else 0
has_linkedin = 1 if cu.linkedin_id else 0
rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" %
(org_user_id, cu.legit_report["legitscore"],
cu.legit_report["lrg_reputation"]["marketplace_memberships"],
cu.legit_report["lrg_reputation"]["total_transactions"],
cu.legit_report["lrg_reputation"]["total_reviews"],
cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
has_facebook, has_twitter, has_linkedin))
#logging.info("CSV FILE")
#logging.info(rv)
file_name = files.blobstore.create(
mime_type='text/plain',
_blobinfo_uploaded_filename="%s-analysis" % org_name)
with files.open(file_name, 'a') as f:
f.write(rv)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
ar = AnalysisReport.query(
AnalysisReport.org_name == org_name).get() or AnalysisReport(
org_name=org_name)
ar.blob_key = blob_key
ar.put()
def generate_csv(org_name):
org = Organization.query(Organization.name == org_name).get()
combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
for cu in combined_users:
org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
has_facebook = 1 if cu.facebook_id else 0
has_twitter = 1 if cu.twitter_id else 0
has_linkedin = 1 if cu.linkedin_id else 0
rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" %
(org_user_id,
cu.legit_report["legitscore"],
cu.legit_report["lrg_reputation"]["marketplace_memberships"],
cu.legit_report["lrg_reputation"]["total_transactions"],
cu.legit_report["lrg_reputation"]["total_reviews"],
cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
has_facebook,
has_twitter,
has_linkedin))
#logging.info("CSV FILE")
#logging.info(rv)
file_name = files.blobstore.create(mime_type='text/plain',
_blobinfo_uploaded_filename="%s-analysis" % org_name)
with files.open(file_name, 'a') as f:
f.write(rv)
files.finalize(file_name)
blob_key = files.blobstore.get_blob_key(file_name)
ar = AnalysisReport.query(AnalysisReport.org_name == org_name).get() or AnalysisReport(org_name=org_name)
ar.blob_key = blob_key
ar.put()
def create_invite(code, org_name):
org = Organization.query(Organization.name == org_name).get()
i = Invite(org=org.key, code=code)
i.put()
def create_invite(code, org_name):
org = Organization.query(Organization.name == org_name).get()
i = Invite(org=org.key, code=code)
i.put()
