def kickoff_compute_value(): org_keys = Organization.query().iter(keys_only=True) for org_key in org_keys: url = url_for("compute_org_value", organization_key=org_key.urlsafe()) if DEVELOPMENT: taskqueue.add(url=url) else: taskqueue.add(url=url, target="1.analyzer") return redirect(url_for("value"))
def organizations_management(): if request.method == "POST": form = CreateOrganizationForm(request.form) if form.validate(): org = Organization(name=form.short_name.data, full_name=form.full_name.data) org.put() else: form = CreateOrganizationForm() existing_orgs = Organization.query().fetch(100) return render_template("admin/organizations.html", existing_orgs=existing_orgs, form=form)
def get(self): self.response.headers['Content-Type'] = 'application/json' key = ndb.Key(urlsafe = self.request.get('entity')) if key.kind() == 'Organization': entity = Organization.get_by_key(key.urlsafe()) elif key.kind() == 'OU': entity = OU.get_by_key(key.urlsafe()) elif key.kind() == 'CN': entity = CN.get_by_key(key.urlsafe()) logging.warning('DEBUG OrganizationTree - key.kind(): %s' % key.kind()) if key.kind() in ['Organization', 'OU']: name = entity.name else: name = entity.first_name + ' ' + entity.last_name treedict = {'key': entity.key.urlsafe(), 'name': name, 'kind': key.kind()} level = treedict # Ascending tree while key.kind() != 'Organization': parent = entity.parent if parent.kind() == 'Organization': entity = Organization.query(Organization.key == parent).get() else: entity = OU.query(OU.key == parent).get() key = entity.key logging.warning('DEBUG OrganizationTree - parent: %s - entity: %s' % (parent, str(entity))) entitydict = {'key': entity.key.urlsafe(), 'name': entity.name, 'kind': parent.kind()} level['parent'] = entitydict level = level['parent'] children = [] ouchildren = OU.query(OU.parent == key).fetch() for ouchild in ouchildren: oudict = {'key': ouchild.key.urlsafe(), 'name': ouchild.name, 'kind': ouchild.key.kind()} children.append(oudict) cnchildren = CN.query(CN.parent == key).fetch() for cnchild in cnchildren: cndict = {'key': cnchild.key.urlsafe(), 'name': cnchild.first_name + ' ' + cnchild.last_name, 'kind': cnchild.key.kind()} children.append(cndict) treedict['child'] = pop_list(treedict) self.response.write(json.dumps(treedict))
def run_analysis(org_key=None): """ Task that deletes all existing CombinedUsers and regenerates them from the underlying IntakeUsers. This task can take a long time to run, so it should be run on a dedicated instance. """ DELETE_BATCH = 500 ANALYZE_BATCH = 50 # Clear out the existing combined users cu_query = CombinedUser.query() if org_key: org_key = ndb.Key(urlsafe=org_key) cu_query = cu_query.filter(CombinedUser.orgs == org_key) while True: results, cursor, more = cu_query.fetch_page(DELETE_BATCH, keys_only=True) ndb.delete_multi(results) if not more: break # Analyze all the intake users, on a per-organization basis if org_key: org_keys = [org_key] else: org_keys = Organization.query().iter(keys_only=True) for org_key in org_keys: counter_key = "analysis::run_count::%s" % org_key.urlsafe() memcache.set(key=counter_key, value=0) iu_query = IntakeUser.query(IntakeUser.org == org_key) for iu_key in iu_query.iter(keys_only=True): memcache.incr(counter_key) #deferred.defer(analyze_user, intake_user_key=iu_key) analyze_user(iu_key) generate_csv(org_key.get().name) return "Great Success"
def analysis(): orgs = Organization.query().fetch(50) orgs_info = {} for org in orgs: total_users = IntakeUser.query(IntakeUser.org == org.key).count() if total_users == 0: continue orgs_info[org.name] = {} counter_key = "analysis::run_count::%s" % org.key.urlsafe() orgs_info[org.name]["analyzed_users"] = memcache.get(counter_key) orgs_info[org.name]["total_users"] = total_users orgs_info[org.name]["org_key"] = org.key.urlsafe() if request.method == "POST": org_key = request.form.get("org_key", None) taskqueue.add(url=url_for("run_analysis", org_key=org_key), target="analyzer") return redirect(url_for("analysis")) return render_template("admin/analysis.html", orgs_info=orgs_info)
def generate_csv(org_name): org = Organization.query(Organization.name == org_name).get() combined_users = CombinedUser.query(CombinedUser.orgs == org.key) rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n" for cu in combined_users: org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0] has_facebook = 1 if cu.facebook_id else 0 has_twitter = 1 if cu.twitter_id else 0 has_linkedin = 1 if cu.linkedin_id else 0 rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" % (org_user_id, cu.legit_report["legitscore"], cu.legit_report["lrg_reputation"]["marketplace_memberships"], cu.legit_report["lrg_reputation"]["total_transactions"], cu.legit_report["lrg_reputation"]["total_reviews"], cu.legit_report["lrg_reputation"]["overall_positive_feedback"], len(cu.legit_report["lrg_reputation"]["permanent_removals"]), has_facebook, has_twitter, has_linkedin)) #logging.info("CSV FILE") #logging.info(rv) file_name = files.blobstore.create( mime_type='text/plain', _blobinfo_uploaded_filename="%s-analysis" % org_name) with files.open(file_name, 'a') as f: f.write(rv) files.finalize(file_name) blob_key = files.blobstore.get_blob_key(file_name) ar = AnalysisReport.query( AnalysisReport.org_name == org_name).get() or AnalysisReport( org_name=org_name) ar.blob_key = blob_key ar.put()
def generate_csv(org_name): org = Organization.query(Organization.name == org_name).get() combined_users = CombinedUser.query(CombinedUser.orgs == org.key) rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n" for cu in combined_users: org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0] has_facebook = 1 if cu.facebook_id else 0 has_twitter = 1 if cu.twitter_id else 0 has_linkedin = 1 if cu.linkedin_id else 0 rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" % (org_user_id, cu.legit_report["legitscore"], cu.legit_report["lrg_reputation"]["marketplace_memberships"], cu.legit_report["lrg_reputation"]["total_transactions"], cu.legit_report["lrg_reputation"]["total_reviews"], cu.legit_report["lrg_reputation"]["overall_positive_feedback"], len(cu.legit_report["lrg_reputation"]["permanent_removals"]), has_facebook, has_twitter, has_linkedin)) #logging.info("CSV FILE") #logging.info(rv) file_name = files.blobstore.create(mime_type='text/plain', _blobinfo_uploaded_filename="%s-analysis" % org_name) with files.open(file_name, 'a') as f: f.write(rv) files.finalize(file_name) blob_key = files.blobstore.get_blob_key(file_name) ar = AnalysisReport.query(AnalysisReport.org_name == org_name).get() or AnalysisReport(org_name=org_name) ar.blob_key = blob_key ar.put()
def create_invite(code, org_name): org = Organization.query(Organization.name == org_name).get() i = Invite(org=org.key, code=code) i.put()