def query():
matching_users = None
if request.method == "POST":
form = QueryForm(request.form)
if form.validate():
#logging.info("Request Data: %s" % request.form)
#logging.info("Form Data: %s" % form.data)
query_data = form.data
is_prehashed = query_data.pop("prehashed")
# for k,v in query_data.items():
# if v:
# query_data[k] = hash_value(v)
# else:
# del query_data[k]
user_sets = find_users(query_data, "api", pre_hashed=is_prehashed)
lrg_summaries = []
for user_set in user_sets:
lrg_summaries.append(generate_lrg_summary([user_set]))
user_sets = zip(user_sets, lrg_summaries)
legit_report, intake_users = generate_legit_report(
query_data, "api", pre_hashed=is_prehashed)
legit_report_pretty = json.dumps(legit_report,
sort_keys=True,
indent=4)
else:
user_sets = None
legit_report = None
legit_report_pretty = None
form = QueryForm()
return render_template("admin/query.html",
form=form,
PII_FIELDS=PII_FIELDS,
DEVELOPMENT=DEVELOPMENT,
user_sets=user_sets,
legit_report=legit_report,
legit_report_pretty=legit_report_pretty)
def query():
matching_users = None
if request.method == "POST":
form = QueryForm(request.form)
if form.validate():
#logging.info("Request Data: %s" % request.form)
#logging.info("Form Data: %s" % form.data)
query_data = form.data
is_prehashed = query_data.pop("prehashed")
# for k,v in query_data.items():
# if v:
# query_data[k] = hash_value(v)
# else:
# del query_data[k]
user_sets = find_users(query_data, "api", pre_hashed=is_prehashed)
lrg_summaries = []
for user_set in user_sets:
lrg_summaries.append(generate_lrg_summary([user_set]))
user_sets = zip(user_sets,lrg_summaries)
legit_report, intake_users = generate_legit_report(
query_data,
"api",
pre_hashed=is_prehashed)
legit_report_pretty = json.dumps(legit_report, sort_keys=True, indent=4)
else:
user_sets = None
legit_report = None
legit_report_pretty = None
form = QueryForm()
return render_template("admin/query.html",
form=form,
PII_FIELDS=PII_FIELDS,
DEVELOPMENT=DEVELOPMENT,
user_sets=user_sets,
legit_report=legit_report,
legit_report_pretty=legit_report_pretty)
def compute_org_value(organization_key):
"""
Task that computes the current value of our data for the given organization.
This task can take a long time and should be run on a backend.
"""
org = ndb.Key(urlsafe=organization_key).get()
vs = ValueSummary.query(ValueSummary.organization == org.key).get()
if not vs:
vs = ValueSummary(organization=org.key)
vs.processed_users = 0
vs.total_users = IntakeUser.query(IntakeUser.org == org.key).count()
vs.is_running = True
vs.overlap_counts = {}
vs.org_name = org.full_name
vs.put()
overlap_counts = defaultdict(int)
for intake_user in IntakeUser.query(IntakeUser.org == org.key):
#logging.info(intake_user.pii_dict())
user_sets = find_users(intake_user.pii_dict(), pre_hashed=True)
#logging.info("USER SETS:")
#logging.info(user_sets)
lrg_summary = generate_lrg_summary(user_sets, [org.key])
#logging.info("LRG SUMMARY:")
#logging.info(lrg_summary)
if lrg_summary:
overlap_counts[lrg_summary["marketplace_memberships"]] += 1
vs.processed_users += 1
vs.put()
vs.is_running = False
vs.overlap_counts = overlap_counts
vs.put()
return jsonify(success=True, processed_users=vs.processed_users)
def compute_org_value(organization_key):
"""
Task that computes the current value of our data for the given organization.
This task can take a long time and should be run on a backend.
"""
org = ndb.Key(urlsafe=organization_key).get()
vs = ValueSummary.query(ValueSummary.organization == org.key).get()
if not vs:
vs = ValueSummary(organization=org.key)
vs.processed_users = 0
vs.total_users = IntakeUser.query(IntakeUser.org == org.key).count()
vs.is_running = True
vs.overlap_counts = {}
vs.org_name = org.full_name
vs.put()
overlap_counts = defaultdict(int)
for intake_user in IntakeUser.query(IntakeUser.org == org.key):
#logging.info(intake_user.pii_dict())
user_sets = find_users(intake_user.pii_dict(), pre_hashed=True)
#logging.info("USER SETS:")
#logging.info(user_sets)
lrg_summary = generate_lrg_summary(user_sets, [org.key])
#logging.info("LRG SUMMARY:")
#logging.info(lrg_summary)
if lrg_summary:
overlap_counts[lrg_summary["marketplace_memberships"]] += 1
vs.processed_users += 1
vs.put()
vs.is_running = False
vs.overlap_counts = overlap_counts
vs.put()
return jsonify(success=True, processed_users=vs.processed_users)
