Example #1
0
def analyze_user(intake_user_key):
    #logging.debug("Analyzing User Key: %s" % intake_user_key)
    intake_user = intake_user_key.get()
    counter_key = "analysis::run_count::%s" % intake_user.org.urlsafe()
    
    count = CombinedUser.query(CombinedUser.intake_users == intake_user.key).count(1)
    if not count:  
        # Generate a legit report
        legit_report, intake_users = get_legit_report(intake_user, api_type="api")

        # Create a combined user
        cu = CombinedUser.create(intake_users, legit_report)
        cu.put()
        
    # Check one off the list
    #memcache.decr(counter_key)
    
    return "Great Success"
Example #2
0
def analyze_user(intake_user_key):
    #logging.debug("Analyzing User Key: %s" % intake_user_key)
    intake_user = intake_user_key.get()
    counter_key = "analysis::run_count::%s" % intake_user.org.urlsafe()

    count = CombinedUser.query(
        CombinedUser.intake_users == intake_user.key).count(1)
    if not count:
        # Generate a legit report
        legit_report, intake_users = get_legit_report(intake_user,
                                                      api_type="api")

        # Create a combined user
        cu = CombinedUser.create(intake_users, legit_report)
        cu.put()

    # Check one off the list
    #memcache.decr(counter_key)

    return "Great Success"
Example #3
0
def run_analysis(org_key=None):
    """
    Task that deletes all existing CombinedUsers and regenerates them from the
    underlying IntakeUsers. This task can take a long time to run, so it should
    be run on a dedicated instance. 
    """
    DELETE_BATCH = 500
    ANALYZE_BATCH = 50

    # Clear out the existing combined users
    cu_query = CombinedUser.query()
    if org_key:
        org_key = ndb.Key(urlsafe=org_key)
        cu_query = cu_query.filter(CombinedUser.orgs == org_key)

    while True:
        results, cursor, more = cu_query.fetch_page(DELETE_BATCH,
                                                    keys_only=True)
        ndb.delete_multi(results)
        if not more:
            break

    # Analyze all the intake users, on a per-organization basis
    if org_key:
        org_keys = [org_key]
    else:
        org_keys = Organization.query().iter(keys_only=True)

    for org_key in org_keys:
        counter_key = "analysis::run_count::%s" % org_key.urlsafe()
        memcache.set(key=counter_key, value=0)

        iu_query = IntakeUser.query(IntakeUser.org == org_key)
        for iu_key in iu_query.iter(keys_only=True):
            memcache.incr(counter_key)
            #deferred.defer(analyze_user, intake_user_key=iu_key)
            analyze_user(iu_key)

        generate_csv(org_key.get().name)

    return "Great Success"
Example #4
0
def run_analysis(org_key=None):
    """
    Task that deletes all existing CombinedUsers and regenerates them from the
    underlying IntakeUsers. This task can take a long time to run, so it should
    be run on a dedicated instance. 
    """
    DELETE_BATCH = 500
    ANALYZE_BATCH = 50
    
    # Clear out the existing combined users    
    cu_query = CombinedUser.query()
    if org_key:
        org_key = ndb.Key(urlsafe=org_key)
        cu_query = cu_query.filter(CombinedUser.orgs == org_key)
    
    while True:
        results, cursor, more = cu_query.fetch_page(DELETE_BATCH, keys_only=True)
        ndb.delete_multi(results)
        if not more:
            break            
    
    # Analyze all the intake users, on a per-organization basis  
    if org_key:
        org_keys = [org_key]
    else:
        org_keys = Organization.query().iter(keys_only=True)
    
    for org_key in org_keys:
        counter_key = "analysis::run_count::%s" % org_key.urlsafe()
        memcache.set(key=counter_key, value=0)
        
        iu_query = IntakeUser.query(IntakeUser.org == org_key)        
        for iu_key in iu_query.iter(keys_only=True):
            memcache.incr(counter_key)            
            #deferred.defer(analyze_user, intake_user_key=iu_key)
            analyze_user(iu_key)
        
        generate_csv(org_key.get().name)

    return "Great Success"
Example #5
0
def generate_csv(org_name):
    org = Organization.query(Organization.name == org_name).get()
    combined_users = CombinedUser.query(CombinedUser.orgs == org.key)

    rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
    for cu in combined_users:
        org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
        has_facebook = 1 if cu.facebook_id else 0
        has_twitter = 1 if cu.twitter_id else 0
        has_linkedin = 1 if cu.linkedin_id else 0
        rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" %
               (org_user_id, cu.legit_report["legitscore"],
                cu.legit_report["lrg_reputation"]["marketplace_memberships"],
                cu.legit_report["lrg_reputation"]["total_transactions"],
                cu.legit_report["lrg_reputation"]["total_reviews"],
                cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
                len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
                has_facebook, has_twitter, has_linkedin))

    #logging.info("CSV FILE")
    #logging.info(rv)

    file_name = files.blobstore.create(
        mime_type='text/plain',
        _blobinfo_uploaded_filename="%s-analysis" % org_name)
    with files.open(file_name, 'a') as f:
        f.write(rv)

    files.finalize(file_name)
    blob_key = files.blobstore.get_blob_key(file_name)

    ar = AnalysisReport.query(
        AnalysisReport.org_name == org_name).get() or AnalysisReport(
            org_name=org_name)
    ar.blob_key = blob_key
    ar.put()
Example #6
0
def generate_csv(org_name):
    org = Organization.query(Organization.name == org_name).get()
    combined_users = CombinedUser.query(CombinedUser.orgs == org.key)
    
    rv = "UserID,LegitScore,Memberships,Transactions,Reviews,PositiveReviewPercentage,Bannings,Facebook,Twitter,Linkedin\n"
    for cu in combined_users:
        org_user_id = [ui for ui in cu.user_id if ui.startswith(org_name)][0]
        has_facebook = 1 if cu.facebook_id else 0
        has_twitter = 1 if cu.twitter_id else 0
        has_linkedin = 1 if cu.linkedin_id else 0
        rv += ("%s,%.1f,%d,%d,%d,%.2f,%d,%d,%d,%d\n" % 
                    (org_user_id,
                     cu.legit_report["legitscore"],
                     cu.legit_report["lrg_reputation"]["marketplace_memberships"],
                     cu.legit_report["lrg_reputation"]["total_transactions"],
                     cu.legit_report["lrg_reputation"]["total_reviews"],
                     cu.legit_report["lrg_reputation"]["overall_positive_feedback"],
                     len(cu.legit_report["lrg_reputation"]["permanent_removals"]),
                     has_facebook,
                     has_twitter,
                     has_linkedin))
    
    #logging.info("CSV FILE")
    #logging.info(rv)
                     
    file_name = files.blobstore.create(mime_type='text/plain',
                                       _blobinfo_uploaded_filename="%s-analysis" % org_name)
    with files.open(file_name, 'a') as f:
        f.write(rv)
        
    files.finalize(file_name) 
    blob_key = files.blobstore.get_blob_key(file_name)
    
    ar = AnalysisReport.query(AnalysisReport.org_name == org_name).get() or AnalysisReport(org_name=org_name)
    ar.blob_key = blob_key
    ar.put()