def prepOutput(member_list): output = profiles.Profiles(params['output path'], settings = params) for member in member_list: #inconsistent. i do this earlier in eval_portal member['profile'] = output.formatProfile(member) all_profiles = params['header template'] + '\n'.join(member['profile'] for member in member_list) edit_summ = params['edit summary'] % (params['type'] + " " + params['subtype']) output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page = params[params['subtype']]['subpage'])
def getMemberData(member, i): member['username'] = tools.formatSummaries( member['username']) #Strips anything that looks like markup. member['item'] = i profile = profiles.Profiles(member['page path'], settings=params) member['profile'] = profile.formatProfile(member) return member
def publishLearnings(learning_pages): report = params['header template'] + '\n'.join(page['formatted'] for page in learning_pages) # print report edit_sum = params['edit summary'] pub = profiles.Profiles(params['output path'], params['type']) pub.publishProfile(report, params['output path'], edit_sum)
def main(): """ Read config and run """ # Read config args = get_args() config = get_namelist(args) clim = climatology.GridClim(config) minyr = config.getint('profiles', 'minyr') maxyr = config.getint('profiles', 'maxyr') gridfiles = [] dts, fnames = tools.get_dt_files(config, minyr, maxyr) # Grid data for dt, fname in zip(dts, fnames): prof = profiles.Profiles(config, fname, dt) prof.grid_data() prof.write_grid() gridfiles.append(prof.fout) clim.accumulate_profiles(prof) # Calculate monthly climatology clim.calc_clim() clim.write_clim() # Calculate anomalies if config.getboolean('anomalies', 'calc_anomalies'): for dt, gridfile in zip(dts, gridfiles): tools.calc_anom(gridfile, dt, clim) # Finished print '\nFinished!\n'
def prepOutput(member_list): all_profiles = params['header template'] + '\n'.join( member['profile'] for member in member_list) #different from guide edit_summ = params['edit summary'] % (params['subtype'] + " " + params['type']) output = profiles.Profiles( params['output path'], params['type']) #stupid tocreate a new profile object here? output.publishProfile(all_profiles, params['output path'], edit_summ, edit_sec=params['output section'])
def rankProfiles(): """ rank Eval portal profiles by number of recent edits. """ profile_page = profiles.Profiles(params['output path'], params['output page id'], params) profile_list = profile_page.getPageSectionData( level=params['profile toclevel']) # print profile_list for profile in profile_list: profile['title'].encode("utf8") #so redundant! quote1 = "'" quote2 = "'" usernames = quote1 + "','".join(x['title'] for x in profile_list) + quote2 conn = MySQLdb.connect(host=grantsbot_settings.host, db=grantsbot_settings.dbname, read_default_file=grantsbot_settings.defaultcnf, use_unicode=True, charset="utf8") cursor = conn.cursor() query = "SELECT rev_user_text, recent_edits FROM eval_profiles WHERE rev_user_text IN (%s);" % usernames # print query # print type(query) cursor.execute(query) rows = cursor.fetchall() output = [{ 'edits': row[1], 'username': row[0].decode("utf8") } for row in rows] # print output for profile in profile_list: for o in output: if profile['title'] == o['username']: profile['edits'] = o['edits'] break # continue for profile in profile_list: if "edits" not in profile: profile['edits'] = 0 profile['text'] = profile_page.getPageText(profile['index']) # print len(profile_list) plist_sorted = sorted(profile_list, key=lambda item: item['edits'], reverse=True) plist_text = {'profiles': '\n\n'.join([x['text'] for x in plist_sorted])} formatted_profiles = profile_page.formatProfile(plist_text) edit_summ = params['edit summary'] % (params['type'], ) profile_page.publishProfile(formatted_profiles, params['output path'], edit_summ, edit_sec=params['output section'])
def findProfiles(cursor): profile_page = profiles.Profiles( "Programs:Evaluation_portal/Parlor/Introductions", "whocares") profile_list = profile_page.getPageSectionData() # usernames = [x['username'] for x in profile_list] quote1 = "('" quote2 = "')" usernames = quote1 + "'),('".join(x['username'] for x in profile_list) + quote2 print usernames #for profile in profile_list: cursor.execute("INSERT INTO eval_profiles (rev_user_text) VALUES%s;" % usernames) conn.commit()
def main(): """ Read config and run """ # Read config args = get_args() config = get_namelist(args) clim = climatology.GridClim(config) minyr = config.getint('profiles', 'minyr') maxyr = config.getint('profiles', 'maxyr') maxmonth = config.getint('profiles', 'maxmonth') minmonth = config.getint('profiles', 'minmonth') mupdate = config.getboolean('profiles', 'monthly_update') gridfiles = [] dts, fnames = tools.get_dt_files(config, minyr, maxyr, minmonth, maxmonth, mupdate) #cminyr = config.getint('climatology', 'minyr') #cmaxyr = config.getint('climatology', 'maxyr') #cdts, cfnames = tools.get_dt_files(config, cminyr, cmaxyr) # Grid data for dt, fname in zip(dts, fnames): prof = profiles.Profiles(config, fname, dt) prof.grid_data() prof.write_grid() gridfiles.append(prof.fout) clim.accumulate_profiles(prof) # Calculate monthly climatology if config.getboolean('climatology', 'calc_climatology'): print('Got up to here') clim.calc_clim() clim.write_clim() # Calculate anomalies if config.getboolean('anomalies', 'calc_anomalies'): for dt, gridfile in zip(dts, gridfiles): tools.calc_anom(gridfile, dt, clim) # Move the anomalies to the anomaly folder: anomdir = config.get('anomalies','anomdir') profdir = config.get('grid','dir') files = glob.glob(profdir + '*_anom_*') for f in files: call('mv ' + f + ' ' + anomdir, shell = True) # Finished print('\nFinished!\n')
def getMemberData(member, input_params, params, tools = False): profile = profiles.Profiles(member['page path'], input_params[0], id=member['page id'], params = params) member = profile.setTimeValues(member, input_params) if member['username']: member['username'] = tools.formatSummaries(member['username']) #sloppy, but better. Strips anything that looks like markup from username. else: pass if input_params[0] == 'guide': #necessary? member['title'] = tools.titleFromPath(member['page path']) #new function to get title from path infobox = profile.getPageText(0) member = profile.scrapeInfobox(member, infobox) else: pass member['profile'] = profile.formatProfile(input_params, member) return member
def getLearnings(grant_pages): #should this always be the same name for page in grant_pages: report = profiles.Profiles(page['page path'], page['page id'], settings=params) rep_secs = report.getPageSectionData() for sec in rep_secs: if sec['title'] == 'Lessons learned': page['lessons'] = report.getPageText(sec['index']) page['title'] = page['page path'][7:-7] page['formatted'] = report.formatProfile(page) # print page['lessons'] continue else: pass grant_pages = [p for p in grant_pages if p.has_key("lessons")] return grant_pages
def prepOutput(member_list, input_params, params, tools): member_list = tools.dedupeMemberList(member_list, 'datetime', 'page path') #not the ideal place for this member_list.sort(key=operator.itemgetter('datetime'), reverse=True) #abstract this? ###MOVE THIS IF### if input_params[2] == 'new': #won't work yet. need to do this earlier member_list = member_list[0:10] #Do this in an earlier step? also, 'new' is no longer a valid subtype! else: pass all_profiles = params['header template'] + '\n'.join(member['profile'] for member in member_list) # print all_profiles if 'subpage' in params[input_params[2]]: sb_page = params[input_params[2]]['subpage'] else: sb_page = "" if 'output section' in params[input_params[2]]: edit_sec = params[input_params[2]]['output section'] else: edit_sec = "" edit_summ = params['edit summary'] % (input_params[1] + " " + input_params[0]) output = profiles.Profiles(params['output path'], profile_type) #stupid tocreate a new profile object here. output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page, edit_sec)
def prepOutput(featured_list): first_subpage = params[params['subtype']]['first subpage'] number_featured = params[params['subtype']]['number featured'] featured_list = tools.addDefaults(featured_list) output = profiles.Profiles( params[params['subtype']]['output path'], settings=params ) #stupid tocreate a new profile object here. and stupid to re-specify the path below i = first_subpage for f in featured_list: if i <= first_subpage + (number_featured - 1): f['profile'] = output.formatProfile(f) f['profile'] = params['header template'] + '\n' + f['profile'] edit_summ = params['edit summary'] % (params['subtype'] + " " + params['type']) output.publishProfile(f['profile'], params[params['subtype']]['output path'], edit_summ, sb_page=i) i += 1 else: break
def getFeaturedProfiles(): """ Gets info about the top-billed profiles in a guide. """ featured_list = [] profile_page = profiles.Profiles( params[params['subtype']]['input page path'], params[params['subtype']]['input page id'], params) profile_list = profile_page.getPageSectionData( level=params[params['subtype']]['profile toclevel']) for profile in profile_list: # print profile text = profile_page.getPageText(profile['index']) profile = profile_page.scrapeInfobox(profile, text) if len(profile['summary']) > 1 and len(profile['image']) > 1: profile['action'] = params[params['subtype']]['action'] profile['summary'] = tools.formatSummaries(profile['summary']) featured_list.append(profile) shuffle(featured_list) featured_list = featured_list[:params[params['subtype']] ['number featured']] return featured_list
def prepOutput(featured_list): subpage = params[params['subtype']]['first subpage'] i = 0 number_featured = params[params['subtype']]['number featured'] featured_list = tools.addDefaults(featured_list) output = profiles.Profiles( params[params['subtype']]['output path'], settings=params ) #stupid tocreate a new profile object here. and stupid to re-specify the path below for f in featured_list: if i < number_featured: f['profile'] = output.formatProfile(f) f['profile'] = params['header template'] + '\n' + f['profile'] edit_summ = params['edit summary'] % (params['subtype'] + " " + params['type']) output.publishProfile(f['profile'], params[params['subtype']]['output path'], edit_summ, sb_page=subpage) i += 1 subpage += 1 #should fix idealab gallery to make this work there, too else: break
def getMemberData(member): profile = profiles.Profiles(member['page path'], id=member['page id'], settings = params) infobox = profile.getPageText(0) #zero is the top section member = profile.scrapeInfobox(member, infobox, redict = params['infobox params']) revs = [] main_revs = profile.getPageEditInfo() revs.extend(main_revs) if member['talkpage id']: talk_revs = profile.getPageEditInfo(page = member['talkpage id']) if talk_revs: revs.extend(talk_revs) revs.sort(key=operator.itemgetter('revid'), reverse=True) member['timestamp'] = revs[0]['timestamp'] revs.sort(key=operator.itemgetter('revid'), reverse=False) #this doesn't seem to be working member['username'] = "******" + revs[0]['user'] + "]]" member['created'] = revs[0]['timestamp'] if params['formatted fields']: for field in params['formatted fields']: #do I need the 'if' above this line? try: member[field] = tools.formatSummaries(member[field]) except: pass return member
def rankProfiles(): """ rank IdeaLab profiles by number of recent edits. """ profile_page = profiles.Profiles(params['output path'], params['output page id'], params) profile_list = profile_page.getPageSectionData() for profile in profile_list: profile['title'].encode("utf8") #so redundant! profile['text'] = profile_page.getPageText(profile['index']) main_edits = profile_page.getUserRecentEditInfo( profile['title'], params['main namespace']) talk_edits = profile_page.getUserRecentEditInfo( profile['title'], params['talk namespace']) profile['edits'] = main_edits + talk_edits plist_sorted = sorted(profile_list, key=lambda item: item['edits'], reverse=True) plist_text = {'profiles': '\n\n'.join([x['text'] for x in plist_sorted])} formatted_profiles = profile_page.formatProfile(plist_text) edit_summ = params['edit summary'] % (params['type'], ) profile_page.publishProfile(formatted_profiles, params['output path'], edit_summ)
def getFeaturedProfiles(): """ Gets info about the top-billed profiles in a guide. """ featured_list = [] profile_page = profiles.Profiles( params[params['subtype']]['input page path'], params[params['subtype']]['input page id'], params) profile_list = profile_page.getPageSectionData( level=params[params['subtype']]['profile toclevel']) for profile in profile_list: text = profile_page.getPageText(profile['index']) profile = profile_page.scrapeInfobox(profile, text) if params['subtype'] == 'intro': profile['page path'] = params[params['subtype']][ 'input page path'] + "#" + profile['title'].lstrip() if len(profile['name']) > 1: profile['title'] = profile['name'] else: pass featured_list.append(profile) elif params['subtype'] == 'question': profile['summary'] = re.sub( "\=\=(.*?)\=\=", "", text) #remove the question title from the summary. profile['page path'] = params[params['subtype']][ 'input page path'] + "#" + profile['title'].lstrip() featured_list.append(profile) else: if len(profile['summary']) > 1: featured_list.append(profile) else: pass for f in featured_list: f['action'] = params[params['subtype']]['action'] f['summary'] = tools.formatSummaries(f['summary']) return featured_list
def __init__(self): #create a list of student Object #self__list_student will store object of Student class self.__list_student = [] self.__list_job = [] self.__list_settings = [] self.__list_profiles = [] ############################################################## #for epic5 self.__list_pending_friend = [] self.__list_accept_friend = [] ############################################################### #add title for the student_data.csv if not os.path.isfile(FILENAME): with open(FILENAME, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow( ("User_Name", "Password", "First_Name", "Last_Name")) #add data from student_data.csv to __self.__list_student with open(FILENAME, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_student.append( s.Student(row[0], row[1], row[2], row[3])) #add title for the job_data.csv if not os.path.isfile(FILENAME_JOB): with open(FILENAME_JOB, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow(("Title", "Description", "Employer", "Location", "Salary", "Post_Name")) #add data from job_data to __self.__list_job with open(FILENAME_JOB, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_job.append( j.Job(row[0], row[1], row[2], row[3], row[4], row[5])) #add title for the settings.csv if not os.path.isfile(FILENAME_STG): with open(FILENAME_STG, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow(("user", "email_notif", "sms_notif", "targeted_ads", "language")) #add data from settings.csv to __self.__list_settings with open(FILENAME_STG, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_settings.append( stg.Settings(row[0], row[1], row[2], row[3], row[4])) #add title for the profiles.csv if not os.path.isfile(FILENAME_PRO): with open(FILENAME_PRO, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow(("user", "title", "major", "university", "bio", "experience", "education")) #add data from profiles.csv with open(FILENAME_PRO, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_profiles.append( pro.Profiles(row[0], row[1], row[2], row[3], row[4], row[5], row[6])) ############################### BEGIN ############################################### #new for epic5 #add title for the Pending_Friends.csv if not os.path.isfile(FILENAME_PENDING): with open(FILENAME_PENDING, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow(("Request_name", "Pending_name")) #add data from Pending_Friends.csv to __self.__Pending_Friend with open(FILENAME_PENDING, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_pending_friend.append( pf.Pending_Friend(row[0], row[1])) #add title for the accept_friends.csv if not os.path.isfile(FILENAME_ACCEPT): with open(FILENAME_ACCEPT, "w") as file: writer_csv = csv.writer(file) writer_csv.writerow(("Request", "Accept")) #add data from accept_friends.csv to __self.__accept_friend with open(FILENAME_ACCEPT, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_accept_friend.append( af.Accept_Friend(row[0], row[1]))
def __init__(self): #create a list of student Object #self__list_student will store object of Student class self.__list_student = [] self.__list_job = [] self.__list_settings = [] self.__list_profiles = [] self.__list_save_job = [] if not os.path.isfile(FILENAME_SAVE_JOB): with open(FILENAME_SAVE_JOB, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("User_Name","Title")) with open(FILENAME_SAVE_JOB, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_save_job.append(sa.Save(row[0], row[1])) if not os.path.isfile(FILENAME_APP): with open(FILENAME_APP, "w") as file: writer_csv = csv.writer(file) #add title for the student_data.csv if not os.path.isfile(FILENAME): with open(FILENAME, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("User_Name","Password","First_Name","Last_Name","Tier", "DayOfLastJobApplication")) #add data from student_data.csv to __self.__list_student with open(FILENAME, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_student.append( s.Student(row[0], row[1], row[2], row[3])) #add title for the job_data.csv if not os.path.isfile(FILENAME_JOB): with open(FILENAME_JOB, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("Title","Description","Employer","Location","Salary","Post_Name")) #add data from job_data to __self.__list_job with open(FILENAME_JOB, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_job.append( j.Job(row[0], row[1], row[2], row[3], row[4], row[5])) #add title for the settings.csv if not os.path.isfile(FILENAME_STG): with open(FILENAME_STG, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("user", "email_notif","sms_notif","targeted_ads","language")) #add data from settings.csv to __self.__list_settings with open(FILENAME_STG, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_settings.append( stg.Settings(row[0], row[1], row[2], row[3], row[4])) #add title for the profiles.csv if not os.path.isfile(FILENAME_PRO): with open(FILENAME_PRO, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("user", "title","major","university","bio", "experience", "education")) #add data from profiles.csv with open(FILENAME_PRO, "r") as file: reader_csv = csv.reader(file) for row in reader_csv: if row != []: self.__list_profiles.append( pro.Profiles(row[0], row[1], row[2], row[3], row[4], row[5], row[6])) #add title for the pending_messages.csv if not os.path.isfile(FILENAME_PRO): with open(FILENAME_MES, "w") as file: writer_csv = csv.writer(file) #add title for the new_jobs.csv if not os.path.isfile(FILENAME_NEW_JOB): with open(FILENAME_NEW_JOB, "w") as file: writer_csv = csv.writer(file) #writer_csv.writerow(("jobTitle", "List of NOT seen")) #create del_job_app file if not os.path.isfile(FILENAME_DEL_JOB): with open(FILENAME_DEL_JOB, "w") as file: writer_csv = csv.writer(file)
def prepOutput(member_list): all_profiles = params[params['subtype']]['header template'] + '\n'.join(member['profile'] for member in member_list if 'profile' in member.keys())#if no profile field was created for some reason, ignore edit_summ = params['edit summary'] % (params['type'] + " " + params['subtype']) output = profiles.Profiles(params['output path'], params['type']) #stupid tocreate a new profile object here. output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page = params[params['subtype']]['subpage'], edit_sec = params['output section'])
def getMemberData(member): profile = profiles.Profiles(member['page path'], id=member['page id'], settings = params) infobox = profile.getPageText(section = 0) #zero is the top section member = profile.scrapeInfobox(member, infobox, trans_tag = "<translate>") member['profile'] = profile.formatProfile(member) return member