def prepOutput(member_list):
output = profiles.Profiles(params['output path'], settings = params)
for member in member_list: #inconsistent. i do this earlier in eval_portal
member['profile'] = output.formatProfile(member)
all_profiles = params['header template'] + '\n'.join(member['profile'] for member in member_list)
edit_summ = params['edit summary'] % (params['type'] + " " + params['subtype'])
output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page = params[params['subtype']]['subpage'])
def getMemberData(member, i):
member['username'] = tools.formatSummaries(
member['username']) #Strips anything that looks like markup.
member['item'] = i
profile = profiles.Profiles(member['page path'], settings=params)
member['profile'] = profile.formatProfile(member)
return member
def publishLearnings(learning_pages):
report = params['header template'] + '\n'.join(page['formatted']
for page in learning_pages)
# print report
edit_sum = params['edit summary']
pub = profiles.Profiles(params['output path'], params['type'])
pub.publishProfile(report, params['output path'], edit_sum)
def main():
""" Read config and run """
# Read config
args = get_args()
config = get_namelist(args)
clim = climatology.GridClim(config)
minyr = config.getint('profiles', 'minyr')
maxyr = config.getint('profiles', 'maxyr')
gridfiles = []
dts, fnames = tools.get_dt_files(config, minyr, maxyr)
# Grid data
for dt, fname in zip(dts, fnames):
prof = profiles.Profiles(config, fname, dt)
prof.grid_data()
prof.write_grid()
gridfiles.append(prof.fout)
clim.accumulate_profiles(prof)
# Calculate monthly climatology
clim.calc_clim()
clim.write_clim()
# Calculate anomalies
if config.getboolean('anomalies', 'calc_anomalies'):
for dt, gridfile in zip(dts, gridfiles):
tools.calc_anom(gridfile, dt, clim)
# Finished
print '\nFinished!\n'
def prepOutput(member_list):
all_profiles = params['header template'] + '\n'.join(
member['profile'] for member in member_list) #different from guide
edit_summ = params['edit summary'] % (params['subtype'] + " " +
params['type'])
output = profiles.Profiles(
params['output path'],
params['type']) #stupid tocreate a new profile object here?
output.publishProfile(all_profiles,
params['output path'],
edit_summ,
edit_sec=params['output section'])
def rankProfiles():
"""
rank Eval portal profiles by number of recent edits.
"""
profile_page = profiles.Profiles(params['output path'],
params['output page id'], params)
profile_list = profile_page.getPageSectionData(
level=params['profile toclevel'])
# print profile_list
for profile in profile_list:
profile['title'].encode("utf8") #so redundant!
quote1 = "'"
quote2 = "'"
usernames = quote1 + "','".join(x['title'] for x in profile_list) + quote2
conn = MySQLdb.connect(host=grantsbot_settings.host,
db=grantsbot_settings.dbname,
read_default_file=grantsbot_settings.defaultcnf,
use_unicode=True,
charset="utf8")
cursor = conn.cursor()
query = "SELECT rev_user_text, recent_edits FROM eval_profiles WHERE rev_user_text IN (%s);" % usernames
# print query
# print type(query)
cursor.execute(query)
rows = cursor.fetchall()
output = [{
'edits': row[1],
'username': row[0].decode("utf8")
} for row in rows]
# print output
for profile in profile_list:
for o in output:
if profile['title'] == o['username']:
profile['edits'] = o['edits']
break
# continue
for profile in profile_list:
if "edits" not in profile:
profile['edits'] = 0
profile['text'] = profile_page.getPageText(profile['index'])
# print len(profile_list)
plist_sorted = sorted(profile_list,
key=lambda item: item['edits'],
reverse=True)
plist_text = {'profiles': '\n\n'.join([x['text'] for x in plist_sorted])}
formatted_profiles = profile_page.formatProfile(plist_text)
edit_summ = params['edit summary'] % (params['type'], )
profile_page.publishProfile(formatted_profiles,
params['output path'],
edit_summ,
edit_sec=params['output section'])
def findProfiles(cursor):
profile_page = profiles.Profiles(
"Programs:Evaluation_portal/Parlor/Introductions", "whocares")
profile_list = profile_page.getPageSectionData()
# usernames = [x['username'] for x in profile_list]
quote1 = "('"
quote2 = "')"
usernames = quote1 + "'),('".join(x['username']
for x in profile_list) + quote2
print usernames
#for profile in profile_list:
cursor.execute("INSERT INTO eval_profiles (rev_user_text) VALUES%s;" %
usernames)
conn.commit()
def main():
""" Read config and run """
# Read config
args = get_args()
config = get_namelist(args)
clim = climatology.GridClim(config)
minyr = config.getint('profiles', 'minyr')
maxyr = config.getint('profiles', 'maxyr')
maxmonth = config.getint('profiles', 'maxmonth')
minmonth = config.getint('profiles', 'minmonth')
mupdate = config.getboolean('profiles', 'monthly_update')
gridfiles = []
dts, fnames = tools.get_dt_files(config, minyr, maxyr, minmonth, maxmonth,
mupdate)
#cminyr = config.getint('climatology', 'minyr')
#cmaxyr = config.getint('climatology', 'maxyr')
#cdts, cfnames = tools.get_dt_files(config, cminyr, cmaxyr)
# Grid data
for dt, fname in zip(dts, fnames):
prof = profiles.Profiles(config, fname, dt)
prof.grid_data()
prof.write_grid()
gridfiles.append(prof.fout)
clim.accumulate_profiles(prof)
# Calculate monthly climatology
if config.getboolean('climatology', 'calc_climatology'):
print('Got up to here')
clim.calc_clim()
clim.write_clim()
# Calculate anomalies
if config.getboolean('anomalies', 'calc_anomalies'):
for dt, gridfile in zip(dts, gridfiles):
tools.calc_anom(gridfile, dt, clim)
# Move the anomalies to the anomaly folder:
anomdir = config.get('anomalies','anomdir')
profdir = config.get('grid','dir')
files = glob.glob(profdir + '*_anom_*')
for f in files:
call('mv ' + f + ' ' + anomdir, shell = True)
# Finished
print('\nFinished!\n')
def getMemberData(member, input_params, params, tools = False): profile = profiles.Profiles(member['page path'], input_params[0], id=member['page id'], params = params) member = profile.setTimeValues(member, input_params) if member['username']: member['username'] = tools.formatSummaries(member['username']) #sloppy, but better. Strips anything that looks like markup from username. else: pass if input_params[0] == 'guide': #necessary? member['title'] = tools.titleFromPath(member['page path']) #new function to get title from path infobox = profile.getPageText(0) member = profile.scrapeInfobox(member, infobox) else: pass member['profile'] = profile.formatProfile(input_params, member) return member
def getLearnings(grant_pages): #should this always be the same name
for page in grant_pages:
report = profiles.Profiles(page['page path'],
page['page id'],
settings=params)
rep_secs = report.getPageSectionData()
for sec in rep_secs:
if sec['title'] == 'Lessons learned':
page['lessons'] = report.getPageText(sec['index'])
page['title'] = page['page path'][7:-7]
page['formatted'] = report.formatProfile(page)
# print page['lessons']
continue
else:
pass
grant_pages = [p for p in grant_pages if p.has_key("lessons")]
return grant_pages
def prepOutput(member_list, input_params, params, tools):
member_list = tools.dedupeMemberList(member_list, 'datetime', 'page path') #not the ideal place for this
member_list.sort(key=operator.itemgetter('datetime'), reverse=True) #abstract this?
###MOVE THIS IF###
if input_params[2] == 'new': #won't work yet. need to do this earlier
member_list = member_list[0:10] #Do this in an earlier step? also, 'new' is no longer a valid subtype!
else:
pass
all_profiles = params['header template'] + '\n'.join(member['profile'] for member in member_list)
# print all_profiles
if 'subpage' in params[input_params[2]]:
sb_page = params[input_params[2]]['subpage']
else: sb_page = ""
if 'output section' in params[input_params[2]]:
edit_sec = params[input_params[2]]['output section']
else: edit_sec = ""
edit_summ = params['edit summary'] % (input_params[1] + " " + input_params[0])
output = profiles.Profiles(params['output path'], profile_type) #stupid tocreate a new profile object here.
output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page, edit_sec)
def prepOutput(featured_list):
first_subpage = params[params['subtype']]['first subpage']
number_featured = params[params['subtype']]['number featured']
featured_list = tools.addDefaults(featured_list)
output = profiles.Profiles(
params[params['subtype']]['output path'], settings=params
) #stupid tocreate a new profile object here. and stupid to re-specify the path below
i = first_subpage
for f in featured_list:
if i <= first_subpage + (number_featured - 1):
f['profile'] = output.formatProfile(f)
f['profile'] = params['header template'] + '\n' + f['profile']
edit_summ = params['edit summary'] % (params['subtype'] + " " +
params['type'])
output.publishProfile(f['profile'],
params[params['subtype']]['output path'],
edit_summ,
sb_page=i)
i += 1
else:
break
def getFeaturedProfiles():
"""
Gets info about the top-billed profiles in a guide.
"""
featured_list = []
profile_page = profiles.Profiles(
params[params['subtype']]['input page path'],
params[params['subtype']]['input page id'], params)
profile_list = profile_page.getPageSectionData(
level=params[params['subtype']]['profile toclevel'])
for profile in profile_list:
# print profile
text = profile_page.getPageText(profile['index'])
profile = profile_page.scrapeInfobox(profile, text)
if len(profile['summary']) > 1 and len(profile['image']) > 1:
profile['action'] = params[params['subtype']]['action']
profile['summary'] = tools.formatSummaries(profile['summary'])
featured_list.append(profile)
shuffle(featured_list)
featured_list = featured_list[:params[params['subtype']]
['number featured']]
return featured_list
def prepOutput(featured_list):
subpage = params[params['subtype']]['first subpage']
i = 0
number_featured = params[params['subtype']]['number featured']
featured_list = tools.addDefaults(featured_list)
output = profiles.Profiles(
params[params['subtype']]['output path'], settings=params
) #stupid tocreate a new profile object here. and stupid to re-specify the path below
for f in featured_list:
if i < number_featured:
f['profile'] = output.formatProfile(f)
f['profile'] = params['header template'] + '\n' + f['profile']
edit_summ = params['edit summary'] % (params['subtype'] + " " +
params['type'])
output.publishProfile(f['profile'],
params[params['subtype']]['output path'],
edit_summ,
sb_page=subpage)
i += 1
subpage += 1 #should fix idealab gallery to make this work there, too
else:
break
def getMemberData(member):
profile = profiles.Profiles(member['page path'], id=member['page id'], settings = params)
infobox = profile.getPageText(0) #zero is the top section
member = profile.scrapeInfobox(member, infobox, redict = params['infobox params'])
revs = []
main_revs = profile.getPageEditInfo()
revs.extend(main_revs)
if member['talkpage id']:
talk_revs = profile.getPageEditInfo(page = member['talkpage id'])
if talk_revs:
revs.extend(talk_revs)
revs.sort(key=operator.itemgetter('revid'), reverse=True)
member['timestamp'] = revs[0]['timestamp']
revs.sort(key=operator.itemgetter('revid'), reverse=False) #this doesn't seem to be working
member['username'] = "******" + revs[0]['user'] + "]]"
member['created'] = revs[0]['timestamp']
if params['formatted fields']:
for field in params['formatted fields']: #do I need the 'if' above this line?
try:
member[field] = tools.formatSummaries(member[field])
except:
pass
return member
def rankProfiles():
"""
rank IdeaLab profiles by number of recent edits.
"""
profile_page = profiles.Profiles(params['output path'],
params['output page id'], params)
profile_list = profile_page.getPageSectionData()
for profile in profile_list:
profile['title'].encode("utf8") #so redundant!
profile['text'] = profile_page.getPageText(profile['index'])
main_edits = profile_page.getUserRecentEditInfo(
profile['title'], params['main namespace'])
talk_edits = profile_page.getUserRecentEditInfo(
profile['title'], params['talk namespace'])
profile['edits'] = main_edits + talk_edits
plist_sorted = sorted(profile_list,
key=lambda item: item['edits'],
reverse=True)
plist_text = {'profiles': '\n\n'.join([x['text'] for x in plist_sorted])}
formatted_profiles = profile_page.formatProfile(plist_text)
edit_summ = params['edit summary'] % (params['type'], )
profile_page.publishProfile(formatted_profiles, params['output path'],
edit_summ)
def getFeaturedProfiles():
"""
Gets info about the top-billed profiles in a guide.
"""
featured_list = []
profile_page = profiles.Profiles(
params[params['subtype']]['input page path'],
params[params['subtype']]['input page id'], params)
profile_list = profile_page.getPageSectionData(
level=params[params['subtype']]['profile toclevel'])
for profile in profile_list:
text = profile_page.getPageText(profile['index'])
profile = profile_page.scrapeInfobox(profile, text)
if params['subtype'] == 'intro':
profile['page path'] = params[params['subtype']][
'input page path'] + "#" + profile['title'].lstrip()
if len(profile['name']) > 1:
profile['title'] = profile['name']
else:
pass
featured_list.append(profile)
elif params['subtype'] == 'question':
profile['summary'] = re.sub(
"\=\=(.*?)\=\=", "",
text) #remove the question title from the summary.
profile['page path'] = params[params['subtype']][
'input page path'] + "#" + profile['title'].lstrip()
featured_list.append(profile)
else:
if len(profile['summary']) > 1:
featured_list.append(profile)
else:
pass
for f in featured_list:
f['action'] = params[params['subtype']]['action']
f['summary'] = tools.formatSummaries(f['summary'])
return featured_list
def __init__(self):
#create a list of student Object
#self__list_student will store object of Student class
self.__list_student = []
self.__list_job = []
self.__list_settings = []
self.__list_profiles = []
##############################################################
#for epic5
self.__list_pending_friend = []
self.__list_accept_friend = []
###############################################################
#add title for the student_data.csv
if not os.path.isfile(FILENAME):
with open(FILENAME, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(
("User_Name", "Password", "First_Name", "Last_Name"))
#add data from student_data.csv to __self.__list_student
with open(FILENAME, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_student.append(
s.Student(row[0], row[1], row[2], row[3]))
#add title for the job_data.csv
if not os.path.isfile(FILENAME_JOB):
with open(FILENAME_JOB, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(("Title", "Description", "Employer",
"Location", "Salary", "Post_Name"))
#add data from job_data to __self.__list_job
with open(FILENAME_JOB, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_job.append(
j.Job(row[0], row[1], row[2], row[3], row[4], row[5]))
#add title for the settings.csv
if not os.path.isfile(FILENAME_STG):
with open(FILENAME_STG, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(("user", "email_notif", "sms_notif",
"targeted_ads", "language"))
#add data from settings.csv to __self.__list_settings
with open(FILENAME_STG, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_settings.append(
stg.Settings(row[0], row[1], row[2], row[3], row[4]))
#add title for the profiles.csv
if not os.path.isfile(FILENAME_PRO):
with open(FILENAME_PRO, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(("user", "title", "major", "university",
"bio", "experience", "education"))
#add data from profiles.csv
with open(FILENAME_PRO, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_profiles.append(
pro.Profiles(row[0], row[1], row[2], row[3], row[4],
row[5], row[6]))
############################### BEGIN ###############################################
#new for epic5
#add title for the Pending_Friends.csv
if not os.path.isfile(FILENAME_PENDING):
with open(FILENAME_PENDING, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(("Request_name", "Pending_name"))
#add data from Pending_Friends.csv to __self.__Pending_Friend
with open(FILENAME_PENDING, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_pending_friend.append(
pf.Pending_Friend(row[0], row[1]))
#add title for the accept_friends.csv
if not os.path.isfile(FILENAME_ACCEPT):
with open(FILENAME_ACCEPT, "w") as file:
writer_csv = csv.writer(file)
writer_csv.writerow(("Request", "Accept"))
#add data from accept_friends.csv to __self.__accept_friend
with open(FILENAME_ACCEPT, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_accept_friend.append(
af.Accept_Friend(row[0], row[1]))
def __init__(self):
#create a list of student Object
#self__list_student will store object of Student class
self.__list_student = []
self.__list_job = []
self.__list_settings = []
self.__list_profiles = []
self.__list_save_job = []
if not os.path.isfile(FILENAME_SAVE_JOB):
with open(FILENAME_SAVE_JOB, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("User_Name","Title"))
with open(FILENAME_SAVE_JOB, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_save_job.append(sa.Save(row[0], row[1]))
if not os.path.isfile(FILENAME_APP):
with open(FILENAME_APP, "w") as file:
writer_csv = csv.writer(file)
#add title for the student_data.csv
if not os.path.isfile(FILENAME):
with open(FILENAME, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("User_Name","Password","First_Name","Last_Name","Tier", "DayOfLastJobApplication"))
#add data from student_data.csv to __self.__list_student
with open(FILENAME, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_student.append(
s.Student(row[0], row[1], row[2], row[3]))
#add title for the job_data.csv
if not os.path.isfile(FILENAME_JOB):
with open(FILENAME_JOB, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("Title","Description","Employer","Location","Salary","Post_Name"))
#add data from job_data to __self.__list_job
with open(FILENAME_JOB, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_job.append(
j.Job(row[0], row[1], row[2], row[3], row[4], row[5]))
#add title for the settings.csv
if not os.path.isfile(FILENAME_STG):
with open(FILENAME_STG, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("user", "email_notif","sms_notif","targeted_ads","language"))
#add data from settings.csv to __self.__list_settings
with open(FILENAME_STG, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_settings.append(
stg.Settings(row[0], row[1], row[2], row[3], row[4]))
#add title for the profiles.csv
if not os.path.isfile(FILENAME_PRO):
with open(FILENAME_PRO, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("user", "title","major","university","bio", "experience", "education"))
#add data from profiles.csv
with open(FILENAME_PRO, "r") as file:
reader_csv = csv.reader(file)
for row in reader_csv:
if row != []:
self.__list_profiles.append(
pro.Profiles(row[0], row[1], row[2], row[3], row[4],
row[5], row[6]))
#add title for the pending_messages.csv
if not os.path.isfile(FILENAME_PRO):
with open(FILENAME_MES, "w") as file:
writer_csv = csv.writer(file)
#add title for the new_jobs.csv
if not os.path.isfile(FILENAME_NEW_JOB):
with open(FILENAME_NEW_JOB, "w") as file:
writer_csv = csv.writer(file)
#writer_csv.writerow(("jobTitle", "List of NOT seen"))
#create del_job_app file
if not os.path.isfile(FILENAME_DEL_JOB):
with open(FILENAME_DEL_JOB, "w") as file:
writer_csv = csv.writer(file)
def prepOutput(member_list): all_profiles = params[params['subtype']]['header template'] + '\n'.join(member['profile'] for member in member_list if 'profile' in member.keys())#if no profile field was created for some reason, ignore edit_summ = params['edit summary'] % (params['type'] + " " + params['subtype']) output = profiles.Profiles(params['output path'], params['type']) #stupid tocreate a new profile object here. output.publishProfile(all_profiles, params['output path'], edit_summ, sb_page = params[params['subtype']]['subpage'], edit_sec = params['output section'])
def getMemberData(member): profile = profiles.Profiles(member['page path'], id=member['page id'], settings = params) infobox = profile.getPageText(section = 0) #zero is the top section member = profile.scrapeInfobox(member, infobox, trans_tag = "<translate>") member['profile'] = profile.formatProfile(member) return member