def break_into_chunks(listin, pat='^# '):
"""Break a list of md text lines into a list of chunks based on
pat. pat is used to determine whether we are breaking into sections
or subsections."""
list_out = txt_mixin.txt_list([])
inds = listin.findallre(pat)
# if we are searching for subsections on a slide that has none,
# inds will be empty
if not inds:
return [listin]
start_ind = inds[0]
if start_ind != 0:
first_chunk = listin[0:start_ind]
list_out.append(first_chunk)
for end_ind in inds[1:]:
chunk = listin[start_ind:end_ind]
list_out.append(chunk)
start_ind = end_ind
last_chunk = listin[start_ind:]
list_out.append(last_chunk)
return list_out
def __init__(self, raw_list):
#print('raw_list=')
#print('\n'.join(raw_list))
self.title = raw_list.pop(0)
self.dec_line = raw_list.pop(0)
clean_content = clean_list(raw_list)
self.content = txt_mixin.txt_list(clean_content)
grade_inds = self.content.findall(':grade:`')
if len(grade_inds) > 0:
grades = None
for ind in grade_inds:
grade_line = self.content[ind]
q = grade_pat.search(grade_line)
grade_str = q.group(1)
try:
cur_grade = float(grade_str)
except ValueError:
print('Could not convert grade to float: %s' % grade_str)
cur_grade = None
if grades is None:
grades = [cur_grade]
else:
grades.append(cur_grade)
#print('grades='+str(grades))
filt_grades = filter(None, grades)
if len(filt_grades) == 0:
self.grade = None
else:
self.grade = mean(filt_grades)
def validate_spreadsheet(self, path, debug=1):
db = txt_database.db_from_file(path)
mylist = txt_mixin.txt_list(db.labels)
def get_db_attr_by_pat(pat):
inds = mylist.findallre(pat)
assert len(inds) > 0, "did not find a match for %s in labels" % pat
assert len(inds) == 1, "found more than one match for %s" %s
ind = inds[0]
fn_label = db.labels[ind]
attr = db.attr_label_dict[fn_label]
vect = getattr(db, attr)
return vect
fn_pat = '[Ff]irst[ _][Nn]ame'
try:
self.first_names = get_db_attr_by_pat(fn_pat)
except:
print('delimited text file must have a column label that matches %s' % \
fn_pat)
self.emails = get_db_attr_by_pat('[Ee]mail')
if debug:
print('first_names:' + str(self.first_names))
print('emails:' + str(self.emails))
check_for_blanks(self.first_names)
check_for_blanks(self.emails)
assert len(self.first_names) == len(self.emails), \
"lengthes of first names and emails do not match"
def Execute(self, namespace, figfolder='figs',\
def_ext='.png', dpi=100, **kwargs):
if not os.path.exists(figfolder):
os.mkdir(figfolder)
python_report_env.Execute(self, namespace=namespace, **kwargs)
keys = ['caption','filename','label']
mypat = '^#('+'|'.join(keys)+')'
comments = [item for item in self.list if item.find('#')==0]
if comments[0].find('#pyfig') == 0:
comments.pop(0)
com_list = txt_mixin.txt_list(comments)
start_inds = com_list.findallre(mypat)
end_inds = start_inds[1:]+[None]
pat2 = '^#('+'|'.join(keys)+')'+':(.*)'
p2 = re.compile(pat2)
keysfound = []
for si, ei in zip(start_inds, end_inds):
chunk = ''.join(com_list[si:ei])
q2 = p2.match(chunk)
if q2:
key = q2.group(1)
body = q2.group(2)
body = body.replace('#',' ')
setattr(self, key, body)
keysfound.append(key)
assert 'filename' in keysfound, "#filename: was not found in " + \
self.code +'\n'*2+ \
'NOTE: it must be in the beginning comments.'
fno, ext = os.path.splitext(self.filename)
if not ext:
ext = def_ext
self.nameout = fno+ext
self.pathout = os.path.join(figfolder, self.nameout)
PU.mysave(self.pathout, dpi=dpi)
def __init__(self, raw_list):
#print('raw_list=')
#print('\n'.join(raw_list))
self.title = raw_list.pop(0)
self.dec_line = raw_list.pop(0)
clean_content = clean_list(raw_list)
self.content = txt_mixin.txt_list(clean_content)
grade_inds = self.content.findall(':grade:`')
if len(grade_inds) > 0:
grades = None
for ind in grade_inds:
grade_line = self.content[ind]
q = grade_pat.search(grade_line)
grade_str = q.group(1)
try:
cur_grade = float(grade_str)
except ValueError:
print('Could not convert grade to float: %s' % grade_str)
cur_grade = None
if grades is None:
grades = [cur_grade]
else:
grades.append(cur_grade)
#print('grades='+str(grades))
filt_grades = filter(None, grades)
if len(filt_grades) == 0:
self.grade = None
else:
self.grade = mean(filt_grades)
def to_Maxima(self, pathout, attrlist=['U0'], num_bodes=2, \
base_mod_name='maxima_bode', ND=True, **kwargs):
"""Create a Maxima batch file for the system. attrlist is a
list of strings referring to the element matrices. The
elements of attrlist should be in order, starting with U0 and
stopping at U_n."""
#consider adding ratdenom and ratnumer
mylist = txt_mixin.txt_list()
out = mylist.append
out('showtime:all$')
out('nolabels:true$')
#ratvars(mubz,EIbz,Lbz,abz,betabz,c1bz,c2bz,c3bz,c4bz,s)$
out('grind:true$')
for attr in attrlist:
U = getattr(self, attr)
Uline = sympy_utils.matrix_to_Maxima_string(U,attr)
out(Uline)
Usys_line = self.Usys_Maxima(attrlist)
out(Usys_line)
mylist.extend(maxima_bv)
bode_lines = self.Maxima_bodes(ND=ND, **kwargs)
mylist.extend(bode_lines)
save_lines = self.save_Maxima_bodes_to_Fortran(num_bodes, \
base_mod_name, \
ND=ND)
mylist.extend(save_lines)
self.maxima_list = mylist
txt_mixin.dump(pathout, mylist)
def Find_Ratios(self):
self.ratios = []
for firstname, team in zip(self.firstnames, self.teams):
firstname = firstname.capitalize()
ratings_sheet = team.ratings_sheet
if ratings_sheet is None:
self.ratios.append(1.0)
else:
search_names = ratings_sheet.first_names
mylist = txt_mixin.txt_list(search_names)
inds = mylist.findall(firstname)
if len(inds) == 0:
if alternates.has_key(firstname):
inds = mylist.findall(alternates[firstname])
if len(inds) == 1:
ratio = float(ratings_sheet.ratios[inds[0]])
if ratio > 1.1:
ratio = 1.1
elif ratio < 0.8:
ratio = 0.8
self.ratios.append(ratio)
else:
print('===================')
print('did not find match for ' + firstname)
print('in ' + str(mylist))
print('===================')
self.ratios.append(-1)
def list_pdfs_find_handout(pdf_files):
"""Given a list of pdfs from a glob search of a directory, find
the _handout.pdf file and remove it corresponding non-handout file
if found. Return the handout file name and the rest of the pdf
files."""
pdflist = txt_mixin.txt_list(pdf_files)
handout_pat = '_handout.pdf'
handoutinds = pdflist.findall(handout_pat)
if len(handoutinds) == 1:
handoutfile = pdflist[handoutinds[0]]
non_handout_name = handoutfile.replace('_handout','')
non_handout_inds = pdflist.findall(non_handout_name)
if len(non_handout_inds) == 1:
pdflist.pop(non_handout_inds[0])
handout_name = handoutfile
listout = []
# pop the handoutfile from the original list
handoutind = pdflist.find(handoutfile)
pdflist.pop(handoutind)
listout.extend(pdflist)
else:
handout_name = None
listout = pdflist
return handout_name, listout
def Find_Ratios(self):
self.ratios = []
for firstname, team in zip(self.firstnames, self.teams):
firstname = firstname.capitalize()
ratings_sheet = team.ratings_sheet
if ratings_sheet is None:
self.ratios.append(1.0)
else:
search_names = ratings_sheet.first_names
mylist = txt_mixin.txt_list(search_names)
inds = mylist.findall(firstname)
if len(inds) == 0:
if alternates.has_key(firstname):
inds = mylist.findall(alternates[firstname])
if len(inds) == 1:
ratio = float(ratings_sheet.ratios[inds[0]])
if ratio > 1.1:
ratio = 1.1
elif ratio < 0.8:
ratio = 0.8
self.ratios.append(ratio)
else:
print('===================')
print('did not find match for '+firstname)
print('in '+str(mylist))
print('===================')
self.ratios.append(-1)
def open_txt_list(pathin=None):
if pathin is None:
pathin = open_txt()
print('pathin = %s' % pathin)
myfile = txt_mixin.txt_file_with_list(pathin)
mylist = filter(None, myfile.list)
mylist = txt_mixin.txt_list(mylist)
return mylist
def __init__(self, listin, pat=secpat, subclass=None, sublevel=0):
self.list = txt_mixin.txt_list(listin)
self.pat = pat
self.re_p = re.compile(self.pat)
if subclass is None:
subclass = section
self.subclass = subclass
self.sublevel = sublevel
def __init__(self, pathout, title=None):
self.pathout = pathout
self.list = txt_mixin.txt_list([])
if title is not None:
self.insert_title(title)
self.list.append('.. include:: /home/ryan/git/report_generation/header.rst')
self.list.append('')
def get_names(self):
last_name_col = self.find_col_from_list(['lastname','last name', \
'lname', 'lastnames', \
'last names','lnames'])
self.last_names = self.data[:,last_name_col]
self.last_name_col = last_name_col
has_first = self.look_for_first_names()
if has_first:
names = []
for first, last in zip(self.first_names, self.last_names):
curname = last + ', ' + first
names.append(curname)
self.names = txt_mixin.txt_list(copy.copy(names))
else:
self.names = txt_mixin.txt_list(copy.copy(self.last_names))
def _clean_lines(self):
self.clean_lines = txt_mixin.txt_list(self.linesin)
secinds = self.clean_lines.findall('\\section{')
assert len(secinds) < 2, 'Found more than one \\section within a \\subsection'
if len(secinds) == 1:
self.clean_lines.pop(secinds[0])
while not self.clean_lines[-1]:
self.clean_lines.pop(-1)
def _clean_comments(listin):
listout = txt_mixin.txt_list([])
for line in listin:
ind = line.find('#')
if ind == 0:
line = ''
elif ind > 0:
line = line[0:ind]
listout.append(line)
return listout
def _find_matrices_in_header(self):
"""This is the main method for extracting the system matrices
from the comments at the beginning of the data file."""
if not hasattr(self, 'header'):
self._get_header()
self.mat_rows = txt_mixin.txt_list(self.header[1:-1])
self.clean_mat_rows()
self.find_ind_mat_lines()
self.build_mat_code_dict()
self.assign_mat_code_to_attrs()
def load_email_list(pathin):
f = open(pathin)
lines = f.readlines()
f.close()
stringin = ', '.join(lines)
stringin = stringin.strip()
mylist = stringin.split(',')
clean_list = [item.strip() for item in mylist]
out_list = txt_mixin.txt_list(clean_list)
return out_list
def load_email_list(pathin):
f = open(pathin)
lines = f.readlines()
f.close()
stringin = ', '.join(lines)
stringin = stringin.strip()
mylist = stringin.split(',')
clean_list = [item.strip() for item in mylist]
out_list = txt_mixin.txt_list(clean_list)
return out_list
def __init__(self, listin, map_in=None, preface='^'):
self.list = txt_mixin.txt_list(listin)
self.map = map_in
self.keys = self.map.keys()
self.keystr = '|'.join(self.keys)
self.preface = preface
self.pat = self.preface+'('+self.keystr+'){'
self.p = re.compile(self.pat)
self.lines = copy.copy(listin)
self.ind = 0
def __init__(self, lines):
assert lines[0] == ' {', "problem with first line of cell"
assert lines[-1].find(' }') == 0, "problem with last line of cell"
assert lines[1].find('"cell_type":') == 3, "problem with second line"
temp, celltype = lines[1].split(":")
celltype = celltype.strip()
self.celltype = celltype
self.clean_celltype()
self.lines = txt_mixin.txt_list(lines)
self.find_source()
def find_members(self):
lastnames, firstnames = group_list.get_names(self.group_name)
self.lastnames = lastnames
self.firstnames = firstnames
self.firstnames = txt_mixin.txt_list(firstnames)
for n, lastname in enumerate(lastnames):
if alts.has_key(lastname):
self.firstnames[n] = alts[lastname]
self.strip_names()
self.fix_dup_firsts()
self.names = zip(self.firstnames, self.lastnames)
def __init__(self, pyppath):
self.pyp_path = pyppath
self.main_dir, self.pyp_name = os.path.split(self.pyp_path)
self.pyp_list = txt_mixin.txt_list(pytexutils.readfile(pyppath))
self.rst_list = copy.copy(self.pyp_list)
path_no_ext, ext = os.path.splitext(pyppath)
self.rst_path = path_no_ext+'.rst'
self.html_path = path_no_ext+'.html'
self.name, junk = os.path.splitext(self.pyp_name)
self.rst_name = self.name+'.rst'
self.html_name = self.name+'.html'
def find_members(self):
lastnames, firstnames = group_list.get_names(self.group_name)
self.lastnames = lastnames
self.firstnames = firstnames
self.firstnames = txt_mixin.txt_list(firstnames)
for n, lastname in enumerate(lastnames):
if alts.has_key(lastname):
self.firstnames[n] = alts[lastname]
self.strip_names()
self.fix_dup_firsts()
self.names = zip(self.firstnames, self.lastnames)
def _search_for_label(self, search_label, print_msg=True):
search_list = txt_mixin.txt_list(self.labels)
indices = search_list.findall(search_label)
if len(indices) == 0:
if print_msg:
print("did not find %s" % search_label)
return None
elif len(indices) == 1:
return indices[0]
else:
if print_msg:
print("found more than one match for %s" % search_label)
return None
def PaperToPres(pathin, pathout, figstr='height:0.75'):
"""Open an existing LaTeX document and convert it to a skeleton
pyp outline. pathin is the path to the .tex file, presumably a
conference paper or something. pathout is the path to the .pyp
file that will be the output.
This function searches for all occurances of the following
environments: eqnarray, equation, figure, lstlisting, and makes a
slide of each."""
latexlist = txt_mixin.txt_list([],'temp.tex')
latexlist.readfile(pathin)
latexlist.ReplaceInputs()
#figures
b4lines, envlines, afterlines, binds = latexlist.findenvironments('figure',getinds=True)
figlist = [textfiles.latexlist.Figure(lines, ind) for lines, ind in zip(envlines, binds)]
b4lines, eqnlines, afterlines, eqninds = latexlist.findenvironments('equation',getinds=True)
eqnlist = [textfiles.latexlist.Equation(lines, ind) for lines, ind in zip(eqnlines, eqninds)]
mydict = dict(zip(binds, figlist))
eqndict = dict(zip(eqninds, eqnlist))
mydict.update(eqndict)
inds = mydict.keys()
inds.sort()
listout = []
mycounter = textfiles.latexlist.Counter()
for ind in inds:
curitem = mydict[ind]
curlines, mycoutner = curitem.ToPresentationList(mycounter, figstr=figstr)
listout.extend(curlines)
listout.append('')
outlist = txt_mixin.txt_list(listout, pathout)
outlist.tofile()
return listout
def _search_for_label(self, search_label, print_msg=True):
search_list = txt_mixin.txt_list(self.labels)
indices = search_list.findall(search_label)
if len(indices) == 0:
if print_msg:
print("did not find %s" % search_label)
return None
elif len(indices) == 1:
return indices[0]
else:
if print_msg:
print("found more than one match for %s" % search_label)
return None
def __init__(self, listin, preface='^'):#map_in=None
self.list = txt_mixin.txt_list(listin)
#self.map = map_in
#self.keys = self.map.keys()
#self.keystr = '|'.join(self.keys)
self.preface = preface
self.pat = self.preface + '\.\. (py|pyno)::'
self.p = re.compile(self.pat)
self.lines = copy.copy(listin)
self.ind = 0
self.pat2 = "^[ \t]+:label:"
self.p2 = re.compile(self.pat2)
self.pat_code = '^([ \t]+)'#for finding white_space
self.pcode = re.compile(self.pat_code)
def Get_Data(self):
if not hasattr(self, 'label_inds'):
self.find_inds_for_data()
self.data = []
for start, end in zip(self.label_inds, self.total_inds):
curdata = self.list[start:end+1]
#"Total Valid" needs a blank cell next to it
temp = txt_mixin.txt_list(curdata)
ind = temp.findnext("Total Valid")
if ind:
temp[ind] = temp[ind].replace('"Total Valid"','"Total Valid",')
self.data.append(temp)
return self.data
def add_header(self, headername='header.tex',\
headerinserts=[]):
header = []
if self.dir:
search_dir = self.dir
else:
search_dir = os.getcwd()
headerpath = os.path.join(search_dir, headername)
header_list = pytexutils.readfile(headerpath)
header.extend(header_list)
header.extend(headerinserts)
bdstr = '\\begin{document}'
if bdstr not in header:
header.append(bdstr)
self.header = txt_mixin.txt_list(header)
def retreive_time_and_app(self):
N = len(self.team_name)
app_grades = zeros(N)
times = ['']*N
time_penalty = zeros(N)
other_team_list = txt_mixin.txt_list(self.time_app_sheet.team_name)
for i, name in enumerate(self.team_name):
ind = other_team_list.find(name)
times[i] = self.time_app_sheet.time[ind]
app_grades[i] = float(self.time_app_sheet.appearance[ind])
self.time_strs = times
self.times = [parse_time_string(item) for item in times]
self.time_penalties = array([get_timing_grade(time) for time in self.times])
self.appearance = app_grades
def Get_Data(self):
if not self.clean:
self.Run()
self.data_lines = []
if not hasattr(self, 'end_inds'):
self.find_inds_for_data()
self.data = []
for start, end in zip(self.start_inds, self.end_inds):
curdata = self.list[start:end+1]
#"Total Valid" needs a blank cell next to it
temp = txt_mixin.txt_list(curdata)
self.data.append(temp)
return self.data
def __init__(self, pathin, lectnum=1):
txt_mixin.txt_file_with_list.__init__(self, pathin)
self.next_eqn = 1
self.lectnum = lectnum
self.webroot = webroot_pat % lectnum
self.pop_initial_blank_lines()
self.process_includes()
self.pop_header()
self.remove_skip_lines()
self.pop_initial_blank_lines()
start_ind = self.find_start()
stop_ind = self.find_stop()
#Pdb().set_trace()
self.list = txt_mixin.txt_list(self.list[start_ind:stop_ind])
self.pop_initial_blank_lines()
self.find_sections()
def retreive_time_and_app(self):
N = len(self.team_name)
app_grades = zeros(N)
times = [''] * N
time_penalty = zeros(N)
other_team_list = txt_mixin.txt_list(self.time_app_sheet.team_name)
for i, name in enumerate(self.team_name):
ind = other_team_list.find(name)
times[i] = self.time_app_sheet.time[ind]
app_grades[i] = float(self.time_app_sheet.appearance[ind])
self.time_strs = times
self.times = [parse_time_string(item) for item in times]
self.time_penalties = array(
[get_timing_grade(time) for time in self.times])
self.appearance = app_grades
def parse(self):
colA = txt_mixin.txt_list(self.get_col(0))
inds = colA.findall('name', forcestart=1)
inds.append(None)
chunks = []
ratings = []
prevind = inds[0]
for ind in inds[1:]:
cur_chunk = self.alldata[prevind:ind]
chunks.append(cur_chunk)
rating = one_rating(cur_chunk)
ratings.append(rating)
prevind = ind
self.ratings = ratings
self.categories = ratings[0].categories
self.keys = ratings[0].keys
return ratings
def parse(self):
colA = txt_mixin.txt_list(self.get_col(0))
inds = colA.findall('name', forcestart=1)
inds.append(None)
chunks = []
ratings = []
prevind = inds[0]
for ind in inds[1:]:
cur_chunk = self.alldata[prevind:ind]
chunks.append(cur_chunk)
rating = one_rating(cur_chunk)
ratings.append(rating)
prevind = ind
self.ratings = ratings
self.categories = ratings[0].categories
self.keys = ratings[0].keys
return ratings
def cse_to_file(expr_list, filename, outlabels, funcname, inputs=[], ws=" " * 4, headerfile=None, replace_dict={}):
line0 = "from __future__ import division"
line1 = "from scipy import *"
line2 = "def " + funcname + "(" + ", ".join(inputs) + "):"
preamble = [line0, line1, "", line2]
mylist = []
if headerfile:
headerlist = txt_mixin.read(headerfile)
mylist.extend(headerlist)
mylist.extend(cse_to_txtlist(expr_list, outlabels, ws=ws))
if replace_dict:
mylist = txt_mixin.txt_list(mylist)
for key, value in replace_dict.iteritems():
mylist.replaceall(key, value)
mylist = preamble + mylist # don't do the search and replace in the
# preamble
txt_mixin.dump(filename, mylist)
def cse_to_file(expr_list, filename, outlabels, funcname, \
inputs=[], ws=' '*4, headerfile=None, \
replace_dict={}):
line0 = 'from __future__ import division'
line1 = 'from scipy import *'
line2 = 'def '+funcname +'(' + ', '.join(inputs) + '):'
preamble = [line0, line1, '', line2]
mylist = []
if headerfile:
headerlist = txt_mixin.read(headerfile)
mylist.extend(headerlist)
mylist.extend(cse_to_txtlist(expr_list, outlabels, ws=ws))
if replace_dict:
mylist = txt_mixin.txt_list(mylist)
for key, value in replace_dict.iteritems():
mylist.replaceall(key,value)
mylist = preamble + mylist#don't do the search and replace in the
#preamble
txt_mixin.dump(filename, mylist)
def build_speaking_rst(self, runlatex=1):
self.speaking_lines = txt_mixin.txt_list(self.get_speaking_lines())
dec_inds = self.speaking_lines.findallre('^\++$')
start_inds = [item - 1 for item in dec_inds]
speaker_list = None
start_inds.append(None)
prev_ind = start_inds.pop(0)
for ind in start_inds:
curlines = self.speaking_lines[prev_ind:ind]
curspeaker = speaker(curlines, self)
curspeaker.build_rst()
curspeaker.save_rst()
if runlatex:
curspeaker.run_rst()
else:
curspeaker.set_pdfpath()
if speaker_list is None:
speaker_list = [curspeaker]
else:
speaker_list.append(curspeaker)
prev_ind = ind
self.speaker_list = speaker_list
def strip_names(self):
firstnames = [item.strip() for item in self.firstnames]
lastnames = [item.strip() for item in self.lastnames]
self.firstnames = txt_mixin.txt_list(firstnames)
self.lastnames = txt_mixin.txt_list(lastnames)
def __init__(self, pathin):
data, labels = txt_database._open_txt_file(pathin)
txt_database.txt_database.__init__(self, data, labels)
self.last_name_list = txt_mixin.txt_list(self.Last_Name)
self.first_name_list = txt_mixin.txt_list(self.First_Name)
def _load_emails(self):
raw_list = txt_mixin.txt_file_with_list(self.raw_email_filename)
emails = raw_list.list[0].split(',')
emails = [item.strip() for item in emails]
mylist = txt_mixin.txt_list(emails)
self.emails = mylist
import txt_mixin
#from IPython.core.debugger import Pdb
#from pygimp_lecture_utils import lecturerc_path
from pygimp_lecture_pickle_path import lecturerc_path
home_dir = os.path.expanduser('~')
lecture_outline_css_path = rwkos.FindFullPath('git/report_generation/lecture_outline.css')
#rst_line1 = '.. include:: %s/git/report_generation/beamer_header.rst' % home_dir
#rst_title_line = '`\mytitle{@@TITLE@@}`'
#rst_list = txt_mixin.txt_list([rst_line1, '', rst_title_line, ''])
rst_list = txt_mixin.txt_list([])
#replaceall(self, findpat, rep, forcestart=0):
def list_pdfs_find_handout(pdf_files):
"""Given a list of pdfs from a glob search of a directory, find
the _handout.pdf file and remove it corresponding non-handout file
if found. Return the handout file name and the rest of the pdf
files."""
pdflist = txt_mixin.txt_list(pdf_files)
handout_pat = '_handout.pdf'
handoutinds = pdflist.findall(handout_pat)
if len(handoutinds) == 1:
handoutfile = pdflist[handoutinds[0]]
non_handout_name = handoutfile.replace('_handout','')
def assign_lastnames(self, lastnames):
self.lastnames = txt_mixin.txt_list(lastnames)