class NGMParser(object):
"""
Simple prottest result parser.
"""
def __init__(self):
self.VAILD_PAIRS = Literal('[MAIN] Valid pairs found:')
self.INSERT_SIZE = Literal('[MAIN] Estimated insert size:')
self.COMPUTED_ALIGNMENTS = Literal('[MAIN] Alignments computed:')
# These are all the models that are possible to be tested using phyml
self.vp = Suppress(SkipTo(self.VAILD_PAIRS)) + Suppress(self.VAILD_PAIRS) + WORD
self.inssize = Suppress(SkipTo(self.INSERT_SIZE)) + Suppress(self.INSERT_SIZE) + FLOAT
self.compalign = Suppress(SkipTo(self.COMPUTED_ALIGNMENTS)) + Suppress(self.COMPUTED_ALIGNMENTS) + FLOAT
def parse(self, stdout):
try:
valid_pairs = self.vp.parseString(stdout).asList()[0]
insert_size = self.inssize.parseString(stdout).asList()[0]
computed_alignments = self.compalign.parseString(stdout).asList()[0]
except ParseException as err:
logger.error(err)
return valid_pairs, insert_size, computed_alignments
def to_dict(self, file, stdout):
valid_pairs, insert_size, computed_alignments = self.parse(stdout)
samfile = pysam.AlignmentFile(file, "r")
result = {'file': file,
'valid_pairs': valid_pairs,
'insert_size': insert_size,
'computed_alignments': computed_alignments,
'sam': samfile}
return result
class NGMLRParser(object):
"""
Simple prottest result parser.
for the following example output line:
Processed: 75400 (0.00), R/S: 60.15, RL: 7675, Time: 3.00 11.00 10.07, Align: 1.00, 310, 3.04
Done (77 reads mapped (0.10%), 75323 reads not mapped, 75402 lines written)(elapsed: 20m, 0 r/s)
"""
def __init__(self):
self.TOTAL_MAPPED_READS = Literal('Done (')
self.TOTAL_READS = Literal('Processed: ')
# These are all the models that are possible to be tested using phyml
self.tmr = Suppress(SkipTo(self.TOTAL_MAPPED_READS)) + Suppress(self.TOTAL_MAPPED_READS) + FLOAT
self.tr = Suppress(SkipTo(self.TOTAL_READS)) + Suppress(self.TOTAL_READS) + FLOAT
def parse(self, stdout):
try:
total_mapped_reads = self.tmr.parseString(stdout).asList()[0]
total_reads = self.tr.parseString(stdout).asList()[0]
except ParseException as err:
logger.error(err)
return total_mapped_reads, total_reads
def to_dict(self, file, stdout):
total_mapped_reads, total_reads = self.parse(stdout)
samfile = pysam.AlignmentFile(file, "r")
result = {'file': file,
'total_mapped_reads': total_mapped_reads,
'total_reads': total_reads,
'sam': samfile}
return result
def append_urlX_to_X(target, source, **kwargs):
with closing(urlopen(source.url, timeout=kwargs.pop('timeout', None))) as r:
chunk_size = 16 * source.chunk_size
# 11.12.18 harleen - part2 - hack to enable writing to ftp server
# original
# with open(target.path, 'wb') as fp:
# for chunk in iter(curry(r.read, chunk_size), b''):
# fp.write(chunk)
# return target
# changes begin
f = target.path
if f.split(':')[0].lower() == 'ftp':
import io
from io import StringIO
from ftplib import FTP
from pyparsing import Combine, Suppress, Word, printables, alphas, ZeroOrMore, alphanums, Group, delimitedList, nums
ftp_expression = Suppress('ftp://') + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(alphanums) + ZeroOrMore(Suppress('@')) + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(nums) + ZeroOrMore(Suppress('/')) + Word(printables)
username, password, ip, port, file_loc = ftp_expression.parseString(f)
ftp = FTP(ip, username, password)
# buffer = StringIO()
# df.to_csv(buffer, index=False, header=has_header, sep=sep, encoding=encoding)
# text = buffer.getvalue()
# bio = io.BytesIO(str.encode(text))
# ftp.storbinary('STOR '+file_loc, bio)
for chunk in iter(curry(r.read, chunk_size), b''):
ftp.storbinary('APPE ' + file_loc, io.BytesIO(chunk))
return target
else:
with open(target.path, 'wb') as fp:
for chunk in iter(curry(r.read, chunk_size), b''):
fp.write(chunk)
return target
def new_line(self, line):
parse_vtp = Suppress('vtp ') + restOfLine
try:
self.vtp_str = parse_vtp.parseString(line).asList()[-1]
self.info_domain()
self.info_mode()
except ParseException:
pass
def create_names_db():
p = Suppress(SkipTo('"')) + QuotedString('"') + Suppress(SkipTo("."))
with open("persondata_en.ttl") as pd:
for line in pd:
surname, givenname = None, None
if "/name" in line:
name = p.parseString(line)[0]
line = pd.__next__()
if "/surname" in line:
surname = p.parseString(line)[0]
line = pd.__next__()
if "/givenName" in line:
givenname = p.parseString(line)[0]
n = Name(name=name, surname=surname, givenname=givenname)
session.add(n)
session.commit()
def read_and_parse_metrics(raw_data):
token = Suppress("'") + Word(alphanums + "_" + "-") + Suppress("'")
group = Group(
Suppress("(") + token + Suppress(",") + token + Suppress(",") +
Word(nums) + Suppress(")") + Suppress(Optional(',')))
dot = Suppress("{") + OneOrMore(group) + Suppress("}")
data = dot.parseString(raw_data).asList()
return {Conn(c[0], c[1], c[2]) for c in data}
def new_line(self,line):
parse_storm = Suppress('storm-control ') + restOfLine
try:
self.storm_line = parse_storm.parseString(line).asList()[-1]
self.level_info()
self.action_info()
self.type_info()
except ParseException:
pass
def parseReactions(reaction):
name = Word(alphanums + '_-') + ':'
species = (Word(alphanums + "_:#-") + Suppress('()')) + ZeroOrMore(
Suppress('+') + Word(alphanums + "_:#-") + Suppress("()"))
rate = Word(alphanums + "()")
grammar = Suppress(Optional(name)) + ((Group(species) | '0') + Suppress(Optional("<") + "->") +
(Group(species) | '0') + Suppress(rate)) \
^ (species + Suppress(Optional("<") + "->") + Suppress(rate))
result = grammar.parseString(reaction).asList()
if len(result) < 2:
result = [result, []]
if '<->' in reaction and len(result[0]) == 1 and len(result[1]) == 2:
result2 = [result[1], result[0]]
result = result2
return result
def parseReactions(reaction):
name = Word(alphanums + '_-') + ':'
species = (Word(alphanums + "_:#-")
+ Suppress('()')) + ZeroOrMore(Suppress('+') + Word(alphanums + "_:#-")
+ Suppress("()"))
rate = Word(alphanums + "()")
grammar = Suppress(Optional(name)) + ((Group(species) | '0') + Suppress(Optional("<") + "->") +
(Group(species) | '0') + Suppress(rate)) \
^ (species + Suppress(Optional("<") + "->") + Suppress(rate))
result = grammar.parseString(reaction).asList()
if len(result) < 2:
result = [result, []]
if '<->' in reaction and len(result[0]) == 1 and len(result[1]) == 2:
result2 = [result[1], result[0]]
result = result2
return result
class Lindenmayer(object):
def __init__(self, stream):
# Set the default image dimensions ...
self.width = 500
self.height = 500
# ... and the number of iterations.
self.iterations = 5
# Set the default rotation angle in degrees.
self.alpha = 90
# Initialize the branch stack, ...
self.stack = []
# ... the constants, the rules, the variables and the axiom ...
self.const = {'+':'+', '-':'-', '[':'[', ']':']'}
self.rules = {}
self.vars = []
self.axiom = None
# ... and drawing settings.
self.bgcolor = (1.0, 1.0, 1.0)
self.lineLength = 20
self.lineWidth = 5
self.lineColor = (0, 0, 0)
# Calculate the starting position.
self.offset = (0, -self.height*0.5)
print 'Offset :', self.offset
# Finally store the stream ...
self.stream = stream
# ... and initialize the parser.
self.initialize()
def initialize(self):
ParserElement.setDefaultWhitespaceChars(' \t\r')
integer = Regex(r"[+-]?\d+") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
number = Regex(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?") \
.setParseAction(lambda s,l,t: [ float(t[0]) ])
color = Regex(r"#([0-9a-fA-F]{6})")
angle = "'" + Regex(r"(360|3[0-5][0-9]|[12][0-9]{2}|[0-9]{1,2})") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
variable = Word(alphas, exact=1).setParseAction(self.addVar)
colon = Literal(":").suppress()
comma = Literal(",")
lBrace = Literal("(")
rBrace = Literal(")")
lBracket = Literal("[")
rBracket = Literal("]")
lAngle = Literal("<")
rAngle = Literal(">")
plus = Literal("+")
minus = Literal("-")
FTerm = Literal("F")
fTerm = Literal("f")
ZTerm = Literal("Z")
zTerm = Literal("z")
xTerm = Literal("x")
cTerm = Literal("c")
eol = OneOrMore(LineEnd()).suppress()
param = ( angle | color | "!" + number | "|" + number )
self.pList = lBrace + param + ZeroOrMore(comma + param) + rBrace
literal = ((lBracket + ( variable + Optional(self.pList)
| plus + Optional(self.pList) | minus + Optional(self.pList) ) + rBracket)
| (variable + Optional(self.pList) | plus + Optional(self.pList)
| minus + Optional(self.pList)))
terminal = (ZTerm | zTerm | FTerm | fTerm | xTerm | cTerm
| plus | minus | lBracket | rBracket)
lprod = (
(OneOrMore(terminal) + lAngle + variable + rAngle + OneOrMore(terminal))
| (OneOrMore(terminal) + lAngle + variable)
| (variable + rAngle + OneOrMore(terminal))
| variable )
rProd = OneOrMore(literal | terminal)
comment = Suppress((LineStart() + "#" + SkipTo(eol, include=True)))
rules = (
(lprod + Literal("=") + rProd + eol).setParseAction(self.addRule) \
| comment )
defaults = ( ( ("Dimensions" + colon + integer + comma + integer)
| ("Position" + colon + integer + comma + integer)
| ("Iterations" + colon + integer)
| ("Angle" + colon + angle)
| ("Linelength" + colon + number)
| ("Linewidth" + colon + number)
| ("Linecolor" + colon + color)
| ("Background" + colon + color)
| ("Axiom" + colon + rProd) ) + eol ).setParseAction(self.setAttribute)
header = ( defaults | comment )
self.grammar = Suppress(ZeroOrMore(LineEnd())) \
+ ZeroOrMore(header) \
+ OneOrMore(rules)
try:
L = self.grammar.parseString( self.stream )
except ParseException, err:
print err.line
print " "*(err.column-1) + "^"
print err
print 'Rules:', self.rules
class RaxmlParser(object):
def __init__(self):
self.ALPHA_LABEL = Regex(r'alpha\[\d+\]:')
self.LNL_LABEL = Literal('Final GAMMA-based Score of best tree')
self.FRQ_LABEL = Regex(r'Base frequencies: (?=\d+)') ^ \
Regex(r'ML estimate base freqs\[\d+\]:')
self.NAMES_LABEL = Regex(r'Partition: \d+ with name:\s+')
self.RATES_LABEL = Regex(r'rates\[\d+\].+?:')
self.MODEL_LABEL = Literal('Substitution Matrix:')
self.alpha = OneOrMore(
Suppress(SkipTo(self.ALPHA_LABEL)) + Suppress(self.ALPHA_LABEL) +
FLOAT)
self.lnl = Suppress(SkipTo(self.LNL_LABEL)) + \
Suppress(self.LNL_LABEL) + FLOAT
self.frq = OneOrMore(
Group(
Suppress(SkipTo(self.FRQ_LABEL)) + Suppress(self.FRQ_LABEL) +
OneOrMore(FLOAT)))
self.names = OneOrMore(
Suppress(SkipTo(self.NAMES_LABEL)) + Suppress(self.NAMES_LABEL) +
CharsNotIn('\n') + Suppress(LineEnd()))
self.rates = OneOrMore(
Group(
Suppress(SkipTo(self.RATES_LABEL)) +
Suppress(self.RATES_LABEL) + OneOrMore(FLOAT)))
self.model = Suppress(SkipTo(self.MODEL_LABEL)) + \
Suppress(self.MODEL_LABEL) + WORD
MODEL_LABEL = Literal('Substitution Matrix:')
SCORE_LABEL = Literal('Final GAMMA likelihood:')
BOOT_SCORE_LABEL = Literal('Final ML Optimization Likelihood:')
DESC_LABEL = Literal('Model Parameters of Partition')
NAME_LEADIN = Literal(', Name:')
DATATYPE_LEADIN = Literal(', Type of Data:')
ALPHA_LEADIN = Literal('alpha:')
TREELENGTH_LEADIN = Literal('Tree-Length:')
RATES_LABEL = Regex(r'rate \w <-> \w:')
FREQS_LABEL = Regex(r'freq pi\(\w\):')
likelihood = Suppress(
SkipTo(SCORE_LABEL)) + Suppress(SCORE_LABEL) + FLOAT
boot_likelihood = Suppress(
SkipTo(BOOT_SCORE_LABEL)) + Suppress(BOOT_SCORE_LABEL) + FLOAT
description = Suppress(
SkipTo(DESC_LABEL)) + Suppress(DESC_LABEL) + INT + Suppress(
NAME_LEADIN) + SPACEDWORD + Suppress(DATATYPE_LEADIN) + WORD
treelen = Suppress(
SkipTo(TREELENGTH_LEADIN)) + Suppress(TREELENGTH_LEADIN) + FLOAT
alpha = Suppress(SkipTo(ALPHA_LEADIN)) + Suppress(ALPHA_LEADIN) + FLOAT
rates = OneOrMore(
Group(
Suppress(SkipTo(RATES_LABEL)) + Suppress(RATES_LABEL) +
OneOrMore(FLOAT)))
freqs = OneOrMore(
Group(
Suppress(SkipTo(FREQS_LABEL)) + Suppress(FREQS_LABEL) +
OneOrMore(FLOAT)))
# output of running different set of raxml analysis
self.TC_STOCHBI_LABEL = Literal(
'Tree certainty under stochastic bipartition '
'adjustment for this tree:')
self.RTC_STOCHBI_LABEL = Literal(
'Relative tree certainty under stochastic bipartition adjustment for this tree:'
)
self.TCA_STOCHBI_LABEL = Literal(
'Tree certainty including all conflicting bipartitions (TCA) under '
'stochastic bipartition adjustment for this tree:')
self.RTCA_STOCHBI_LABEL = Literal(
'Relative tree certainty including all conflicting bipartitions (TCA) '
'under stochastic bipartition adjustment for this tree:')
self.TC_UNIBI_LABEL = Literal(
'Tree certainty under uniform bipartition '
'adjustment for this tree:')
self.RTC_UNIBI_LABEL = Literal('Relative tree certainty under uniform '
'bipartition adjustment for this tree:')
self.TCA_UNIBI_LABEL = Literal(
'Tree certainty including all conflicting bipartitions (TCA) under '
'uniform bipartition adjustment for this tree:')
self.RTCA_UNIBI_LABEL = Literal(
'Relative tree certainty including all conflicting bipartitions (TCA) '
'under uniform bipartition adjustment for this tree:')
self.tc_stochbi = Suppress(SkipTo(self.TC_STOCHBI_LABEL)) + Suppress(
self.TC_STOCHBI_LABEL) + FLOAT
self.rtc_stochbi = Suppress(SkipTo(self.RTC_STOCHBI_LABEL)) + Suppress(
self.RTC_STOCHBI_LABEL) + FLOAT
self.tca_stochbi = Suppress(SkipTo(self.TCA_STOCHBI_LABEL)) + Suppress(
self.TCA_STOCHBI_LABEL) + FLOAT
self.rtca_stochbi = Suppress(SkipTo(
self.RTCA_STOCHBI_LABEL)) + Suppress(
self.RTCA_STOCHBI_LABEL) + FLOAT
self.tc_unibi = Suppress(SkipTo(self.TC_UNIBI_LABEL)) + Suppress(
self.TC_UNIBI_LABEL) + FLOAT
self.rtc_unibi = Suppress(SkipTo(self.RTC_UNIBI_LABEL)) + Suppress(
self.RTC_UNIBI_LABEL) + FLOAT
self.tca_unibi = Suppress(SkipTo(self.TCA_UNIBI_LABEL)) + Suppress(
self.TCA_UNIBI_LABEL) + FLOAT
self.rtca_unibi = Suppress(SkipTo(self.RTCA_UNIBI_LABEL)) + Suppress(
self.RTCA_UNIBI_LABEL) + FLOAT
# Use these for flag 'a' option
self.boot_likelihood = boot_likelihood
self.freqs = freqs
self.rates = rates
self.alpha = alpha
self.name = description
self.treelen = treelen
self._dash_f_e_parser = (
Group(OneOrMore(self.model)) + likelihood + Group(
OneOrMore(
Group(description + alpha + Suppress(TREELENGTH_LEADIN) +
Suppress(FLOAT) + Group(OneOrMore(rates)) +
Group(OneOrMore(freqs))))))
def parse(self, filename, f_flag=None):
with open(filename) as fl:
s = fl.read()
if f_flag is None:
try:
alphas = self.alpha.parseString(s).asList()
except ParseException as err:
logger.error(err)
alphas = [None]
try:
freqs = self.frq.parseString(s).asList()
except ParseException as err:
logger.error(err)
freqs = [None]
try:
names = self.names.parseString(s).asList()
except ParseException as err:
logger.error(err)
names = [None]
try:
rates = self.rates.parseString(s).asList()
except ParseException:
rates = None
try:
lnl = self.lnl.parseString(s).asList()
except ParseException as err:
logger.error(err)
lnl = [0]
return alphas, freqs, names, rates, lnl
elif f_flag is 'i':
try:
tc_stochbi = self.tc_stochbi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
tc_stochbi = [None]
try:
rtc_stochbi = self.rtc_stochbi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
rtc_stochbi = [None]
try:
tca_stochbi = self.tca_stochbi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
tca_stochbi = [None]
try:
rtca_stochbi = self.rtca_stochbi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
rtca_stochbi = [None]
try:
tc_unibi = self.tc_unibi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
tc_unibi = [None]
try:
rtc_unibi = self.rtc_unibi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
rtc_unibi = [None]
try:
tca_unibi = self.tca_unibi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
tca_unibi = [None]
try:
rtca_unibi = self.rtca_unibi.parseString(s).asList()[0]
except ParseException as err:
logger.error(err)
rtca_unibi = [None]
return tc_unibi, rtc_unibi, tca_unibi, rtca_unibi, tc_stochbi, \
rtc_stochbi, tca_stochbi, rtca_stochbi
elif f_flag is 'a':
try:
model = self.model.parseString(s).asList()
except ParseException:
model = None
try:
alphas = self.alpha.parseString(s).asList()
except ParseException as err:
logger.error(err)
alphas = [None]
try:
freqs = self.freqs.parseString(s).asList()
except ParseException as err:
logger.error(err)
freqs = [None]
try:
name = self.name.parseString(s).asList()
except ParseException as err:
logger.error(err)
name = [None]
try:
rates = self.rates.parseString(s).asList()
except ParseException:
rates = None
try:
likelihood = self.boot_likelihood.parseString(s).asList()
except ParseException:
likelihood = None
try:
treelength = self.treelen.parseString(s).asList()
except ParseException:
treelength = None
return model, treelength, alphas, freqs, name, rates, likelihood
def _dash_f_e_to_dict(self, info_filename, tree_filename):
"""
Raxml provides an option to fit model params to a tree,
selected with -f e.
The output is different and needs a different parser.
"""
with open(info_filename) as fl:
models, likelihood, partition_params = self._dash_f_e_parser.parseFile(
fl).asList()
with open(tree_filename) as fl:
nwk_tree = fl.read().rstrip()
tree = dpy.Tree.get(data=nwk_tree, schema='newick')
d = {'likelihood': likelihood, 'tree': tree, 'partitions': {}}
for model, params in zip(models, partition_params):
subdict = {}
index, name, _, alpha, rates, freqs = params
subdict['alpha'] = alpha
subdict['name'] = name
subdict['rates'] = rates
subdict['frequencies'] = freqs
subdict['model'] = model
d['partitions'][index] = subdict
return d
def _dash_f_i_to_dict(self, info_filename, tree_filename):
"""
Raxml provides an option to fit model params to a tree,
selected with -f i.
The output is different and needs a different parser.
"""
tc_unibi, rtc_unibi, tca_unibi, rtca_unibi, tc_stochbi, rtc_stochbi, tca_stochbi, rtca_stochbi = self.parse(
info_filename, 'i')
with open(tree_filename) as fl:
nwk_tree = fl.read().rstrip()
tree = dpy.Tree.get(data=nwk_tree, schema='newick')
d = {
'tree': tree,
'tc_unibi': tc_unibi,
'rtc_unibi': rtc_unibi,
'tca_unibi': tca_unibi,
'rtca_unibi': rtca_unibi,
'tc_stochbi': tc_stochbi,
'rtc_stochbi': rtc_stochbi,
'tca_stochbi': tca_stochbi,
'rtca_stochbi': rtca_stochbi
}
return d
def _dash_f_a_to_dict(self, info_filename, tree_filename):
"""
Raxml provides an option to fit model params to a tree,
selected with -f a.
The output is different and needs a different parser.
"""
model, treelength, alphas, freqs, name, rates, likelihood = self.parse(
info_filename, f_flag='a')
bs_tree_filename = tree_filename.replace('bestTree',
'bipartitionsBranchLabels')
try:
with open(bs_tree_filename) as fl:
nwk_tree = fl.read().rstrip()
tree = dpy.Tree.get(data=nwk_tree, schema='newick')
except IOError as err:
logger.error('No tree file - raxml analysis failed')
return
result = {
'likelihood': likelihood,
'model': model,
'treelength': treelength,
'part_name': name[1],
'alpha': alphas,
'rates': rates,
'frequences': freqs,
'bs_tree': nwk_tree,
'tree': tree
}
return result
def to_dict(self, info_filename, tree_filename, dash_f=None):
"""
Parse raxml output and return a dict
Option dash_f_e=True will parse the output of a raxml -f e run,
which has different output
"""
if dash_f is 'e':
return self._dash_f_e_to_dict(info_filename, tree_filename)
elif dash_f is 'i':
return self._dash_f_i_to_dict(info_filename, tree_filename)
elif dash_f is 'a':
return self._dash_f_a_to_dict(info_filename, tree_filename)
else:
return self._to_dict(info_filename, tree_filename)
def _to_dict(self, info_filename, tree_filename):
alpha, freqs, names, rates, lnl = self.parse(info_filename)
try:
with open(tree_filename) as fl:
nwk_tree = fl.read().rstrip()
tree = dpy.Tree.get(data=nwk_tree, schema='newick')
except IOError as err:
logger.error('No tree file - raxml analysis failed')
return
n_parts = len(alpha)
assert len(freqs) == n_parts
assert len(names) == n_parts
if rates is not None:
assert len(rates) == n_parts
result = {'likelihood': lnl[0], 'partitions': {}, 'tree': tree}
for i in range(n_parts):
subdict = {
'alpha': alpha[i],
'frequencies': freqs[i],
'name': names[i]
}
if rates is not None:
subdict['rates'] = rates[i]
result['partitions'][i] = subdict
return result
labels = [ 'Northwest', 'North Central', 'Northeast', 'Southwest', 'South Central', 'Southeast' ]
# list of each crop in the report
crop_labels = [ '#2 Yellow Corn', '#1 Yellow Soybeans' ]
ending_index = 0 # initializes to 0 so that it can be later used to divide site_contents into sections
# Loops through each location in labels and finds the data for that location.
# Pyparsing is used to store each data element in a list so that it can be
# used to create a table.
z = 0
while z < len(labels):
starting_index = site_contents.find(labels[z], ending_index)
line = Suppress(Literal(labels[z])) + (Word(nums+'.'+'*') + (Suppress(Literal('\x96')) | Suppress(Literal('-'))) + Word(nums+'.'+'*') + Word(nums+'.'+'*'))*2
ending_index = site_contents.find('\r\n', starting_index)
ending_index1 = site_contents.find('\t', starting_index)
if ending_index1 != -1:
ending_index = min([ending_index, ending_index1])
line = line.parseString(site_contents[starting_index:ending_index])
# Loops through each crop in crop_labels and creates a table using the location
# in labels and the crop in crop_labels.
y = 0
while y < len(crop_labels):
headings = [ 'Date', 'Minimum Bid', 'Maximum Bid', 'Average Bid' ]
data = { 'Date': [string_date], 'Minimum Bid': [line[0].replace('*','')], 'Maximum Bid': [line[1].replace('*','')], 'Average Bid': [line[2].replace('*','')] }
del line[0:3]
data_df = pd.DataFrame(data, columns = headings)
data_df.index = data_df['Date']
data_df = data_df.drop('Date', 1)
replace = re.compile('[ /]') # list of characters to be replaced in the pork cut description
remove = re.compile('[,%#-&()!$+<>?/\'"{}.*@ ]') # list of characters to be removed from the pork cut description
name1 = replace.sub('_', crop_labels[y].upper()) # replace certain characters with '_'
name2 = remove.sub('', name1).upper() # remove certain characters and convert to upper case
name2 = name2.translate(None, '-') # ensure '-' character is removed
while site_contents.find('\r\n', starting_day, ending_day) != -1:
end_line = site_contents.find('\r\n', starting_day)
unparsed.append(site_contents[starting_day:end_line])
starting_day = end_line+2
cattle_slaughter = (Word(alphas) + Word(printables)) | (Word(alphas) + Literal("-")) # grammar for daily cattle slaughter
# Loops through each unparsed line and parses it, appending to number_cattle_slaughter
z = 0
while z < len(unparsed):
parsed.append(cattle_slaughter.parseString(unparsed[z]))
day.append(parsed[z][0])
number_cattle_slaughter.append(parsed[z][1])
z = z + 1
number_cattle_slaughter = [float(g.replace(',','').replace('-','0')) for g in number_cattle_slaughter] # remove commas and convert each value to a float
starting = site_contents.find('Beef', site_contents.find('Meat Production, Live Weight and Dressed Weight')) # store index of beginning of second data section
weight = Suppress(Literal('Beef')) + Word(printables) # grammar for finding weight of beef production
weight = float((weight.parseString(site_contents[starting:]))[0])
starting = site_contents.find('Cattle', starting)
word = Suppress(Word(alphas)) + Word(nums)
weights = Suppress(Word(alphas)) + Word(printables) + Word(nums) + word*4
average_live_weight = float((weights.parseString(site_contents[starting:]))[0].replace(',',''))
average_dressed_weight = float((weights.parseString(site_contents[starting:]))[1])
average_weights = weights.parseString(site_contents[starting:])[2:]
starting = site_contents.find('Cattle', site_contents.find('Federally Inspected Slaughter Head & Percentage by Class'))
suppress = Suppress(Word(printables))
word = Word(alphas) + ZeroOrMore(Word(alphas))
number = Word(printables)
cattle_head = suppress + Suppress(ZeroOrMore(Literal(':'))) + (word + Suppress(ZeroOrMore(Literal(':'))) + number) + ( Suppress(ZeroOrMore(Literal(':'))) + word + Suppress(ZeroOrMore(Literal(':'))) + number + Suppress(ZeroOrMore(Literal(':'))) + suppress ) * 5
y = cattle_head.parseString(site_contents[starting:])
names = [y[0], y[2], y[4], y[6] + ' ' + y[7], y[9] + ' ' + y[10], y[12]]
total_head = [y[1],y[3],y[5],y[8], y[11], y[13]]
total_head = [float(i.replace(',','.')) for i in total_head]
def _interfaceParse___iface_attributes(config, check_disabled):
iface_list = util.get_attributes(config)[0]
# if iface isn`t enable and unused
if iface_list:
iface_dict = {
'shutdown': 'no',
'vlans': [],
'cdp': 'yes',
'dhcp_snoop': {
'mode': 'untrust'
},
'arp_insp': {
'mode': 'untrust'
},
'storm control': {},
'port-security': {},
'ipv6': {}
}
vlan_num = Word(nums +
'-') + ZeroOrMore(Suppress(',') + Word(nums + '-'))
parse_description = Suppress('description ') + restOfLine
parse_type = Suppress('switchport mode ') + restOfLine
parse_port_sec = Suppress('switchport port-security ') + restOfLine
parse_stp_port = Suppress('spanning-tree ') + restOfLine
parse_dhcp_snoop = Suppress('ip dhcp snooping ') + restOfLine
parse_arp_insp = Suppress('ip arp inspection ') + restOfLine
parse_source_guard = Suppress('ip verify source ') + restOfLine
parse_arp_proxy_iface = Optional(
Word(alphas)) + Suppress('ip proxy-arp')
parse_vlans = Suppress('switchport ') + Suppress(
MatchFirst('access vlan ' +
('trunk allowed vlan ' + Optional('add ')))) + vlan_num
class Storm:
def __init__(self):
self.dct = {'type': []}
def new_line(self, line):
parse_storm = Suppress('storm-control ') + restOfLine
try:
self.storm_line = parse_storm.parseString(
line).asList()[-1]
self.level_info()
self.action_info()
self.type_info()
except ParseException:
pass
@catch_exception
def parse_level(self):
parse_level = Word(alphas) + Suppress('level ') + restOfLine
value = parse_level.parseString(self.storm_line).asList()
if 'level' in self.dct:
self.dct['level'].append(value)
else:
self.dct['level'] = [value]
@catch_exception
def parse_action(self):
action = Suppress('action ') + Word(alphas)
self.action = utill(action, self.storm_line)
@catch_exception
def parse_type(self):
type = Word(alphas) + Suppress(
Optional("include")) + Word(alphas)
self.type = utill(type, self.storm_line)
@catch_exception1
def action_info(self):
self.parse_action()
self.dct['action'] = self.action
@catch_exception1
def type_info(self):
self.parse_type()
for each in self.type:
if each not in self.dct['type'] and each in [
'broadcast', 'multicast', 'unicast'
]:
self.dct['type'].append(each)
@catch_exception1
def level_info(self):
self.parse_level()
cl_storm = Storm()
# Reserved options list is using due to 'shutdown' option is usually located at the end of the list, so it breaks cycle if interface is shutdown and function speed increases
for option in iface_list[::-1]:
cl_storm.new_line(option)
iface_dict['storm control'] = cl_storm.dct
if option == 'shutdown':
if check_disabled:
iface_dict['shutdown'] = 'yes'
pass
else:
iface_dict = {'shutdown': 'yes'}
break
if option == 'switchport nonegotiate':
iface_dict['dtp'] = 'no'
continue
if option == 'no cdp enable':
iface_dict['cdp'] = 'no'
continue
if option == 'no mop enabled':
iface_dict['mop'] = 'no'
continue
elif option == 'mop enabled':
iface_dict['mop'] = 'yes'
continue
try:
vlan_add = parse_vlans.parseString(option).asList()
for unit in vlan_add:
if '-' in unit:
range_units = unit.split('-')
range_list = [
i for i in range(int(range_units[0]),
int(range_units[1]) + 1)
]
iface_dict['vlans'].extend(range_list)
else:
iface_dict['vlans'].append(int(unit))
continue
except ParseException:
pass
try:
iface_dict['description'] = parse_description.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
iface_dict['type'] = parse_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
port_sec = parse_port_sec.parseString(option).asList()[-1]
iface_dict[
'port-security'] = parsing_checks.port_security.__ifaceAttributes___port_sec_parse(
port_sec, iface_dict['port-security'])
continue
except ParseException:
pass
try:
dhcp_snoop = parse_dhcp_snoop.parseString(option).asList()[-1]
iface_dict[
'dhcp_snoop'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse(
dhcp_snoop, iface_dict['dhcp_snoop'])
continue
except ParseException:
pass
try:
arp_insp = parse_arp_insp.parseString(option).asList()[-1]
iface_dict[
'arp_insp'] = parsing_checks.ip_iface.__ifaceAttributes___ip_parse(
arp_insp, iface_dict['arp_insp'])
# making a str:str instead of str:list
# see issue #5
if 'mode' in iface_dict['arp_insp']:
iface_dict['arp_insp']['mode'] = iface_dict['arp_insp'][
'mode'][0]
continue
except ParseException:
pass
try:
stp_port = parse_stp_port.parseString(option).asList()[-1]
iface_dict['stp'] = stp_port
continue
except ParseException:
pass
try:
source_guard = parse_source_guard.parseString(
option).asList()[-1]
iface_dict['source_guard'] = source_guard
continue
except ParseException:
pass
try:
ipv6 = parse_ipv6.parseString(option).asList()[-1]
__ifaceAttributes___ipv6_parse(ipv6, iface_dict['ipv6'])
continue
except ParseException:
pass
try:
arp_proxy_iface = parse_arp_proxy_iface.parseString(
option).asList()[-1]
iface_dict['arp_proxy'] = arp_proxy_iface
continue
except ParseException:
pass
return iface_dict
else:
return {'unknown_iface': 1}
)
)
NIL
NIL
)
"MIXED"
(
"BOUNDARY"
"Boundary-00=_H2YnIG887ejM4hn"
)
NIL
NIL
NIL
)
)
"""
parse = Suppress(Word(nums)) + Suppress("(") + Group(OneOrMore( Suppress("(") + Group(OneOrMore(Word(printables))) + Suppress(")") )) + Suppress(")")
for folder in folderlist:
tmp = parse.parseString( folder )
if type in tmp[2][1]:
result.append(tmp[0])
return result
@staticmethod
def folderAnnotation(folder):
pass
def append_dataframe_to_csv(c, df, dshape=None, **kwargs):
if not os.path.exists(c.path) or not os.path.getsize(c.path):
# 11.12.18 harleen - replacing pop and get with 'OR'. this is because even if dict key exists, default value (a method here) is always calcualted. And it errors for FTP locations
# original
# has_header = kwargs.pop('header', c.has_header)
# changes begin
try:
has_header = kwargs.pop('header')
except:
try:
has_header = c.has_header
except:
has_header = False
# changes end
else:
has_header = False
# 11.12.18 harleen - <<see above comment>>
# original
# sep = kwargs.get('sep',
# kwargs.get('delimiter', c.dialect.get('delimiter', ',')))
# changes begin
try:
sep = kwargs.get('sep') or kwargs.get('delimiter') or c.dialect.get('delimiter', ',')
except:
sep = ','
# changes end
encoding = kwargs.get('encoding', c.encoding)
if ext(c.path) in compressed_open:
if sys.version_info[0] >= 3:
kwargs['mode'] = 'at'
kwargs['encoding'] = encoding
elif sys.version_info[0] == 2:
kwargs['mode'] = 'ab' if sys.platform == 'win32' else 'at'
# 11.12.18 harleen: hack to enable writing to ftp server
# original
# with c.open(mode=kwargs.get('mode', 'a')) as f:
# df.to_csv(f,
# header=has_header,
# index=False,
# sep=sep,
# encoding=encoding)
# changes begin
f = c.path
if f.split(':')[0].lower() == 'ftp':
import io
from io import StringIO
from ftplib import FTP
from pyparsing import Combine, Suppress, Word, printables, alphas, ZeroOrMore, alphanums, Group, delimitedList, nums
ftp_expression = Suppress('ftp://') + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(alphanums) + ZeroOrMore(Suppress('@')) + Word(alphanums) + ZeroOrMore(Suppress(':')) + Word(nums) + ZeroOrMore(Suppress('/')) + Word(printables)
username, password, ip, port, file_loc = ftp_expression.parseString(f)
ftp = FTP(ip, username, password)
# bio = io.BytesIO(str.encode(str(df)))
buffer = StringIO()
df.to_csv(buffer, index=False, header=has_header, sep=sep, encoding=encoding)
text = buffer.getvalue()
bio = io.BytesIO(str.encode(text))
# ftp.storbinary('STOR '+file_loc, bio)
ftp.storbinary('APPE ' + file_loc, bio)
else:
with c.open(mode=kwargs.get('mode', 'a')) as f:
df.to_csv(f,
header=has_header,
index=False,
sep=sep,
encoding=encoding)
# changes end
return c
class RaxmlParser(object):
def __init__(self):
self.ALPHA_LABEL = Regex(r"alpha\[\d+\]:")
self.LNL_LABEL = Literal("Final GAMMA-based Score of best tree")
self.FRQ_LABEL = Regex(r"Base frequencies: (?=\d+)") ^ Regex(r"ML estimate base freqs\[\d+\]:")
self.NAMES_LABEL = Regex(r"Partition: \d+ with name:\s+")
self.RATES_LABEL = Regex(r"rates\[\d+\].+?:")
self.MODEL_LABEL = Literal("Substitution Matrix:")
self.alpha = OneOrMore(Suppress(SkipTo(self.ALPHA_LABEL)) + Suppress(self.ALPHA_LABEL) + FLOAT)
self.lnl = Suppress(SkipTo(self.LNL_LABEL)) + Suppress(self.LNL_LABEL) + FLOAT
self.frq = OneOrMore(Group(Suppress(SkipTo(self.FRQ_LABEL)) + Suppress(self.FRQ_LABEL) + OneOrMore(FLOAT)))
self.names = OneOrMore(
Suppress(SkipTo(self.NAMES_LABEL)) + Suppress(self.NAMES_LABEL) + CharsNotIn("\n") + Suppress(LineEnd())
)
self.rates = OneOrMore(
Group(Suppress(SkipTo(self.RATES_LABEL)) + Suppress(self.RATES_LABEL) + OneOrMore(FLOAT))
)
self.model = Suppress(SkipTo(self.MODEL_LABEL)) + Suppress(self.MODEL_LABEL) + WORD
MODEL_LABEL = Literal("Substitution Matrix:")
SCORE_LABEL = Literal("Final GAMMA likelihood:")
DESC_LABEL = Literal("Model Parameters of Partition")
NAME_LEADIN = Literal(", Name:")
DATATYPE_LEADIN = Literal(", Type of Data:")
ALPHA_LEADIN = Literal("alpha:")
TREELENGTH_LEADIN = Literal("Tree-Length:")
RATES_LABEL = Regex(r"rate \w <-> \w:")
FREQS_LABEL = Regex(r"freq pi\(\w\):")
model = Suppress(SkipTo(MODEL_LABEL)) + Suppress(MODEL_LABEL) + WORD
likelihood = Suppress(SkipTo(SCORE_LABEL)) + Suppress(SCORE_LABEL) + FLOAT
description = (
Suppress(SkipTo(DESC_LABEL))
+ Suppress(DESC_LABEL)
+ INT
+ Suppress(NAME_LEADIN)
+ SPACEDWORD
+ Suppress(DATATYPE_LEADIN)
+ WORD
)
alpha = Suppress(ALPHA_LEADIN) + FLOAT
rates = Suppress(RATES_LABEL) + FLOAT
freqs = Suppress(FREQS_LABEL) + FLOAT
self._dash_f_e_parser = (
Group(OneOrMore(model))
+ likelihood
+ Group(
OneOrMore(
Group(
description
+ alpha
+ Suppress(TREELENGTH_LEADIN)
+ Suppress(FLOAT)
+ Group(OneOrMore(rates))
+ Group(OneOrMore(freqs))
)
)
)
)
def parse(self, filename):
with open(filename) as fl:
s = fl.read()
try:
alphas = self.alpha.parseString(s).asList()
except ParseException as err:
logger.error(err)
alphas = [None]
try:
freqs = self.frq.parseString(s).asList()
except ParseException as err:
logger.error(err)
freqs = [None]
try:
names = self.names.parseString(s).asList()
except ParseException as err:
logger.error(err)
names = [None]
try:
rates = self.rates.parseString(s).asList()
except ParseException:
rates = None
try:
lnl = self.lnl.parseString(s).asList()
except ParseException as err:
logger.error(err)
lnl = [0]
return alphas, freqs, names, rates, lnl
def _dash_f_e_to_dict(self, info_filename, tree_filename):
"""
Raxml provides an option to fit model params to a tree,
selected with -f e.
The output is different and needs a different parser.
"""
with open(info_filename) as fl:
models, likelihood, partition_params = self._dash_f_e_parser.parseFile(fl).asList()
with open(tree_filename) as fl:
tree = fl.read()
d = {"likelihood": likelihood, "ml_tree": tree, "partitions": {}}
for model, params in zip(models, partition_params):
subdict = {}
index, name, _, alpha, rates, freqs = params
subdict["alpha"] = alpha
subdict["name"] = name
subdict["rates"] = rates
subdict["frequencies"] = freqs
subdict["model"] = model
d["partitions"][index] = subdict
return d
def to_dict(self, info_filename, tree_filename, dash_f_e=False):
"""
Parse raxml output and return a dict
Option dash_f_e=True will parse the output of a raxml -f e run,
which has different output
"""
if dash_f_e:
return self._dash_f_e_to_dict(info_filename, tree_filename)
else:
return self._to_dict(info_filename, tree_filename)
def _to_dict(self, info_filename, tree_filename):
alpha, freqs, names, rates, lnl = self.parse(info_filename)
try:
with open(tree_filename) as fl:
tree = fl.read().rstrip()
except IOError as err:
logger.error("No tree file - raxml analysis failed")
return
n_parts = len(alpha)
assert len(freqs) == n_parts
assert len(names) == n_parts
if rates is not None:
assert len(rates) == n_parts
result = {"likelihood": lnl[0], "partitions": {}, "ml_tree": tree}
for i in range(n_parts):
subdict = {}
subdict["alpha"] = alpha[i]
subdict["frequencies"] = freqs[i]
subdict["name"] = names[i]
if rates is not None:
subdict["rates"] = rates[i]
result["partitions"][i] = subdict
return result
class RaxmlParser(object):
def __init__(self):
self.ALPHA_LABEL = Regex(r'alpha\[\d+\]:')
self.LNL_LABEL = Literal('Final GAMMA-based Score of best tree')
self.FRQ_LABEL = Regex(r'Base frequencies: (?=\d+)') ^ Regex(r'ML estimate base freqs\[\d+\]:')
self.NAMES_LABEL = Regex(r'Partition: \d+ with name:\s+')
self.RATES_LABEL = Regex(r'rates\[\d+\].+?:')
self.MODEL_LABEL = Literal('Substitution Matrix:')
self.alpha = OneOrMore(Suppress(SkipTo(self.ALPHA_LABEL)) + Suppress(self.ALPHA_LABEL) + FLOAT)
self.lnl = Suppress(SkipTo(self.LNL_LABEL)) + Suppress(self.LNL_LABEL) + FLOAT
self.frq = OneOrMore(Group(Suppress(SkipTo(self.FRQ_LABEL)) + Suppress(self.FRQ_LABEL) + OneOrMore(FLOAT)))
self.names = OneOrMore(Suppress(SkipTo(self.NAMES_LABEL)) + Suppress(self.NAMES_LABEL) + CharsNotIn('\n') + Suppress(LineEnd()))
self.rates = OneOrMore(Group(Suppress(SkipTo(self.RATES_LABEL)) + Suppress(self.RATES_LABEL) + OneOrMore(FLOAT)))
self.model = Suppress(SkipTo(self.MODEL_LABEL)) + Suppress(self.MODEL_LABEL) + WORD
MODEL_LABEL = Literal('Substitution Matrix:')
SCORE_LABEL = Literal('Final GAMMA likelihood:')
DESC_LABEL = Literal('Model Parameters of Partition')
NAME_LEADIN = Literal(', Name:')
DATATYPE_LEADIN = Literal(', Type of Data:')
ALPHA_LEADIN = Literal('alpha:')
TREELENGTH_LEADIN = Literal('Tree-Length:')
RATES_LABEL = Regex(r'rate \w <-> \w:')
FREQS_LABEL = Regex(r'freq pi\(\w\):')
BEST_LEADIN = Literal('Starting final GAMMA-based thorough Optimization on tree ')
PARTITION_LEADIN = Literal('Partition:')
INFERENCE_LEADIN = Literal('Inference[')
model = Suppress(SkipTo(MODEL_LABEL)) + Suppress(MODEL_LABEL) + WORD
likelihood = Suppress(SkipTo(SCORE_LABEL)) + Suppress(SCORE_LABEL) + FLOAT
description = (Suppress(SkipTo(DESC_LABEL)) +
Suppress(DESC_LABEL) + INT +
Suppress(NAME_LEADIN) +
SPACEDWORD +
Suppress(DATATYPE_LEADIN) +
WORD)
alpha = Suppress(ALPHA_LEADIN) + FLOAT
rates = Suppress(RATES_LABEL) + FLOAT
freqs = Suppress(FREQS_LABEL) + FLOAT
self.partition = OneOrMore(Suppress(SkipTo(PARTITION_LEADIN)) + Suppress(PARTITION_LEADIN) + INT)
self.inference = OneOrMore(Suppress(SkipTo(INFERENCE_LEADIN)) + Suppress(INFERENCE_LEADIN) + INT)
self.best = Suppress(SkipTo(BEST_LEADIN)) + Suppress(BEST_LEADIN) + INT
self._dash_f_e_parser = (Group(OneOrMore(model)) +
likelihood +
Group(OneOrMore(Group(description +
alpha +
Suppress(TREELENGTH_LEADIN) +
Suppress(FLOAT) +
Group(OneOrMore(rates)) +
Group(OneOrMore(freqs))
))))
def parse(self, filename):
with open(filename) as fl:
s = fl.read()
try:
best_index = self.best.parseString(s)[0]
except ParseException as err:
logger.error(err)
best_index = 0
try:
n_partitions = max(self.partition.parseString(s).asList()) + 1
except ParseException as err:
logger.error(err)
n_partitions = 1
try:
n_inferences = max(self.inference.parseString(s).asList()) + 1
except ParseException as err:
logger.error(err)
n_inferences = 1
try:
alphas = self.alpha.parseString(s).asList()
except ParseException as err:
logger.error(err)
alphas = [None]
try:
freqs = self.frq.parseString(s).asList()
except ParseException as err:
logger.error(err)
freqs = [None]
try:
names = self.names.parseString(s).asList()
except ParseException as err:
logger.error(err)
names = [None]
try:
rates = self.rates.parseString(s).asList()
except ParseException:
rates = None
try:
lnl = self.lnl.parseString(s).asList()
except ParseException as err:
logger.error(err)
lnl = [0]
alphas = np.array(alphas).reshape(n_inferences, n_partitions)[best_index].tolist()
if n_inferences > 1 and len(freqs) == n_inferences * n_partitions:
logger.debug('Reshaping freqs for multiple inference result')
freqs = np.array(freqs).reshape(n_inferences, n_partitions, len(freqs[0]))[best_index].tolist()
if rates is not None and n_inferences > 1 and len(rates) == n_inferences * n_partitions:
logger.debug('Reshaping rates for multiple inference result')
rates = np.array(rates).reshape(n_inferences, n_partitions, len(rates[0]))[best_index].tolist()
return alphas, freqs, names, rates, lnl
def _dash_f_e_to_dict(self, info_filename, tree_filename):
"""
Raxml provides an option to fit model params to a tree,
selected with -f e.
The output is different and needs a different parser.
"""
with open(info_filename) as fl:
models, likelihood, partition_params = self._dash_f_e_parser.parseFile(fl).asList()
with open(tree_filename) as fl:
tree = fl.read()
d = {'likelihood': likelihood, 'ml_tree': tree, 'partitions': {}}
for model, params in zip(models, partition_params):
subdict = {}
index, name, _, alpha, rates, freqs = params
subdict['alpha'] = alpha
subdict['name'] = name
subdict['rates'] = rates
subdict['frequencies'] = freqs
subdict['model'] = model
d['partitions'][index] = subdict
return d
def to_dict(self, info_filename, tree_filename, dash_f_e=False):
"""
Parse raxml output and return a dict
Option dash_f_e=True will parse the output of a raxml -f e run,
which has different output
"""
logger.debug('info_filename: {} {}'
.format(info_filename, '(FOUND)' if os.path.exists(info_filename) else '(NOT FOUND)'))
logger.debug('tree_filename: {} {}'
.format(tree_filename, '(FOUND)' if os.path.exists(tree_filename) else '(NOT FOUND)'))
if dash_f_e:
return self._dash_f_e_to_dict(info_filename, tree_filename)
else:
return self._to_dict(info_filename, tree_filename)
def _to_dict(self, info_filename, tree_filename):
alpha, freqs, names, rates, lnl = self.parse(info_filename)
try:
with open(tree_filename) as fl:
tree = fl.read().rstrip()
except IOError as err:
logger.error('No tree file - raxml analysis failed')
return
n_parts = len(alpha)
assert len(freqs) == n_parts
assert len(names) == n_parts
if rates is not None:
assert len(rates) == n_parts
result = {'likelihood': lnl[0],
'partitions': {},
'ml_tree': tree}
for i in range(n_parts):
subdict = {}
subdict['alpha'] = alpha[i]
subdict['frequencies'] = freqs[i]
subdict['name'] = names[i]
if rates is not None:
subdict['rates'] = rates[i]
result['partitions'][i] = subdict
return result
def _globalParse___line_attributes(config):
line_list, next_line = util.get_attributes(config)
line_dict = {'log_syng': 'no', 'no_exec': 'no', 'access-class': {}}
parse_login_type = Suppress('login ' +
Optional('authentication ')) + restOfLine
parse_exec_timeout = Suppress('exec-timeout ') + restOfLine
parse_pass_type = Suppress('password ') + restOfLine
parse_privilege = Suppress('privilege level ') + restOfLine
parse_transp_in = Suppress('transport input ') + restOfLine
parse_transp_out = Suppress('transport output ') + restOfLine
parse_rotary = Suppress('rotary ') + restOfLine
parse_access_class = Suppress('access-class') + Word(
alphas + '-') + MatchFirst(['in', 'out']) + Suppress(
Optional(restOfLine))
for option in line_list:
if option == 'logging synchronous':
line_dict['log_syng'] = 'yes'
continue
if option == 'no exec':
line_dict['no_exec'] = 'yes'
continue
try:
item = parse_exec_timeout.parseString(option).asList()[-1]
item = item.split()
if len(item) == 2:
line_dict['exec_timeout'] = int(item[0]) + int(item[1]) / 60
else:
line_dict['exec_timeout'] = int(item[0])
continue
except ParseException:
pass
try:
line_dict['login_type'] = parse_login_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['pass_type'] = parse_pass_type.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['privilege'] = parse_privilege.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['transp_in'] = parse_transp_in.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['transp_out'] = parse_transp_out.parseString(
option).asList()[-1]
continue
except ParseException:
pass
try:
line_dict['rotary'] = parse_rotary.parseString(option).asList()[-1]
continue
except ParseException:
pass
try:
item = parse_access_class.parseString(option).asList()
line_dict['access-class']['name'] = item[0]
line_dict['access-class']['type'] = item[-1]
continue
except ParseException:
pass
return line_dict, next_line
class Lindenmayer(object):
def __init__(self, stream):
# Set the default image dimensions ...
self.width = N
self.height = N
# ... and the number of iterations.
self.iterations = 5
# Set the default rotation angle in degrees.
self.alpha = 2*np.pi/8
self.angle = 0
# Initialize the branch stack, ...
self.stack = []
# ... the constants, the rules, the variables and the axiom ...
self.const = {'+':'+', '-':'-', '[':'[', ']':']'}
self.rules = {}
self.vars = []
self.axiom = None
# ... and drawing settings.
self.bgcolor = (1.0, 1.0, 1.0)
self.lineLength = 20
self.lineWidth = 5
self.lineColor = (0, 0, 0)
# Calculate the starting position.
self.offset = (0, -self.height*0.5)
print 'Offset :', self.offset
# Finally store the stream ...
self.stream = stream
self.arr = bak.calc()#np.zeros((N,N))#
print "Baking Bread..."
# state
self.x = int(self.width/2)#int(self.width/2)
self.y = int(self.height/2)
self.r = 16
self.xparent = self.x
self.yparent = self.y
# ... and initialize the parser.
self.initialize()
# calculates new radius based on a poisson distribution (?)
def poisson(self):
global bubbles, delta
#x1 = min(max(self.x-delta,0),N)
#y1 = min(max(self.y-delta,0),N)
x1 = self.x
y1 = self.y
suma = 0.0
x0 = max(x1-delta,0)
y0 = max(y1-delta,0)
x2 = min(x1+delta,N)
y2 = min(y1+delta,N)
#print x0,y0,x2,y2
#print abs(x0-x2),abs(y0-y2)
cant = 0
for i in range(x0,x2):
for j in range(y0,y2):
#d = np.sqrt((x1-i)*(x1-i) + (y1-j)*(y1-j)).astype(np.float32) # distance
#if(d==0): suma+=bubbles[i,j]
#else:
suma += bubbles[i,j]#*(np.sqrt(2)*delta+1-d)/(np.sqrt(2)*delta)
cant += bubbles[i,j]>0
#suma += np.pi*bubbles[i,j]*bubbles[i,j]
#if(bubbles[i,j]): print "RADIO:" , bubbles[i,j]
factor = delta # (?)
#print "Suma:", suma, "Delta: ", delta
#print factor*(1/suma)
#1/ sum_D (cant*radios*D^2)
#D = np.sqrt(np.power((self.x-self.height/2),2)+np.power((self.y-self.width/2),2))
G = 1#+2.5*self.arr[min(self.width,max(self.x-1,0)),min(self.width,max(self.y-1,0))]#D/300 #gelatinization
#print "G: ", G
x1 = min(max(self.x-delta,0),N)
y1 = min(max(self.y-delta,0),N)
baking = self.arr[x1,y1]*4
#print baking, ": baking"
if(suma > 0):
m = 1/suma
#if(delta*m < 1): return np.random.randint(0,2)
if(cant>10): return 0
return np.random.randint(0,delta*m+2)/baking
else: # free
return np.random.randint(0,delta/2+delta/2)/baking
def rotate(self):
#self.x += self.r
#self.y += self.r
#d = 2*np.pi*random.random()
ang = self.angle+random.random()/6
self.x = self.xparent + np.int32(fdist(self.r)*np.cos(ang))+randint(-int(self.r),int(self.r))
self.y = self.yparent + np.int32(fdist(self.r)*np.sin(ang))+randint(-int(self.r),int(self.r))
#self.x = self.xparent + np.int32(fdist(self.r)*np.cos(ang))+randint(-int(self.r),int(self.r))
#self.y = self.yparent + np.int32(fdist(self.r)*np.sin(ang))+randint(-int(self.r),int(self.r))
#pass
def moveX(self):
self.x += self.r
def mmoveX(self):
self.x -= self.r
def moveY(self):
self.y += self.r
def mmoveY(self):
self.y -= self.r
def initialize(self):
ParserElement.setDefaultWhitespaceChars(' \t\r')
integer = Regex(r"[+-]?\d+") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
number = Regex(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?") \
.setParseAction(lambda s,l,t: [ float(t[0]) ])
color = Regex(r"#([0-9a-fA-F]{6})")
angle = "'" + Regex(r"(360|3[0-5][0-9]|[12][0-9]{2}|[0-9]{1,2})") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
alpha = "'" + Regex(r"(360|3[0-5][0-9]|[12][0-9]{2}|[0-9]{1,2})") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
variable = Word(alphas, exact=1).setParseAction(self.addVar)
colon = Literal(":").suppress()
comma = Literal(",")
lBrace = Literal("(")
rBrace = Literal(")")
lBracket = Literal("[")
rBracket = Literal("]")
lAngle = Literal("<")
rAngle = Literal(">")
plus = Literal("+")
minus = Literal("-")
FTerm = Literal("F")
fTerm = Literal("f")
ZTerm = Literal("Z")
zTerm = Literal("z")
xTerm = Literal("x")
cTerm = Literal("c")
eol = OneOrMore(LineEnd()).suppress()
param = ( angle | color | "!" + number | "|" + number )
self.pList = lBrace + param + ZeroOrMore(comma + param) + rBrace
literal = ((lBracket + ( variable + Optional(self.pList)
| plus + Optional(self.pList) | minus + Optional(self.pList) ) + rBracket)
| (variable + Optional(self.pList) | plus + Optional(self.pList)
| minus + Optional(self.pList)))
terminal = (ZTerm | zTerm | FTerm | fTerm | xTerm | cTerm
| plus | minus | lBracket | rBracket)
lprod = (
(OneOrMore(terminal) + lAngle + variable + rAngle + OneOrMore(terminal))
| (OneOrMore(terminal) + lAngle + variable)
| (variable + rAngle + OneOrMore(terminal))
| variable )
rProd = OneOrMore(literal | terminal)
comment = Suppress((LineStart() + "#" + SkipTo(eol, include=True)))
rules = (
(lprod + Literal("=") + rProd + eol).setParseAction(self.addRule) \
| comment )
defaults = ( ( ("Dimensions" + colon + integer + comma + integer)
| ("Position" + colon + integer + comma + integer)
| ("Iterations" + colon + integer)
| ("Angle" + colon + angle)
| ("Linelength" + colon + number)
| ("Linewidth" + colon + number)
| ("Linecolor" + colon + color)
| ("Background" + colon + color)
| ("Axiom" + colon + rProd) ) + eol ).setParseAction(self.setAttribute)
header = ( defaults | comment )
self.grammar = Suppress(ZeroOrMore(LineEnd())) \
+ ZeroOrMore(header) \
+ OneOrMore(rules)
try:
L = self.grammar.parseString( self.stream )
except ParseException, err:
print err.line
print " "*(err.column-1) + "^"
print err
print 'Rules:', self.rules
class Lindenmayer(object):
def __init__(self, stream):
# Set the default image dimensions ...
self.width = 256
self.height = 256
self.maxZ = 256
# ... and the number of iterations.
self.iterations = 5
# Set the default rotation angle in degrees.
self.alpha = 2*np.pi/5
self.alpha2 = 2*np.pi/5
self.angle = 0
self.angle2 = 0
# Initialize th
# e branch stack, ...
self.stack = []
# ... the constants, the rules, the variables and the axiom ...
self.const = {'+':'+', '-':'-', '[':'[', ']':']'}
self.rules = {}
self.vars = []
self.axiom = None
# ... and drawing settings.
self.bgcolor = (1.0, 1.0, 1.0)
self.lineLength = 20
self.lineWidth = 5
self.lineColor = (0, 0, 0)
# Calculate the starting position.
self.offset = (0, -self.height*0.5)
#print 'Offset :', self.offset
# Finally store the stream ...
self.stream = stream
# state
self.x = int(self.width/2)#int(self.width/2)
self.y = int(self.height/2)
self.z = int(self.maxZ/2)
self.r = int(self.width/6)
#print self.r
self.xparent = self.x
self.yparent = self.y
self.zparent = self.z
# ... and initialize the parser.
self.initialize()
def drawShape(self,field,x,y,z,r,c):
if(c == 0): return
r = int(r)
for i in range(x-r-1,x+r+1):
for j in range(y-r-1,y+r+1):
for k in range(z-r-1,z+r+1):
if(i >= 0 and i < self.width and j >= 0 and j < self.height and k >= 0 and k<self.maxZ):
i2 = i-x
j2 = j-y
k2 = k-z
if(i2*i2+j2*j2+k2*k2 < r*r):
#print float(i), ",", float(j),",", float(k)
field[i][j][k] = np.uint8(0)
#print "ellipse!"
# draw.ellipse((x-r+randint(-r/2,r/2), y-r+randint(-r/2,r/2), x+r, y+r), fill=255)
return
r2 = r
x2 = x
y2 = y
z2 = z
#for h in range(maxx):
r2 = int(r2/(2))
dd = int(r*1.0)
for i in range(2):
x3 = x2+randint(-dd,dd)
y3 = y2+randint(-dd,dd)
z3 = z2+randint(-dd,dd)
self.drawShape(field,x3,y3,z3,r2,c-1)
def rotate(self):
ang = self.angle#+random.random()/10
self.x = self.xparent + np.int32(fdist(self.r)*np.cos(ang))+randint(-int(self.r),int(self.r))
self.y = self.yparent + np.int32(fdist(self.r)*np.sin(ang))+randint(-int(self.r),int(self.r))
self.z = self.zparent
def rotate2(self):
ang = self.angle2#+random.random()/10
self.x = self.xparent
self.y = self.yparent + np.int32(fdist(self.r)*np.cos(ang))+randint(-int(self.r),int(self.r))
self.z = self.zparent + np.int32(fdist(self.r)*np.sin(ang))+randint(-int(self.r),int(self.r))
def moveX(self):
self.x += self.r
def mmoveX(self):
self.x -= self.r
def moveY(self):
self.y += self.r
def mmoveY(self):
self.y -= self.r
def initialize(self):
ParserElement.setDefaultWhitespaceChars(' \t\r')
integer = Regex(r"[+-]?\d+") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
number = Regex(r"[+-]?(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?") \
.setParseAction(lambda s,l,t: [ float(t[0]) ])
color = Regex(r"#([0-9a-fA-F]{6})")
angle = "'" + Regex(r"(360|3[0-5][0-9]|[12][0-9]{2}|[0-9]{1,2})") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
alpha = "'" + Regex(r"(360|3[0-5][0-9]|[12][0-9]{2}|[0-9]{1,2})") \
.setParseAction(lambda s,l,t: [ int(t[0]) ])
variable = Word(alphas, exact=1).setParseAction(self.addVar)
colon = Literal(":").suppress()
comma = Literal(",")
lBrace = Literal("(")
rBrace = Literal(")")
lBracket = Literal("[")
rBracket = Literal("]")
lAngle = Literal("<")
rAngle = Literal(">")
plus = Literal("+")
minus = Literal("-")
FTerm = Literal("F")
fTerm = Literal("f")
ZTerm = Literal("Z")
zTerm = Literal("z")
xTerm = Literal("x")
cTerm = Literal("c")
eol = OneOrMore(LineEnd()).suppress()
param = ( angle | color | "!" + number | "|" + number )
self.pList = lBrace + param + ZeroOrMore(comma + param) + rBrace
literal = ((lBracket + ( variable + Optional(self.pList)
| plus + Optional(self.pList) | minus + Optional(self.pList) ) + rBracket)
| (variable + Optional(self.pList) | plus + Optional(self.pList)
| minus + Optional(self.pList)))
terminal = (ZTerm | zTerm | FTerm | fTerm | xTerm | cTerm
| plus | minus | lBracket | rBracket)
lprod = (
(OneOrMore(terminal) + lAngle + variable + rAngle + OneOrMore(terminal))
| (OneOrMore(terminal) + lAngle + variable)
| (variable + rAngle + OneOrMore(terminal))
| variable )
rProd = OneOrMore(literal | terminal)
comment = Suppress((LineStart() + "#" + SkipTo(eol, include=True)))
rules = (
(lprod + Literal("=") + rProd + eol).setParseAction(self.addRule) \
| comment )
defaults = ( ( ("Dimensions" + colon + integer + comma + integer)
| ("Position" + colon + integer + comma + integer)
| ("Iterations" + colon + integer)
| ("Angle" + colon + angle)
| ("Linelength" + colon + number)
| ("Linewidth" + colon + number)
| ("Linecolor" + colon + color)
| ("Background" + colon + color)
| ("Axiom" + colon + rProd) ) + eol ).setParseAction(self.setAttribute)
header = ( defaults | comment )
self.grammar = Suppress(ZeroOrMore(LineEnd())) \
+ ZeroOrMore(header) \
+ OneOrMore(rules)
try:
L = self.grammar.parseString( self.stream )
except ParseException, err:
print err.line
print " "*(err.column-1) + "^"
print err