def BEGIN(self, *args):
if args[0].lower() != 'waveform':
raise ParseError("Can't understand args for BEGIN {args}")
args_dic = self.single_letter_arg(*args[1:])
time_step = args_dic['R']
Npos = args_dic['N']
measure_time_step = None
if 'M' in args_dic.keys():
measure_time_step = args_dic['M']
if self._wavedata is not None:
data = self._wavedata
else:
line = self.file.readline()
data = {}
while line != 'END\n' and line != 'END' and line != '':
line = line.split(' ')
data[line[0]] = np.asarray(line[1:], float)
line = self.file.readline()
if line != 'END\n' and line != 'END':
raise ParseError("Can't find end + {line}")
self._wavedata = data
if 'E' in data:
indices = ['X', 'Y', 'Z', 'E']
else:
indices = ['X', 'Y', 'Z']
X = np.asarray([data[idx] for idx in indices])
if np.shape(X)[1] != Npos:
raise ParseError(
f"Mismatch in waveform! {np.shape(X)[1]} != {Npos}")
self.run_waveform(time_step, X, measure_time_step=measure_time_step)
self._wavedata = None
def parse_number(self, s):
"""Parses integers in bases 10 and 16 and floats."""
start = 1 if s[0] in ["-", "+"] else 0
all_digits = lambda x: all(map(lambda c: c.isdigit(), x))
ishex = lambda c: c.isdigit() or ord(c.lower()) in range(
ord("a"), ord("f"))
all_hex = lambda x: all(map(ishex, x))
if all_digits(s[start:]):
try:
return (Tokenizer.INTEGER, int(s))
except ValueError:
raise ParseError("%d:%d: Invalid integer '%s'" %
(self.lineno, self.column, s))
if s[start:].startswith("0x") and all_hex(s[start + 2:]):
try:
return (Tokenizer.INTEGER, int(s, base=16))
except ValueError:
raise ParseError("%d:%d: Invalid hexadecimal integer '%s'" %
(self.lineno, self.column, s))
if any(map(lambda c: c == ".", s)) or any(map(lambda c: c == "e", s)):
try:
return (Tokenizer.FLOAT, float(s))
except ValueError:
raise ParseError("%d:%d: Invalid float '%s'" %
(self.lineno, self.column, s))
raise ParseError("%d:%d: Invalid number '%s'" %
(self.lineno, self.column, s))
def parse_arguments(parsers,arg_strings):
""" Applies argument parsers to strings by pairing them one by one.
Returns a list of the result.
parsers - list of functions (e.g. `[name, distance]`)
arg_strings - list of strings.
Raises ParseError if lists of different length.
"""
if len(arg_strings) < len(parsers):
raise ParseError("Too few arguments.")
if len(arg_strings) > len(parsers):
raise ParseError("Too many arguments.")
return [p(s) for (p,s) in zip(parsers, arg_strings)]
def create(self):
assert (self.type in ['A', 'CNAME', 'MX', 'AAAA'])
assert (self.data)
if self._id:
raise AlreadyExists(self.fqdn)
# it works like in Zone.create()
url = Zerigo._url_api \
+ Host._url_template.substitute(zone_id=self.__zone._id)
Zerigo._logger.debug('retrieving ' + url)
template = self._conn.get(url)
tree = ElementTree()
tree.parse(template.body_file)
type = tree.find('host-type')
hostname = tree.find('hostname')
data = tree.find('data')
if type is None or hostname is None or data is None:
raise ParseError()
type.text = self.type
if 'nil' in hostname.attrib:
del hostname.attrib['nil']
hostname.text = self.hostname
if 'nil' in data.attrib:
del data.attrib['nil']
data.text = self.data
form = StringIO.StringIO()
tree.write(form)
form = form.getvalue()
url = Zerigo._url_api + Host._url_create.substitute(
zone_id=self.__zone._id)
# exact same block in Zone; we should find a way to factore this
try:
Zerigo._logger.debug('posting ' + url)
host = self._conn.post(url, body=form)
except restkit.RequestFailed as errors:
errors = StringIO.StringIO(errors)
tree = ElementTree()
tree.parse(errors)
errors = [err.text for err in tree.findall('error')]
raise CreateError(self.fqdn, ', '.join(errors))
# read the id
tree = ElementTree()
tree.parse(host.body_file)
id = tree.find('id')
if id is None or not id.text:
raise ParseError()
self._id = id.text
Zerigo._logger.debug('host ' + self.fqdn + ' created with id: ' +
self._id)
def distance(s):
""" Interprets string `s` as floating point distance.
Distance must be positive.
Raises `ParseError` on error.
"""
try:
d = float(s)
if d <= 0:
raise ParseError("Distance must be positive")
return d
except ValueError:
raise ParseError(f"'{s}' not a distance")
def match_word(self):
next_token = self.scanner.next_token()
if next_token.is_word():
return next_token.value()
else:
msg = 'expected word but found {}'.format(next_token.token_type)
raise ParseError(msg)
def readline(self):
if self._next_line is not None:
ret = self._next_line
self._next_line = None
return ret
line = self.file.readline()
if line == '':
return None, None
line = line.strip()
line = line.split(' ')
command = line[0]
args = line[1:]
if command.lower() in ['laser', 'focus', 'savemeasure',
'camera', 'focusint', 'begin']:
pass
elif command.lower() in ['piezoslope', 'piezoreset']:
args = tuple()
elif command.lower() in ['piezo', 'motor']:
args = self.read_move_args(args)
else:
raise ParseError(f"Unknown command {command}")
fun = getattr(self, command)
self._prev_line = fun, args
return fun, args
def _lite_inner(el, elref, elpos, bexpr):
# If the element does not preceed lambda position or
# is not already in the lambdas function arguments
if elref in argsig or elpos.lineno > srcpos.lineno:
return
# If the element does proceed the lambda expression
elif elpos.lineno < srcpos.lineno or elpos.colno < srcpos.colno:
# Get the index of the element from element expression list
index = bexpr.index(el)
# Make a copy of the element
# nc = copy.deepcopy(el)
# Replicate the expression to FuncArgReference
# To make this work we need to create a ParseSymbol
# for the lite-weights (MatchExpr result, etc)
nc = cntrl[0].replicate(elref)
nc.name = el.ident
nc.ident = el.ident
psymbol = ParseSymbol(
ExpressionType.SYMBOL_REF,
[nc],
elref.token,
el.ident)
# nc.exprs[0] = cntrl[0].replicate(elref)
# Replace base expression with funcarg reference
# bexpr[index] = nc
bexpr[index] = psymbol
# Copy to function arg newlist to indicate it is new
# newlist.append(nc)
newlist.append(psymbol)
# Add to closures list
clist.append(el)
else:
raise ParseError(
"Unhandled lambda free variable location {}".format(
elpos))
def degrees(s):
""" Interprets string `s` as floating point angle.
Raises `ParseError` on error.
"""
try:
return float(s)
except ValueError:
raise ParseError(f"'{s}' not a valid angle")
def match(self, expected_token_type):
next_token = self.scanner.next_token()
if next_token.token_type == expected_token_type:
return True
else:
msg = 'expected {} but found {}'.format(expected_token_type, \
next_token.token_type)
raise ParseError(msg)
def _parse_desc(node):
""" A quick'n'dirty description parser """
desc = ''
for n in node:
if n.tag == 'p':
desc += '<p>' + _join_lines(n.text) + '</p>'
elif n.tag == 'ol' or n.tag == 'ul':
desc += '<ul>'
for c in n:
if c.tag == 'li':
desc += '<li>' + _join_lines(c.text) + '</li>'
else:
raise ParseError('Expected <li> in <%s>, got <%s>', n.tag,
c.tag)
desc += '</ul>'
else:
raise ParseError('Expected <p>, <ul>, <ol> in <%s>, got <%s>',
node.tag, n.tag)
return desc
def colour(s):
""" Maps from string `s` to Boolean pen state.
Valid colours are keys in dictionary `drawing.colours`.
Raises `ParseError` on error.
"""
try:
return drawing.colours[s]
except KeyError:
cs = ", ".join(drawing.colours)
raise ParseError(f"'{s}' is not a valid colour. [Known colours: {cs}]")
def __read_id(self, zone):
assert (zone)
tree = ElementTree()
tree.parse(zone.body_file)
id = tree.find('id')
if id is None or not id.text:
self._id = None
raise ParseError()
self._id = id.text # used by Zerigo to do almost all operations.
def parse_assignments(self, results=[]):
next_token = self.scanner.peek()
if next_token.is_eof():
return results
elif next_token.is_word():
assignment = self.parse_assignment()
return self.parse_assignments(results + [assignment])
else:
msg = 'expected assignment or EOF, but found {}'
raise ParseError(msg.format(next_token))
def pen_state(s):
""" Maps from string `s` to Boolean pen state.
'up' : False, 'down' : True
Raises `ParseError` on error.
"""
if "up" == s:
return False
elif "down" == s:
return True
else:
raise ParseError(f"'{s}' not a valid pen state")
def parse(line):
""" Parse a command string.
Returns a pair, where the first component is the command function
and the second component the arguments.
Raises ParseError on error.
"""
try:
# Split line at space, first word in c, all others in list cc.
c, *cc = line.split()
# The functions are named the same as the commands, so getattr can
# be used to retrieve the function from the turtle module.
# This raises AttributeError if not found.
op = getattr(turtle,c)
parsers = command_parsers[op]
cargs = parse_arguments(parsers,cc)
return (op,cargs)
except (KeyError, AttributeError):
raise ParseError(f"Unknown command '{c}'")
except ParseError as pe:
raise ParseError(f"{pe}")
def get_news_feed():
feed = feedparser.parse(AppUrl.OPINION)
if feed is None:
raise InputError("RSS feed", feed, "None RSS feed")
if feed.bozo:
exc = feed.bozo_exception
raise ParseError('Bozo bit set. Ill-formed XML.')
if 'status' not in feed:
raise ParseError('No status in extracted feed.')
logging.debug('Feed extraction status : ' + str(feed['status']))
if feed['status'] != 200:
raise ConnectionError(AppUrl.OPINION, feed['status'])
news = []
for entry in feed['entries']:
title = unidecode(entry['title'])
author = unidecode(entry['author'])
date = entry['published']
link = entry['link'].replace(AppUrl.RSS_ARGS, '')
try:
typ = get_news_type(link)
except InputError:
logging.exception("InputError is raised.")
continue
logging.debug('Extracted item titled {} dated {} authored {}'.format(
title, date, author))
item = {}
item['title'] = title
item['datetime'] = entry['published_parsed']
item['link'] = link
item['print_time'] = date
item['author'] = author
item['type'] = typ
news.append(item)
logging.debug("Extracted {} news items.".format(len(news)))
return news
def focus(self, stage, start_offset, stop_offset, step):
start_offset, stop_offset, step = float(
start_offset), float(stop_offset), float(step)
if stage.lower() == 'piezo':
kwargs_list = [{
'start_offset': start_offset,
'stop_offset': stop_offset,
'step': step,
'stage': self.md.piezo,
'Nloops': 2,
'wait': True,
'checkid': self.lockid
}]
elif stage.lower() == 'motor':
kwargs_list = [{
'start_offset': start_offset,
'stop_offset': stop_offset,
'step': step,
'stage': self.md.motor,
'Nloops': 1,
'wait': True,
'checkid': self.lockid
}]
elif stage.lower() == 'both':
kwargs_list = [{
'start_offset': start_offset,
'stop_offset': stop_offset,
'step': step,
'stage': self.md.motor,
'Nloops': 1,
'wait': True,
'checkid': self.lockid
}, {
'start_offset': -2,
'stop_offset': 2,
'step': 1,
'stage': self.md.piezo,
'Nloops': 2,
'wait': True,
'checkid': self.lockid
}]
else:
self.md.unlock()
self.lockid = None
raise ParseError(f"Don't know {stage}")
for kwargs in kwargs_list:
self.focus_delegate.focus(**kwargs)
data, z_best, error = self.focus_delegate.get_result()
if error is not None:
self.handle_error(error)
def list(self):
url = Zerigo._url_api + Zerigo._url_zones
Zerigo._logger.debug('retrieving ' + url)
reply = self._conn.get(url)
try:
Zerigo._logger.debug(reply.headers['x-query-count'] +
' zone(s) for account: ' + Zerigo.user)
except KeyError:
raise ParseError()
tree = ElementTree()
tree.parse(reply.body_file)
list = {}
zones = tree.getiterator('zone')
for it in zones:
name = it.find('domain')
id = it.find('id')
if id is None or name is None or id.text is None or name.text is None:
raise ParseError()
list[name.text] = Zone(name.text, id.text)
return list
def __init__(self, node: ET.Element, props: Dict, **kwargs):
filesets = kwargs['filesets']
if 'source' in node.attrib: # some old files may have these
raise ParseError(
'`source` attribute is deprecated. Switch to `src`.')
if 'source-set' in node.attrib:
raise ParseError(
'`source-set` attribute is deprecated. Switch to `src-set`.')
if 'src' in node.attrib:
self.source = [untempl(node.attrib['src'], props)]
elif 'src-set' in node.attrib:
self.source = filesets[node.attrib['src-set']].get_files()
else:
self.source = None
self.file = untempl(
node.attrib['file'], props
) if 'file' in node.attrib else None # file overrides source if specified
if self.file:
self.output = [self.file]
elif 'output' in node.attrib:
self.output = [untempl(node.attrib['output'], props)]
else:
self.output = [path + '.o' for path in self.source]
self.compiler = compilers.find(
name=node.attrib['compiler'] if 'compiler' in node.attrib else
None, # name overrides lang if specified
lang=node.attrib['lang'] if 'lang' in node.attrib else None)
self._preset_name = node.attrib[
'preset'] if 'preset' in node.attrib else None
self.params = parse_preset(node, props)
self.for_shlib = False
def parse_string(self, s):
unescape = {
"\\'": "'",
"\\\\": "\\",
"\\a": "\a",
"\\b": "\b",
"\\c": "\c",
"\\f": "\f",
"\\n": "\n",
"\\r": "\r",
"\\t": "\t",
"\\v": "\v",
'\\"': '"',
}
if not (s[0] == '"' and s[-1] == '"'):
raise ParseError("%d:%d Invalid string: %s" %
(self.lineno, self.column, s))
else:
s = s[1:-1]
out = ""
quote = ""
for c, (a, b) in enumerate(zip(s, s[1:])):
if a == "\\" and quote == "":
quote += a
else:
if quote != "":
if (quote + a) not in unescape:
raise ParseError(
"%d:%d Invalid escape sequence: %s" %
(self.lineno, self.column + c, quote + a))
out += unescape[quote + a]
else:
out += a
quote = ""
return (Tokenizer.STRING, out)
def list(self):
# This list (but up to 300 hosts) is also returned when we get
# <_url_list>
if (self._id is None):
raise NotFound(self.name)
url = Zerigo._url_api + Zone._url_hosts.substitute(zone_id=self._id)
Zerigo._logger.debug('retrieving ' + url)
reply = self._conn.get(url)
try:
Zerigo._logger.debug(reply.headers['x-query-count'] +
' host(s) for zone: ' + self.name)
except:
raise ParseError()
tree = ElementTree()
tree.parse(reply.body_file)
list = {}
hosts = tree.getiterator('host')
for it in hosts:
id = it.find('id')
hostname = it.find('hostname')
type = it.find('host-type')
data = it.find('data')
if id is None or type is None or data is None:
raise ParseError()
if hostname is None \
or 'nil' in hostname.attrib and hostname.attrib['nil'] == 'true':
hostname = '@' # Bind notation
else:
hostname = hostname.text
host = Host(hostname, self.name, id.text, self)
host.type = type.text
host.data = data.text
list[hostname] = host
return list
def _extract_text(html):
"""Given the html file, extract the text from it"""
if html is None:
raise InputError("HTML", html, "None HTML")
soup = BeautifulSoup(html)
if soup is None:
raise ParseError("Soup not made. Is HTML valid?")
news_all = soup.find_all('p')
if news_all is None:
raise ParseError("Error finding element p in HTML")
if len(news_all) == 0:
raise ParseError("Element p not found in the HTML")
logging.debug("Some p elements present in HTML. Getting text.")
# These comes specifically from studying the HTML layout of website.
text = u""
for n in news_all:
if n is None:
logging.debug("Found a None p element")
continue
elif not n.has_attr("class"):
logging.debug("Found a p element with no class attribute")
elif u"body" in n["class"]:
logging.debug("Found news element. Extracting tag.")
text += _extract_text_tag(n)
else:
logging.debug("Element p is not of body class")
if text == u"":
raise ParseError("Found no element p of class body")
logging.debug("News text extracted. Returning.")
return text
def parse_time(time_string):
time_string = time_string.lower().strip()
recurrence, time_descriptor = time_string.split(' ', 1)
is_recurring = None
if recurrence == 'after':
is_recurring = False
elif recurrence == 'every':
is_recurring = True
else:
raise ParseError(
"I don't understand '{}'; lines should start with 'after' or 'every'"
.format(time_string))
if ' ' in time_descriptor:
# E.g. "after 3 seconds", "every 20 minutes"
delay, units = time_descriptor.split(' ', 1)
delay = float(delay)
else:
# E.g. "every day"
delay = 1
units = time_descriptor
if units.startswith('sec'):
pass
elif units.startswith('min'):
delay *= 60
elif units.startswith('hour'):
delay *= 3600
elif units.startswith('day'):
delay *= 3600 * 24
else:
raise ParseError("I don't understand '{}'; unknown unit '{}'".format(
time_string, units))
return TimeTrigger(delay, is_recurring)
def modify_schedwalkxmlfile_by_walkrule(self, s_id, p_id, walkrule):
'''
Modify the schedwalk_xmlfile according to newly rxed walkrule
- s_id and p_id are string
'''
for child in self.walkrule_root:
if child.get('s_id') == s_id and child.get('p_id') == p_id:
self.walkrule_root.remove(
child) #old walkrule for session is deleted
#write the new s_walkrule
new_session = ET.Element('session')
new_session.set('s_id', s_id)
new_session.set('p_id', p_id)
for step_dict in walkrule:
new_conn = ET.SubElement(new_session, 'connection')
new_conn.set('dpid', str(step_dict['conn'][0]))
new_conn.set('from', step_dict['conn'][1])
new_type = ET.SubElement(new_conn, 'type')
new_type.text = step_dict['typ']
new_wcs = ET.SubElement(new_conn, 'wildcards')
new_wcs.set('src_ip', step_dict['wc'][0])
new_wcs.set('dst_ip', step_dict['wc'][1])
new_wcs.set('tp_dst', str(step_dict['wc'][2]))
new_rule = ET.SubElement(new_conn, 'rule')
if new_type.text == 'forward':
new_rule.set('fport', step_dict['rule'][0])
new_rule.set('duration', str(step_dict['rule'][1]))
elif new_type.text == 'modify_forward':
new_rule.set('fport', step_dict['rule'][2])
new_rule.set('duration', str(step_dict['rule'][3]))
new_rule.set('new_dst_ip', step_dict['rule'][0])
new_rule.set('new_dst_mac', step_dict['rule'][1])
else:
raise ParseError('Unexpected type', new_type.text)
self.walkrule_root.append((new_session))
self.indent(self.walkrule_root, 0)
"""
print '***'
print ET.dump(self.walkrule_root)
print '***'
"""
self.walkrule_tree.write(self.walkrule_xmlfile_url)
def get_walkruledict_forsp(self, s_id, p_id):
#s_id and p_id are string
hmfromdpid_dict = {}
dict_ = {}
for session in self.walkrule_root:
if session.get('s_id') == s_id and session.get('p_id') == p_id:
for conn in session.iter('connection'):
dpid = conn.get('dpid')
#
typ = conn.find('type').text
wc = conn.find('wildcards')
wc_dict = {}
wc_dict['src_ip'] = wc.get('src_ip')
wc_dict['dst_ip'] = wc.get('dst_ip')
wc_dict['tp_dst'] = wc.get('tp_dst')
#
rule = conn.find('rule')
rule_dict = {}
if typ == 'forward':
rule_dict['fport'] = rule.get('fport')
rule_dict['duration'] = rule.get('duration')
elif typ == 'modify_forward':
rule_dict['new_dst_ip'] = rule.get('new_dst_ip')
rule_dict['new_dst_mac'] = rule.get('new_dst_mac')
rule_dict['fport'] = rule.get('fport')
rule_dict['duration'] = rule.get('duration')
else:
raise ParseError('Unexpected type', typ)
#
if dpid in hmfromdpid_dict:
hmfromdpid_dict[dpid] += 1
else:
hmfromdpid_dict[dpid] = 0
#
tup = dpid, hmfromdpid_dict[dpid]
dict_[tup] = {
'typ': typ,
'wc_dict': wc_dict,
'rule_dict': rule_dict
}
return [dict_, hmfromdpid_dict]
def parse_children(self,
tokens_list,
sym_table,
end_type=None,
expect_next=None):
next_token = tokens_list.lookahead()
while not isinstance(next_token, end_type):
if isinstance(next_token, EOFToken):
raise ParseError(f"EOF Reached for {self.__class__.__name__}")
if expect_next and next_token.__class__ not in expect_next:
raise SyntaxError(
f"{self.__class__.__name__} expects one of: {expect_next}. {self.row}:{self.col}"
)
child_tree = next_token.parse_tree(tokens_list, sym_table)
if child_tree:
yield child_tree
next_token = tokens_list.lookahead()
def create(self):
# do not assert on that, because the id is initialized in the
# constructor, and the result is not available to the user.
if (self._id):
raise AlreadyExists(self.name)
# get the template, can be cached
url = Zerigo._url_api + Zone._url_template
Zerigo._logger.debug('retrieving ' + url)
template = self._conn.get(url)
# parse and use it to create the form to post
tree = ElementTree()
tree.parse(template.body_file)
domain = tree.find('domain')
if domain is None:
raise ParseError()
if 'nil' in domain.attrib:
del domain.attrib['nil']
domain.text = self.name
form = StringIO.StringIO()
tree.write(form)
form = form.getvalue()
# post it and read reply
url = Zerigo._url_api + Zone._url_create
try:
Zerigo._logger.debug('posting ' + url)
zone = self._conn.post(url, body=form)
except restkit.RequestFailed as errors:
errors = StringIO.StringIO(errors)
tree = ElementTree()
tree.parse(errors)
errors = [err.text for err in tree.findall('error')]
raise CreateError(self.name, ', '.join(errors))
self.__read_id(zone)
Zerigo._logger.debug('zone ' + self.name + ' created with id: ' +
self._id)
async def build_target(self, target: BuildTarget,
session: aiohttp.ClientSession):
# Simple DFS here
async with self.lock:
if target.up_to_date:
return
target.up_to_date = True
await target.replace_all_variables(self.parser)
await target.process_dependencies(self.parser)
target.replace_special_vars()
dependency_builds = []
for dependency in target.dependencies_targets:
dependency_builds.append(self.build_target(dependency, session))
await asyncio.gather(*dependency_builds)
newest_time = 0
target_files_time = target.get_latest_modification_time()
up_to_date = True
for file in target.dependencies_files_only:
if not os.path.exists(file):
raise ParseError(
f"Error! The file {file} does not exist but is in dependencies "
f"of a target {', '.join(target.target_files)}")
if os.path.getmtime(file) > target_files_time:
up_to_date = False
for dep_target in target.dependencies_targets:
newest_time = max(newest_time,
dep_target.get_latest_modification_time())
if up_to_date and newest_time < target.get_latest_modification_time(
) and target.targets_exist():
target.up_to_date = True
else:
await self.get_compiled_file(session, target)
def _deep_inner(el, elref, elpos, bexpr):
# If the element does not preceed lambda position or
# is not already in the lambdas function arguments
if elref in argsig or elpos.lineno > srcpos.lineno:
return
# If the element does proceed the lambda expression
elif elpos.lineno < srcpos.lineno or elpos.colno < srcpos.colno:
# Get the index of the element from element expression list
index = bexpr.index(el)
# Make a copy of the element
nc = copy.deepcopy(el)
# Replicate the expression to FuncArgReference
nc.exprs[0] = cntrl[0].replicate(elref)
# Replace base expression with funcarg reference
bexpr[index] = nc
# Copy to function arg newlist to indicate it is new
newlist.append(nc)
# Add to closures list
clist.append(el)
else:
raise ParseError(
"Unhandled lambda free variable location {}".format(
elpos))
