def found_instance(self, name, element):
check_isinstance(name, str)
where = element.where
if name in self.instances:
msg = 'Duplicated instances?'
raise DPInternalError(msg, where=where)
self.instances[name] = SemanticInformationForEntity(element)
def find_corrections(x, parents):
# expect an iterator
s = x.where.string[x.where.character:x.where.character_end]
for suggestion in correct(x, parents):
a, b = suggestion
if isinstance(a, str):
if not a in s:
msg = 'Invalid suggestion %s. Could not find piece %r in %r.' % (
suggestion, a, s)
raise DPInternalError(msg)
a_index = s.index(a)
a_len = len(a) # in bytes
a_char = x.where.character + a_index
a_char_end = a_char + a_len
a_where = Where(x.where.string, a_char, a_char_end)
else:
check_isinstance(a, Where)
a_where = a
check_isinstance(b, str)
sub = (a_where, b)
subs.append(sub)
return x
def found_uncertain_constant(self, name, element):
check_isinstance(name, str)
where = element.where
infer_debug('found uncertain constant: %s' % name)
if name in self.constants: # XXX
msg = 'Duplicated constants?'
raise DPInternalError(msg, where=where)
self.uncertain_constants[name] = SemanticInformationForEntity(element)
def step(self):
url = self.queue.pop(-1)
if url in self.visited:
return
o = urlparse.urlparse(url)
if self.ignore(url, o):
self.skipped.add(url)
return
logger.debug('requests %s ... ' % url)
try:
url2, res = self.get_maybe_follow(url)
except AppError as e:
s = unicode(e).encode('utf8')
s = saxutils.unescape(s)
if '500' in s:
self.failed[url] = s
logger.error('failed %s' % url)
return
elif '404' in s:
self.not_found[url] = s
logger.error('not found %s' % url)
return
else:
msg = 'Cannot classify this as 404 or 500:'
msg += '\n' + str(s)
raise DPInternalError(msg)
if url2 != url:
self.visited[url] = 'redirect to %s' % url2
logger.debug('redirected %s -> %s' % (url, url2))
self.visited[url2] = res
if res.content_type == 'text/html':
#print res.html
urls = list(find_links(res.html, url2))
logger.debug('read %s %s: %d links' %
(url2, res.status, len(urls)))
for u in urls:
p = urlparse.urlparse(u)
invalid = False
invalid = invalid or '../../../' in p.path
invalid = invalid or '//' in p.path
if invalid:
msg = 'We generated a URL that is weird: '
msg += '\n URL: %s ' % u
msg += '\n generated by: %s ' % url2
if url != url2:
msg += '\n redirected from: %s ' % url
raise ValueError(msg)
self.queue.append(u)
self.referrers[u].add(url2)
def load_spec(self, spec_name, thing_name, context=None):
from mcdp_library.specs_def import specs
spec = specs[spec_name]
parsing_function = spec.parse
if parsing_function is None:
msg = 'Cannot parse %s because the parsing function is not given.' % spec_name
raise DPInternalError(msg)
res = self._load_generic(thing_name, spec_name, parsing_function,
context)
check_isinstance(res, spec.klass)
return res
def parse(string, context=None):
try:
return f(string, context)
except DPSemanticError as e:
if e.where is not None:
if e.where.string != string:
msg = 'I expected this error to refer to somewhere in this string.'
msg += '\n string: %r' % string
msg += '\n e.where.string: %r' % e.where.string
msg += '\n' + indent(traceback.format_exc(e), 'e > ')
raise DPInternalError(msg)
raise
def change_was_inside_YAML(view, data_event, disk_map):
''' Checks whether the change was inside a YAML file. '''
if not 'name' in data_event['arguments']:
msg = 'Expected all events to have a "name" argument.'
raise DPInternalError(msg)
name = data_event['arguments']["name"]
for i in range(len(name) + 1):
p = name[:i]
p_schema = view._schema.get_descendant(p)
p_hint = disk_map.get_hint(p_schema)
if isinstance(p_hint, HintFileYAML):
return True, p
else:
return False, None
def make_resource(self, dp, s):
if not isinstance(dp, str):
raise DPInternalError((dp, s))
if not dp in self.names:
msg = 'Unknown design problem %r.' % dp
raise DPSemanticError(msg)
ndp = self.names[dp]
if not s in ndp.get_rnames():
msg = 'Unknown resource %r for design problem %r.' % (s, dp)
msg += ' Known functions: %s.' % format_list(ndp.get_rnames())
raise DPSemanticError(msg)
return CResource(dp, s)
def get_svg_for_visualization(e, image_source, library_name, spec, name, thing,
refined, make_relative, library):
svg_data0 = spec.get_png_data_syntax(image_source=image_source,
name=name,
thing=thing,
data_format='svg',
library=library)
fragment = bs(svg_data0)
if fragment.svg is None:
msg = 'Cannot interpret fragment.'
msg += '\n' + indent(svg_data0, '> ')
raise DPInternalError(msg)
assert fragment.svg is not None
style = {}
for a in ['width', 'height']:
if a in fragment.svg.attrs:
value = fragment.svg.attrs[a]
del fragment.svg.attrs[a]
style['max-%s' % a] = value
add_style(fragment.svg, **style)
remove_doctype_etc(fragment)
remove_all_titles(fragment.svg)
if refined is not None:
table = identifier2ndp(refined)
else:
table = {}
def link_for_dp_name(identifier0):
identifier = identifier0 # todo translate
if identifier in table:
a = table[identifier]
libname = a.libname if a.libname is not None else library_name
href0 = '/repos/%s/shelves/%s/libraries/%s/models/%s/views/syntax/' % (
e.repo_name, e.shelf_name, libname, a.name)
return make_relative(href0)
else:
return None
add_html_links_to_svg(fragment.svg, link_for_dp_name)
svg_data = to_html_stripping_fragment(fragment)
return svg_data
def warn_language(element, which, msg, context):
"""
element: a namedtuplewhere
which: one of the strings
msg: a string
"""
check_isinstance(msg, str)
where = element.where
msg2 = msg.strip() + '\n\n' + indent(str(where), ' ' * 4)
logger.debug(msg2)
if context is not None:
w = MCDPWarning(which=which, where=where, msg=msg)
context.warnings.append(w)
else:
msg = 'Context is None so I cannot record this.'
raise DPInternalError(msg)
logger.debug(msg)
def order_dps(name2dp, connections):
""" Returns a total order consistent with the partial order """
names = set(name2dp)
# List the ones that have no functions or no resources
# a >= 10 g (no functions)
# b <= 10 s (no resources)
# # no_functions = set()
# # no_resources = set()
# #
# # for name, ndp in name2dp.items():
# # if not ndp.get_fnames():
# # no_functions.add(name)
# # if not ndp.get_rnames():
# # no_resources.add(name)
#
# print('no_functions: %s' % no_functions)
# print('no_resources: %s' % no_resources)
G = get_connection_graph(names, connections)
# I should probably think more about this
# for nf in no_functions:
# for nr in no_resources:
# G.add_edge(nr, nf)
Gu = G.to_undirected()
if not is_connected(Gu):
msg = 'The graph is not weakly connected. (missing constraints?)'
msg += '\nNames: %s' % names
msg += '\nconnections: %s' % connections
raise DPSemanticError(msg)
l = list(topological_sort(G))
if not (set(l) == names):
msg = 'names = %s\n returned = %s\n connections: %s' % (names, l,
connections)
msg += '\n graph: %s %s' % (list(Gu.nodes()), list(Gu.edges()))
raise DPInternalError(msg)
return l
def eval_lfunction_invplus_ops(fs, context):
if len(fs) == 1:
raise DPInternalError(fs)
elif len(fs) > 2: # pragma: no cover
mcdp_dev_warning('Maybe this should be smarter?')
rest = eval_lfunction_invplus_ops(fs[1:], context)
return eval_lfunction_invplus_ops([fs[0], rest], context)
else:
Fs = map(context.get_ftype, fs)
R = Fs[0]
if all(isinstance(_, RcompUnits) for _ in Fs):
tu = get_types_universe()
if not tu.leq(Fs[1], Fs[0]):
msg = 'Inconsistent units %s and %s.' % (Fs[1], Fs[0])
raise_desc(DPSemanticError, msg, Fs0=Fs[0], Fs1=Fs[1])
if not tu.equal(Fs[1], Fs[0]):
msg = 'This case was not implemented yet. Differing units %s and %s.' % (
Fs[1], Fs[0])
raise_desc(DPNotImplementedError, msg, Fs0=Fs[0], Fs1=Fs[1])
dp = InvPlus2(R, tuple(Fs))
elif all(isinstance(_, Rcomp) for _ in Fs):
dp = InvPlus2(R, tuple(Fs))
elif all(isinstance(_, Nat) for _ in Fs):
dp = InvPlus2Nat(R, tuple(Fs))
else: # pragma: no cover
msg = 'Cannot find operator for these types.'
raise_desc(DPInternalError, msg, Fs=Fs)
return create_operation_lf(context,
dp=dp,
functions=fs,
name_prefix='_invplus',
op_prefix='_',
res_prefix='_result')
def create_operation_lf(context, dp, functions, name_prefix=None,
op_prefix='_op', res_prefix='_res', allow_conversion=True):
if name_prefix is None:
name_prefix = '_%s' % type(dp).__name__
name = context.new_name(name_prefix)
name_result = context.new_res_name(res_prefix)
rnames = []
for i, f in enumerate(functions):
ni = context.new_fun_name('%s%s' % (op_prefix, i))
rnames.append(ni)
_rnames = rnames[0] if len(rnames) == 1 else rnames
ndp = dpwrap(dp, name_result, _rnames)
connections = []
tu = get_types_universe()
for i, f in enumerate(functions):
# source resource
Fi = context.get_ftype(f)
# function
Fhave = ndp.get_rtype(rnames[i])
# print('------- argu %d' % i)
#
# print('I need to connect function %s of type %s to resource %s of new NDP with type %s'%
# (f, Fi, rnames[i], Fhave))
#
# print('Fi: %s' % Fi)
# print('Fhave: %s' % Fhave)
if not tu.equal(Fi, Fhave):
if not allow_conversion:
msg = ('The types are %s and %s are not equal, and '
'allow_conversion is False' % (Fi, Fhave))
raise DPInternalError(msg)
# print('creating conversion')
conversion = get_conversion(Fhave, Fi)
if conversion is None:
msg = 'I need a conversion from %s to %s' % (Fi, Fhave)
raise DPInternalError(msg)
else:
# print('Conversion: %s' % conversion.repr_long())
# print('Creating recursive...')
f = create_operation_lf(context, conversion, [f],
name_prefix='_conversion_for_%s' % name_result,
allow_conversion=False)
c = Connection(dp2=f.dp, s2=f.s, dp1=name, s1=rnames[i])
connections.append(c)
context.add_ndp(name, ndp)
for c in connections:
context.add_connection(c)
res = context.make_function(name, name_result)
return res
def create_operation(context, dp, resources, name_prefix=None, op_prefix=None, res_prefix=None):
"""
This is useful to create operations that take possibly many inputs
and produce one output.
Example use:
R = mult_table_seq(resources_types)
dp = ProductN(tuple(resources_types), R)
from mcdp_lang.helpers import create_operation
r = create_operation(context, dp, resources,
name_prefix='_prod', op_prefix='_factor',
res_prefix='_result')
"""
if name_prefix is None:
name_prefix = '_%s' % type(dp).__name__
# new name for the ndp
name = context.new_name(name_prefix)
if op_prefix is None:
op_prefix = '_op'
if res_prefix is None:
res_prefix = '_res'
name_result = context.new_res_name(res_prefix)
connections = []
fnames = []
for i, r in enumerate(resources):
ni = context.new_fun_name('%s%s' % (op_prefix, i))
fnames.append(ni)
fnames_ = fnames[0] if len(fnames) == 1 else fnames
ndp = dpwrap(dp, fnames_, name_result)
context.add_ndp(name, ndp)
tu = get_types_universe()
for i, r in enumerate(resources):
# this is where we check for types
# source resource
R = context.get_rtype(r)
# function
F = ndp.get_ftype(fnames[i])
if not tu.equal(F, R):
conversion = get_conversion(R, F)
if conversion is None:
msg = 'I need a conversion from %s to %s' % (R, F)
raise DPInternalError(msg)
else:
r = create_operation(context, conversion, [r],
name_prefix='_conversion_for_%s' % name_result)
R = context.get_rtype(r)
assert tu.equal(F, R)
c = Connection(dp1=r.dp, s1=r.s, dp2=name, s2=fnames[i])
connections.append(c)
for c in connections:
context.add_connection(c)
res = context.make_resource(name, name_result)
return res
def ast_to_html(s,
parse_expr,
ignore_line=None,
add_line_gutter=True,
encapsulate_in_precode=True,
postprocess=None):
"""
postprocess = function applied to parse tree
"""
check_isinstance(s, str)
# if parse_expr is None:
# raise Exception('Please add specific parse_expr (default=Syntax.ndpt_dp_rvalue)')
if ignore_line is None:
ignore_line = lambda _lineno: False
original_lines = s.split('\n')
s_lines, s_comments = isolate_comments(s)
assert len(s_lines) == len(s_comments)
num_empty_lines_start = 0
for line in s_lines:
if line.strip() == '':
num_empty_lines_start += 1
else:
break
num_empty_lines_end = 0
for line in reversed(s_lines):
if line.strip() == '':
num_empty_lines_end += 1
else:
break
full_lines = s_lines[num_empty_lines_start:len(s_lines) -
num_empty_lines_end]
from mcdp_report.out_mcdpl import extract_ws
for_pyparsing0 = "\n".join(full_lines)
# remove also initial and final whitespace
extra_before, for_pyparsing, extra_after = extract_ws(for_pyparsing0)
# parse the string 'for_pyparsing'
block0 = parse_wrap(parse_expr, for_pyparsing)[0]
# print indent(recursive_print(block0), ' block0 |')
assert isnamedtuplewhere(block0)
# now transform everything so that it refers to s
transform_original_s = s
def transform(x, parents): # @UnusedVariable
w0 = x.where
s0 = w0.string
def translate(line, col):
# add initial white space on first line
if line == 0:
col += len(extra_before)
# add the initial empty lines
line += num_empty_lines_start
return line, col
# these are now in the original string transform_original_s
line, col = translate(*line_and_col(w0.character, s0))
character = location(line, col, transform_original_s)
if w0.character_end is None:
character_end = None
else:
line, col = translate(*line_and_col(w0.character_end, s0))
character_end = location(line, col, transform_original_s)
where = Where(string=transform_original_s,
character=character,
character_end=character_end)
return get_copy_with_where(x, where)
block = namedtuple_visitor_ext(block0, transform)
assert isnamedtuplewhere(block)
if postprocess is not None:
block = postprocess(block)
if not isnamedtuplewhere(block):
raise ValueError(block)
snippets = list(print_html_inner(block))
# the len is > 1 for mcdp_statements
assert len(snippets) == 1, snippets
snippet = snippets[0]
transformed_p = snippet.transformed
# if block.where.character != 0:
# assert False
# transformed_p = for_pyparsing[:block.where.character] + transformed_p
# re-add the initial and final space here
transformed_p = extra_before + transformed_p + extra_after
def sanitize_comment(x):
x = x.replace('>', '>')
x = x.replace('<', '<')
return x
# re-add the initial and final lines
transformed = ''
transformed += "\n".join(s_lines[:num_empty_lines_start])
if num_empty_lines_start:
transformed += '\n'
transformed += transformed_p
if num_empty_lines_end:
transformed += '\n'
transformed += "\n".join(s_lines[len(original_lines) -
num_empty_lines_end:])
lines = transformed.split('\n')
if len(lines) != len(s_comments):
msg = 'Lost some lines while pretty printing: %s, %s' % (
len(lines), len(s_comments))
raise DPInternalError(msg)
# print('transformed', transformed)
out = ""
for i, (line, comment) in enumerate(zip(lines, s_comments)):
lineno = i + 1
if ignore_line(lineno):
continue
else:
# print('line %d' % i)
# print(' oiginal line: %r' % original_lines[i])
# print(' line: %r' % line)
# print(' comment: %r' % comment)
original_line = original_lines[i]
if comment is not None:
assert '#' in original_line
if '#' in original_line:
if '#' in line:
w = line.index('#')
before = line[:
w] # (already transformed) #original_line[:w]
comment = line[w:]
else:
before = line
comment = comment
# print(' before: %r' % comment)
# print(' comment: %r' % comment)
if comment.startswith(unparsable_marker):
unparsable = comment[len(unparsable_marker):]
linec = before + '<span class="unparsable">%s</span>' % sanitize_comment(
unparsable)
else:
linec = before + '<span class="comment">%s</span>' % sanitize_comment(
comment)
else:
linec = line
# print(' linec: %r' % linec)
if add_line_gutter:
out += "<span class='line-gutter'>%2d</span>" % lineno
out += "<span class='line-content'>" + linec + "</span>"
else:
out += linec
if i != len(lines) - 1:
out += '\n'
if MCDPConstants.test_insist_correct_html_from_ast_to_html:
from xml.etree import ElementTree as ET
ET.fromstring(out)
# print 'ast_to_html, out', out
frag = ""
if encapsulate_in_precode:
frag += '<pre><code>'
frag += out
frag += '</code></pre>'
else:
frag += out
assert isinstance(frag, str)
return frag
def generate_view_syntax(e, make_relative):
expr = e.spec.parse_expr
parse_refine = e.spec.parse_refine
source_code = e.thing
context = Context()
class Tmp:
refined = None
def postprocess(block):
if parse_refine is None:
return block
try:
Tmp.refined = parse_refine(block, context)
return Tmp.refined
except DPSemanticError:
return block
try:
highlight = ast_to_html(source_code,
add_line_gutter=False,
parse_expr=expr,
postprocess=postprocess)
def get_link_library(libname):
try:
rname, sname = e.session.get_repo_shelf_for_libname(libname)
except NoSuchLibrary:
raise
url0 = "/repos/%s/shelves/%s/libraries/%s/" % (rname, sname,
libname)
return make_relative(url0)
def get_link(specname, libname, thingname):
# find library. Returns a string or raises error
try:
rname, sname = e.session.get_repo_shelf_for_libname(libname)
except NoSuchLibrary:
msg = 'No such library %r' % libname
logger.debug(msg)
raise
# return None
things = e.db_view.repos[rname].shelves[sname].libraries[
libname].things.child(specname)
if thingname in things:
# check if the thing exists
res = get_link_library(
libname) + '%s/%s/views/syntax/' % (specname, thingname)
# logger.debug(' link for %s = %s' % (thingname, res))
return res
else:
msg = 'No such thing %r' % thingname
logger.debug(msg)
raise NoSuchLibrary(msg)
highlight = add_html_links(highlight, e.library_name, get_link,
get_link_library)
parses = True
error = ''
except (DPSyntaxError, DPNotImplementedError) as exc:
highlight = '<pre class="source_code_with_error">%s</pre>' % source_code
error = exc.__str__()
parses = False
if parses:
mcdp_library = library_from_env(e)
image_source = image_source_from_env(e)
try:
thing = e.spec.load(mcdp_library, e.thing_name, context=context)
svg_data = get_svg_for_visualization(e,
image_source,
e.library_name,
e.spec,
e.thing_name,
thing,
Tmp.refined,
make_relative,
library=mcdp_library)
except (DPSemanticError, DPNotImplementedError) as exc:
logger.error(exc)
from mcdp_web.editor_fancy.app_editor_fancy_generic import html_mark
if exc.where.string != source_code:
msg = 'This exception refers to another file.'
msg += '\n source_code: %r' % source_code
msg += '\n exception.where.string: %r' % exc.where.string
msg += '\n' + indent(traceback.format_exc(exc), 'exc > ')
raise DPInternalError(msg)
try:
highlight = html_mark(highlight, exc.where, "semantic_error")
except NoLocationFound as e:
msg = 'While trying to annotate the exception:'
msg += '\n' + indent(exc, 'exc > ')
raise_wrapped(NoLocationFound, e, msg)
error = exc.error + "\n" + format_where(exc.where)
svg_data = None
else:
svg_data = None
check_isinstance(highlight, str)
res = {
'source_code': source_code,
'error': unicode(error, 'utf-8'),
'highlight': unicode(highlight, 'utf-8'),
# 'realpath': realpath,
'current_view': 'syntax',
'explanation1_html': None,
'explanation2_html': None,
'svg_data':
unicode(svg_data, 'utf-8') if svg_data is not None else None,
'parses': parses, # whether it parses
}
return res