def enumerate_model(s: str, vars: List[str], logic: str, bitwidth: int):
t = sexpdata.loads(s)
res = set()
if logic == "lia":
rc = treeutil.get_range_constraints(vars, 2 ** bitwidth)
t = ["and", t, rc]
while True:
s_giving = smtutil.list2sexp(t)
if logic == "lia":
res_sat = get_model_lia(s_giving, vars)
elif logic == "bv":
res_sat = get_model_bv(s_giving, vars, bitwidth)
else:
assert False
if res_sat == None:
break
else:
# print(res_sat)
if logic == "bv":
res_sat1 = {k: int(sexpdata.loads(v)[1]._val[2:]) for k, v in res_sat.items()}
elif logic == "lia":
res_sat1 = {k: int(v) for k, v in res_sat.items()}
else:
assert False
res.add(str(res_sat1))
blocks = []
for k, v in res_sat.items():
blocks += [["distinct", k, v]]
t = ["and"] + [t] + blocks
return res
def get_socket_response(self, connection, recursive=False):
command, data = self.recv_socket_data(connection)
log.debug('Socket Request Type: %s' % command)
if command == 'result':
return data
# Process generator to avoid pendant messages
data = gen_to_string(data)
if command == 'read':
if self.read_busy:
return
self.read_busy = True
return [Symbol(":read-string"), 0, 1]
if command == 'read-mode':
log.debug("Entering read mode...")
self.read_mode = True
return [Symbol(":read-string"), 0, 1]
if command == 'error':
if recursive:
log.debug('Waiting for repl output...')
self.finished_output.wait()
return
msg = loads(data)
stack = self.get_callstack()
if connection == self.euslime_internal_connection:
raise EuslispInternalError(msg, stack)
else:
raise EuslispError(msg, stack)
if command == 'abort':
msg = loads(data)
if msg:
msg = "'{}'".format(msg) # Better formatting
raise AbortEvaluation(msg)
raise Exception("Unhandled Socket Request Type: %s" % command)
def try_loads(sexp):
entry = loads(sexp)
try:
entry = loads(sexp)
assert get_user(entry) != None
assert get_time(entry)
assert get_session(entry)
return entry
except:
return None
def solve(logging, variable_types,monotonic_indices,input_features,datapoint, smtFileName = "",prefix_path=""):
start_time = time.time()
try:
optimatsatsolver_path = "optimathsat"
cmd = optimatsatsolver_path+" "+ prefix_path +smtFileName.name
elapsed_time = time.time() - start_time
p = subprocess.Popen([cmd, ""], shell=True,
stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
p.wait()
elapsed_time = time.time() - start_time
counter_examples = []
returned_value = p.stdout.read().decode('UTF-8')
if "unsat" in returned_value:
logging.debug("Unsat: No model")
return None, elapsed_time, None
noOfLines = len(returned_value.splitlines())
f_y_parsedsexp = loads(returned_value.splitlines()[noOfLines-1])
f_y = 0.0
if '-' in returned_value.splitlines()[noOfLines-1]:
f_y = -1 * float((1.0*f_y_parsedsexp[0][1][1][1])/f_y_parsedsexp[0][1][1][2])
else:
f_y = float((1.0*f_y_parsedsexp[0][1][1])/f_y_parsedsexp[0][1][2])
cg_parsedsexp = {}
count = 1
for monotonic_index in monotonic_indices:
cg_parsed = loads(returned_value.splitlines()[count])
cg = 0
if variable_types[monotonic_index] == "Int":
cg = int(cg_parsed[0][1])
else:
if '/' in returned_value.splitlines()[count]:
if '-' in returned_value.splitlines()[count]:
cg = -1 * float(cg_parsed[0][1][1][1]/cg_parsed[0][1][1][2])
else:
cg = float(cg_parsed[0][1][1]/cg_parsed[0][1][2])
else:
cg = float(cg_parsed[0][1])
count = count + 1
cg_parsedsexp[monotonic_index] = cg
logging.debug('violation = %2f'%(f_y))
for i in range(0,input_features):
if i in monotonic_indices:
datapoint[i] = cg_parsedsexp[i]
os.remove(prefix_path +smtFileName.name)
return datapoint,elapsed_time,f_y
except Exception as e:
print("Exception "+ str(e))
elapsed_time = time.time() - start_time
os.remove(prefix_path +smtFileName.name)
return None,elapsed_time,None
def tree_dist(x, y):
key = (x, y)
if key not in _tree_dist_cache:
x = sexpdata.loads(x)
y = sexpdata.loads(y)
dist = zss.simple_distance(
x, y, lambda w: w[1:]
if isinstance(w, list) else [], lambda w: w[0].value()
if isinstance(w, list) else w.value(), lambda w, z: 0
if w == z else 1)
_tree_dist_cache[key] = dist
return _tree_dist_cache[key]
def crossover(offspring, p1_index, p2_index):
p_1 = loads(offspring[p1_index][0])
p_2 = loads(offspring[p2_index][0])
position_1 = random.randint(0, node_number(p_1))
position_2 = random.randint(0, node_number(p_2))
p1_subtree = find_subtree(p_1, position_1, 0, p_1)[1]
p2_subtree = find_subtree(p_2, position_2, 0, p_2)[1]
p_1 = replace(p_1, position_1, p2_subtree, 0)[0]
p_2 = replace(p_2, position_2, p1_subtree, 0)[0]
return [[dumps(p_1), 0], [dumps(p_2), 0]]
def response(self, flow: http.HTTPFlow) -> None:
if self.should_persist(flow):
sexp_file = Path.home() / ".cache/mitmproxy-flows.lisp"
sexp = []
if os.path.isfile(sexp_file):
text = Path(sexp_file).read_text()
if text:
# strip leading quote for sexpdata
text = text[1:]
sexp = sexpdata.loads(text)
else:
# create empty file
open(sexp_file, 'a').close()
obj = {}
obj['url'] = flow.request.pretty_url
obj['status-code'] = flow.response.status_code
obj['request-content'] = \
self.get_pretty_json(flow.request.content.decode())
obj['response-content'] = \
self.get_pretty_json(flow.response.content.decode())
sexp.append(obj)
# prepend a quote for valid Lisp code
Path(sexp_file).write_text("'" + sexpdata.dumps(sexp))
def flatten(offset=0):
import itertools
from sexpdata import loads
with open('bookmarks.txt', 'r') as f:
sexp = loads(f.read())
# https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-list-of-lists
flat_list = []
depth = 1
for sublist in sexp[1:]: # skip Symbol(bookmarks)
if len(sublist) == 2:
flat_list.append([depth] + sublist)
else:
flat_list.append([depth] + sublist[:2])
for item in sublist[2:]:
flat_list.append([depth + 1] + item)
df = pd.DataFrame(flat_list)
df[3] = df[2] # page
df[2] = df[1] # title
df[1] = 0 # open
# page
df[3] = df[3].apply(lambda x: int(x.replace('#', '')) - offset)
df.to_csv('bookmarks.csv', sep=";", header=None, index=None)
print('wrote to bookmarks.csv')
def str_str_test():
print('str_str_test')
sexp = str(
''' (Answer 3(ObjList((CoqGoal((fg_goals(((name 3)(ty(Prod(Name(Id n))(Ind(((Mutind(MPfile(DirPath((Id Datatypes)(Id Init)(Id Coq))))(DirPath())(Id nat))0)(Instance())))(App(Ind(((Mutind(MPfile(DirPath((Id Logic)(Id Init)(Id Coq))))(DirPath())(Id eq))0)(Instance())))((Ind(((Mutind(MPfile(DirPath((Id Datatypes)(Id Init)(Id Coq))))(DirPath())(Id nat))0)(Instance())))(App(Const((Constant(MPfile(DirPath((Id Nat)(Id Init)(Id Coq))))(DirPath())(Id add))(Instance())))((Construct((((Mutind(MPfile(DirPath((Id Datatypes)(Id Init)(Id Coq))))(DirPath())(Id nat))0)1)(Instance())))(Rel 1)))(Rel 1)))))(hyp()))))(bg_goals())(shelved_goals())(given_up_goals()))))))\n'
''')
print(f'sexp = {sexp}')
psexp = loads(sexp)
def __init__(self, text: str):
self.text = text
_s = sexpdata.loads(clean_sexp(text))
assert len(_s) >= 4
self._center_len = 1
self.prev_constraints: Optional[Any] = None
if not self.RuleClass.is_any(_s[0]):
self.prev_constraints = self.SeqRuleClass(_s[0], False)
self.next_constraints: Optional[Any] = None
if not self.RuleClass.is_any(_s[2]):
self.next_constraints = self.SeqRuleClass(_s[2], True)
self.constraints: Optional[Any] = None
if not self.RuleClass.is_any(_s[1]):
self.constraints = self.SeqRuleClass(_s[1], True)
self._center_len = len(self.constraints)
self.operations: List[Tuple[bool, str]] = []
for v in _s[3:]:
feature = sexpdata.dumps(v)
remove = False
if feature.startswith("^"):
remove = True
feature = feature[1:]
self.operations.append((remove, feature))
def test_any(self):
sample = clean_sexp(""" < (?* [副詞 * * * * ((数量相対名詞修飾))] ) > """)
in_parts = [""" ( ?* ) """, sample]
golds = [True, False]
for intext, gold in zip(in_parts, golds):
p = sexpdata.loads(intext)
self.assertEqual(gold, TagRule.is_any(p))
def exec_internal(self, cmd_str, force_repl_socket=False):
if force_repl_socket:
# When the command must be evaluated in the main thread due to
# e.g. Thread Special variables
# Locks are performed from outside to yield results before release
connection = self.euslime_connection
log.info('exec_internal(repl): %s' % cmd_str)
else:
connection = self.euslime_internal_connection
log.info('exec_internal: %s' % cmd_str)
lock = self.euslime_internal_connection_lock
log.debug('Acquiring lock: %s' % lock)
lock.acquire()
try:
self.clear_socket_stack(connection)
connection.send(cmd_str + self.delim)
gen = self.get_socket_result(connection)
res = gen_to_string(gen)
if not force_repl_socket:
lock.release()
# Keep nil as a symbol to be dump-reversible
res = loads(res, nil=None)
# But return python-false objects if the response is solely 'nil'
if res == Symbol("lisp:nil") or res == Symbol("nil"):
return []
return res
except Exception:
if not force_repl_socket and lock.locked():
lock.release()
raise
def process(self, data):
data = loads(data)
if data[0] == Symbol(":emacs-rex"):
cmd, form, pkg, thread, comm_id = data
self.handler.command_id = comm_id
self.handler.package = pkg
else:
form = data
comm_id = None
func = form[0].value().replace(':', '_').replace('-', '_')
args = form[1:]
log.info("func: %s" % func)
log.info("args: %s" % args)
try:
gen = getattr(self.handler, func)(*args)
if not gen:
if comm_id:
for r in self.make_response(comm_id, None):
yield r
return
for resp in gen:
if isinstance(resp, EuslispResult):
for r in self.make_response(self.handler.command_id,
resp.value):
yield r
else:
yield self.dumps(resp)
except Exception as e:
log.error(traceback.format_exc())
for r in self.make_error(self.handler.command_id, e):
yield r
def fields_remove_entry(values_modify, re_flags=re.IGNORECASE):
'''Remove a list of fields from the Eeschema template.'''
if type(values_modify) is not list:
values_modify = [values_modify]
config = read_config_file(PATH_EESCHEMA_CONFIG)
values = [p for p in config if p.startswith("FieldNames")]
changes = False
delete_list = []
if len(values) == 1:
values = after(values[0], "FieldNames=")
values = de_escape(values)
values = sexpdata.loads(values)
# TODO validate it, was using an undefined variable `name`
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
values = values.replace("\\", '/')
for value_modify in values_modify:
for idx, value in enumerate(values[1:]):
search = value[1]
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
search = value[1].replace("\\", '/')
if re.findall(value_modify, search[1], re_flags):
changes = True # The name in really in the 'name'.
delete_list.append(idx) # We want to delete this entry.
for delete in sorted(set(delete_list), reverse=True):
del values[delete+1]
if changes:
s = sexpdata.dumps(values)
config = update_config_file(config, "FieldNames", escape(s))
write_config_file(PATH_EESCHEMA_CONFIG, config)
def bom_plugin_remove_entry(name, re_flags=re.IGNORECASE):
'''Remove a BOM plugin entry to the Eeschema configuration file.'''
config = read_config_file(PATH_EESCHEMA_CONFIG)
bom_plugins_raw = [p for p in config if p.startswith("bom_plugins")]
new_list = []
new_list.append(sexpdata.Symbol("plugins"))
changes = False
if len(bom_plugins_raw) == 1:
bom_plugins_raw = after(bom_plugins_raw[0], "bom_plugins=")
bom_plugins_raw = de_escape(bom_plugins_raw)
bom_list = sexpdata.loads(bom_plugins_raw)
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
name = name.replace("\\", '/')
for plugin in bom_list[1:]:
search = plugin[1]
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
search = plugin[1].replace("\\", '/')
if re.findall(name, search, re_flags):
changes = True # The name in really in the 'name'.
continue # We want to delete this entry.
else:
for entry in plugin[2:]:
if entry[0] == sexpdata.Symbol('opts') and re.findall(r'nickname\s*=\s*'+name, entry[1], re_flags):
changes = True
continue # The name is in the 'nickname'.
new_list.append(plugin) # This plugin remains on the list.
if changes:
s = sexpdata.dumps(new_list)
config = update_config_file(config, "bom_plugins", escape(s))
write_config_file(PATH_EESCHEMA_CONFIG, config)
def _parse_preceptors(self, raw_preceptors):
"""Takes raw preceptor data and gives usable data"""
data = sexpdata.loads("(" + raw_preceptors + ")")
self.time = data[0][-1][-1]
self.gyr = data[2][-1][-3:]
self.filter_acc(data[3][-1][-3:])
self.state = {}
for arg in data[4:]:
if sexpdata.dumps(arg[0]) == 'HJ':
self.state[sexpdata.dumps(arg[-2][-1]).replace(
'j', 'e')] = float(arg[-1][-1])
for arg in data[4:]:
if sexpdata.dumps(arg[0]) == 'See':
for a in arg[1:]:
if sexpdata.dumps(a[0]) == 'mypos':
self.pos = a[1:]
if sexpdata.dumps(a[0]) == 'myorien':
self.orr = a[-1]
break
break
# Debug
# print('(_parse_preceptors) message -> ', raw_preceptors)
# for s in self.state:
# print("(_parse_preceptors)", s, self.state[s])
# if self.is_fallen():
# print(self.time, self.gyr, self.acc, self.pos, self.orr, self.is_fallen())
return self.state, self.acc, self.gyr, self.pos, self.orr, float(
self.time), self.is_fallen(),
def fields_add_entry(values_modify, re_flags=re.IGNORECASE):
'''Add a list of fields to the Eeschema template.'''
if type(values_modify) is not list:
values_modify = [values_modify]
config = read_config_file(PATH_EESCHEMA_CONFIG)
values = [p for p in config if p.startswith("FieldNames")]
changes = False
if len(values) == 1:
values = after(values[0], "FieldNames=")
values = de_escape(values)
values = sexpdata.loads(values)
# TODO validate it, was using an undefined variable `name`
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
values = values.replace("\\", '/')
for idx, value_modify in enumerate(values_modify):
value_found = False
for idx, value in enumerate(values[1:]):
search = value[1]
if sys.platform.startswith(PLATFORM_WINDOWS_STARTS_WITH):
search = value[1].replace("\\", '/')
if re.findall(value_modify, search[1], re_flags):
value_found = True
if not value_found:
values.append([sexpdata.Symbol('field'), [sexpdata.Symbol('name'), value_modify]])
changes = True
if changes:
s = sexpdata.dumps(values)
config = update_config_file(config, "FieldNames", escape(s))
write_config_file(PATH_EESCHEMA_CONFIG, config)
def test_rule(self):
in_samples = [
clean_sexp("""< (?* [副詞 * * * * ((数量相対名詞修飾))] ) >"""),
clean_sexp("""< (?*) ((ダミー1 ^ダミー2)) >"""),
clean_sexp("""< (?* [* * * * * ((付属))]) >"""),
clean_sexp("""< (?* [* * * * * ((付属))]*) >"""),
]
gold_fc_strs = [
"None",
"""FC<SI=['ダミー1'], SNI=['ダミー2']>""",
"None",
"None",
]
gold_nums = [
2,
1,
2,
2,
]
for intext, gold_fc_str, gold_num in zip(in_samples, gold_fc_strs,
gold_nums):
p = sexpdata.loads(intext)
tr = TagRule(p)
self.assertEqual(gold_fc_str, str(tr.feature_constraints))
self.assertEqual(gold_num, len(tr.rules))
def get_message(line):
m = Message()
sp = line.split("::")
m.timestamp = sp[0]
m.verb = sp[1]
m.content = loads(sp[2])
return m
def _parse_module_versions(bap_output):
module_versions = {}
for line in bap_output.splitlines():
if 'module_versions:' in line:
version_sexp = line.split('module_versions:')[-1].strip()
module_versions = dict(sexpdata.loads(version_sexp))
return module_versions
def recieve_msg(self, log=None):
msg, address = self.s.recvfrom(8192)
msg = tostring(loads(msg[:-1].decode("utf-8")))
if msg[0] == "think":
if log is None:
log = self.recieve_log
if log:
print(
"\033[38;5;12m[INFO]" +
("\033[38;5;13mno \033[4m" + self.no + "\033[0m ") *
(self.no != "") +
"\t\033[0m\033[38;5;10mGet \033[4mthink\033[0m msg.\t\033[38;5;9mPORT "
"\033[4m" + str(self.recieve_port) +
"\033[0m\033[38;5;9m ← \033[4m" + str(address[1]) +
"\033[0m\t\033[38;5;6mIP \033[4m" + address[0] + "\033[0m")
self.send_msg("(done)", self.player_port)
return self.recieve_msg(log)
else:
if log is None:
log = self.recieve_log
if log:
print(
"\033[38;5;12m[INFO]" +
("\033[38;5;13mno \033[4m" + self.no + "\033[0m ") *
(self.no != "") +
"\t\033[0m\033[38;5;10mGet msg.\t\033[38;5;9mPORT \033[4m"
+ str(self.recieve_port) +
"\033[0m\033[38;5;9m ← \033[4m" + str(address[1]) +
"\033[0m\t\033[38;5;6mIP \033[4m" + address[0] + "\033[0m")
return msg, address
def rename_ext_lib(tarlib, deslib):
for i in range(len(extliblist)):
if tarlib in extliblist[i]:
data[extliblist[i][1]][1] = sexpdata.loads(deslib)
list_ext()
return 1
return 0
def test_parse_iomap_3():
expr = "'(Q ((AA2 \"3.3-V LVTTL\") (AA1 \"3.3-V LVTTL\")))"
l = parser.parse_iomap(sexpdata.loads(expr))
assert (l == [
data.IOMAP('Q[0]', pin='AA2', iostd='3.3-V LVTTL'),
data.IOMAP('Q[1]', pin='AA1', iostd='3.3-V LVTTL'),
])
def process_s_expr(s_expr_str, tagset):
return process_tree(
loads(
s_expr_str
)[0],
tagset
)
def merge_multiple_observations(parser_output):
ret = parser_output
if parser_output:
lines = parser_output.splitlines()
if lines and len(lines) > 1:
ret = []
counter = 0
preds = [sexpdata.Symbol("^")]
for l in lines:
o = sexpdata.loads(l)
if not ret:
ret.append(o[0])
ret.append(o[1])
for p in o[2]:
if type(p) == list:
np = [p[0]]
for a in p[1:]:
if type(a) == sexpdata.Symbol:
n = a.value()
if n[0].islower():
np.append(
sexpdata.Symbol(
a.value() + "_" + str(counter)
)
)
continue
np.append(a)
preds.append(np)
# print(sexpdata.dumps(o))
counter += 1
ret.append(preds)
ret = print_pred(ret)
return ret
def exec_internal(self, cmd_str):
self.clear_socket_stack()
log.info('exec_internal: %s' % cmd_str)
self.euslime_connection.send(cmd_str + self.delim)
gen = self.get_socket_response()
res = ''.join(list(gen))
return loads(res)
def parse_conf(self, path):
def paired(iterable):
"""s -> (s0, s1), (s2, s3), (s4, s5), ..."""
cursor = iter(iterable)
return zip(cursor, cursor)
def unwrap_if_sexp_symbol(datum):
"""
Convert Symbol(':key') to ':key' (Symbol isn't hashable for dict keys).
"""
return datum.value() if isinstance(datum, sexpdata.Symbol) else datum
def sexp2dict(sexps):
"""
Transforms a nested list structure from sexpdata to dict.
NOTE: This probably isn't general for all S-expression shapes parsed by
sexpdata, focused only on .ensime thus far.
"""
newdict = {}
# Turn flat list into associative pairs
for key, value in paired(sexps):
key = str(unwrap_if_sexp_symbol(key)).lstrip(':')
# Recursively transform nested lists
if isinstance(value, list) and value and isinstance(value[0], list):
newdict[key] = [sexp2dict(value[0])]
else:
newdict[key] = value
return newdict
conf = sexpdata.loads(Util.read_file(path))
return sexp2dict(conf)
def parse_leadsheet(fn,verbose=False):
with open(fn,'r') as f:
contents = "\n".join(f.readlines())
parsed = sexpdata.loads("({})".format(contents.replace("'","")))
parts = [('default','',[])]
for p in parsed:
if not isinstance(p, list):
parts[-1][2].append(p.value())
elif not isinstance(p[0], list) and p[0].value() == 'part':
def strval(x):
return x.value() if isinstance(x,sexpdata.Symbol) else str(x)
part_type = next((' '.join(strval(x) for x in l[1:]) for l in p if isinstance(l,list) and l[0].value() == "type"), None)
title = next((' '.join(strval(x) for x in l[1:]) for l in p if isinstance(l,list) and l[0].value() == "title"), '')
parts.append((part_type, title, []))
chord_parts = [x for x in parts if x[0]=='chords']
if len(chord_parts) == 0:
chord_parts = [x for x in parts if x[0]=='default']
assert len(chord_parts) == 1, 'Wrong number of chord parts!'
chords_raw = [x for x in chord_parts[0][2] if x[0].isupper() or x in ("|", "/")]
chords = []
partial_measure = []
last_chord = None
for c in chords_raw:
if c == "|":
length_each = constants.WHOLE//(len(partial_measure)*constants.RESOLUTION_SCALAR)
for chord in partial_measure:
for x in range(length_each):
chords.append(chord)
partial_measure = []
else:
if c != "/":
last_chord = parse_chord(c,verbose)
partial_measure.append(last_chord)
melody = []
for part_type, title, part_data in parts:
if part_type == 'melody':
melody_raw = [x for x in part_data if x[0].islower()]
melody_proc = [parse_note(x) for x in melody_raw]
mlen = sum(dur for n,dur in melody_proc)
if mlen < len(chords):
melody_proc.append((None, len(chords)-mlen))
melody.extend(melody_proc)
# print "Raw Chords: " + " ".join(chords_raw)
# print "Raw Melody: " + " ".join(melody_raw)
# print "Parsed chords: "
# repeat_print(chords)
# print "Parsed melody: "
# pprint(melody)
clen = len(chords)
mlen = sum(dur for n,dur in melody)
# Might have multiple melodies over the same chords
assert mlen % clen == 0, "Notes and chords don't match in {}: {}, {}".format(fn, clen,mlen)
return chords, melody
def main(argv):
model = _model()
fake_utts = []
real_utts = []
lfs = []
train_file = os.path.join(
FLAGS.data_dir, "data",
"{}.paraphrases.train.examples".format(FLAGS.dataset))
with open(train_file) as f:
train_str = f.read()
train_data = sexpdata.loads("({})".format(train_str))
num_train = len(train_data)
if FLAGS.max_examples is not None:
num_train = min(num_train, FLAGS.max_examples)
num_train = int(num_train * FLAGS.train_frac)
train_data = train_data[:num_train]
for datum in train_data:
real = datum[1][1]
fake = datum[2][1]
lf = sexpdata.dumps(datum[3][1]).replace("\\.", ".")
fake_utts.append(fake)
real_utts.append(real)
lfs.append(lf)
model.train(real_utts, fake_utts, lfs)
model.save(FLAGS.write_model)
def ast_from_sexp(s):
'''
Generate an ast from a simple s-expression string.
The top node is assumed to be a function definition.
'''
def recursive_ast_from_sexp(parent, sub_sexp):
if not sub_sexp:
return
elif sub_sexp.__class__.__name__ == 'list':
level_node = sub_sexp.pop(0)
level_label = 'NOT SYMBOL'
if level_node.__class__.__name__ == "Symbol":
level_label = level_node.value()
if Node.get_label(parent) == 'root':
level_label = 'FunctionDef: ' + level_label
new_node = Node(level_label)
parent.addkid(new_node)
for e in sub_sexp:
recursive_ast_from_sexp(new_node, e)
try:
exp = sexpdata.loads(s)
root = Node('root')
recursive_ast_from_sexp(root, exp)
return root
except Exception:
print s
def test_rules(self):
tags_rule_text = clean_sexp(
"""( ?* ^< (?*) ((ダミーA)) > * < (?*) ((^ダミーB)) >? ?* )""")
p = sexpdata.loads(tags_rule_text)
for order in [True, False]:
tagr = TagsRule(p, order)
self.assertEqual(True, len(str(tagr)) > 0)
def remove_bom_plugin_entry(kicad_config_path, name, re_flags=re.IGNORECASE):
# Remove a BOM plugin enttry to the Eeschema configuration file.
config = read_config_file(os.path.join(kicad_config_path, "eeschema"))
bom_plugins_raw = [p for p in config if p.startswith("bom_plugins")]
new_list = []
new_list.append(sexpdata.Symbol("plugins"))
changes = False
if len(bom_plugins_raw) == 1:
bom_plugins_raw = after(bom_plugins_raw[0], "bom_plugins=")
bom_plugins_raw = de_escape(bom_plugins_raw)
bom_list = sexpdata.loads(bom_plugins_raw)
for plugin in bom_list[1:]:
if re.findall(name, plugin[1], re_flags):
changes = True # The name in really in the 'name'.
continue # We want to delete this entry.
else:
for entry in plugin[2:]:
if entry[0]==sexpdata.Symbol('opts') and\
re.findall('nickname\s*=\s*'+name, entry[1], re_flags):
changes = True
continue # The name is in the 'nickname'.
new_list.append(plugin) # This plugin remains on the list.
if changes:
s = sexpdata.dumps(new_list)
config = update_config_file(config, "bom_plugins", escape(s))
write_config_file(os.path.join(kicad_config_path, "eeschema"), config)
def read_exec_time(filename):
if not os.path.exists(filename):
return None
assert isinstance(filename, str)
with open(filename, "r") as f:
sexp = sexpdata.loads(f.read())
m = sexp_to_map(sexp)
return geometric_mean([parse_time(t) for t in m["raw_execution_time"]])
def sp_decode(msg: str) -> dict:
"""
Decode the given message to JSON format
:param msg: message to convert
:return: JSON formatted message
"""
rtn = sexpdata.loads(msg)
return _sp_decode(rtn)
def run():
if len(sys.argv)>1:
with open('test.isp') as f:
for line in f: eval_(loads(line))
else:
print '''
--------------------
Ispy Interpreter v'''+str(VERSION)+'''
--------------------'''
while True:
sys.stdout.write("?> ")
input = raw_input()
input = input.replace('[', '(')
input = input.replace(']', ')')
if not input: break
if input.split()[0] == "'": continue
print(str(eval_(loads(input, true='true', false='false'))))
def filter_parser_output(parses, word2ids, sources, targets):
filtered = {}
if sources and targets and parses and word2ids:
for key in parses:
allfound = False
toKeep = set()
if key in sources and key in targets and key in word2ids:
allfound = True
wids = word2ids[key]
for w in sources[key].strip().split():
if w in wids:
toKeep.update(wids[w])
else:
allfound = False
break
for w in targets[key].strip().split():
if w in wids:
toKeep.update(wids[w])
else:
allfound = False
break
if allfound and toKeep:
filtered[key] = "(O (name " + str(key) + ") (^ "
o = sexpdata.loads(parses[key])
for p in o[2]:
if type(p) == list:
# print("p: " + str(p))
pidstring = sexpdata.dumps(p[-1])
# print("pidstring: " + pidstring)
result = idPattern.match(pidstring)
keep_this_one = True
if result:
keep_this_one = False
pids = result.group(1).split(",")
for pid in pids:
try:
pid = int(result.group(1)) \
if result else None
pid = remove_thousands(pid)
except ValueError:
pid = None
# print("pid number: " + str(pid))
if not pid or pid in toKeep:
keep_this_one = True
break
if keep_this_one:
filtered[key] += print_pred(p)
# print("filtered[key]: " + filtered[key])
filtered[key] += "))"
parser_output = ""
if parses:
for key in parses:
if key in filtered:
parser_output += filtered[key] + "\n"
else:
parser_output += parses[key] + "\n"
return parser_output
def probabilitiesFromFile(filename):
probs = []
file = open(filename, "r")
for line in file:
tokens = line.split("|")
given = tupleFromString(tokens[0])
frame = makeListOfListsHashable(loads(tokens[1]))
prob = float(tokens[2])
probs.append((given, frame, prob))
return probs
def parse(source):
t = sx.loads(source)
funcs = {}
main = []
for l in t:
if l[0] == sx.Symbol("define"):
func(funcs, l)
else:
main.append(translate_expr(l))
return funcs, main
def get_dataset(dataset_file):
"""
Return data in the open dataset_file as a list of (query,
target_value) tuples.
"""
examples = json.load(dataset_file)
dataset = []
for example in examples:
query = example['utterance']
target_data = sexpdata.loads(example['targetValue'])
targets = [symbol_to_string(description[1]) for description in target_data[1:]]
dataset.append((query, targets))
return dataset
def get_new_output(self, string, ntok):
ptr = 1
a = string.split()
finalstring = ''
for i in a:
fl1 = i.find('(')
if fl1 != -1:
#then there it has to be a tag
finalstring += i[0] + '"' + i[1:] + '" '
else:
#there has to be an ending tag, which means there has to be a token
fl1 = i.find(')')
self.hm[ptr] = i[:fl1]
finalstring += '"' + unicode(ptr) + '"' + i[fl1:] + ' '
ptr += 1
return sexpdata.loads(finalstring)
def lineReceived(self, line):
self.log.debug("RCVD::" + line)
try:
rdata = loads(line)
except Exception:
self.error(10, "Invalid syntax")
else:
if len(rdata) == 0:
self.error(12, "Empty set")
return
if type(rdata[0]) != Symbol:
self.error(13, "First element is not a symbol")
return
if rdata[0].value() in self.handlers:
self.handlers[rdata[0].value()](rdata[1:])
else:
self.error(11, "Unrecognized command")
def parse(path):
"""Parse an ``.ensime`` config file from S-expressions.
Args:
path (str): Path of an ``.ensime`` file to parse.
Returns:
dict: Configuration values with string keys.
"""
def paired(iterable):
"""s -> (s0, s1), (s2, s3), (s4, s5), ..."""
cursor = iter(iterable)
return zip(cursor, cursor)
def unwrap_if_sexp_symbol(datum):
"""Convert Symbol(':key') to ':key' (Symbol isn't hashable for dict keys).
"""
return datum.value() if isinstance(datum, sexpdata.Symbol) else datum
def sexp2dict(sexps):
"""Transforms a nested list structure from sexpdata to dict."""
newdict = {}
# Turn flat list into associative pairs
for key, value in paired(sexps):
key = str(unwrap_if_sexp_symbol(key)).lstrip(':')
# Recursively transform nested lists
if isinstance(value, list) and value:
if isinstance(value[0], list):
newdict[key] = [sexp2dict(val) for val in value]
elif isinstance(value[0], sexpdata.Symbol):
newdict[key] = sexp2dict(value)
else:
newdict[key] = value
else:
newdict[key] = value
return newdict
conf = sexpdata.loads(Util.read_file(path))
return sexp2dict(conf)
def main(src):
with open(src) as f:
data = '(' + f.read() + ')'
sexp = sexpdata.loads(data)
constrs = []
for s in sexp:
if s[0].value() == u'declare-variable':
name = s[1].value()
t = s[2]
if isinstance(t, Symbol):
if t.value() == u'String':
tp = str
elif t.value() == u'Int':
tp = int
elif isinstance(t, list) and t[0].value() == u'_' \
and t[1].value() == u'BitVec':
assert t[2] == 32
tp = int
else:
assert False, t
varmap[name] = tp
elif s[0].value() == u'assert':
c, _ = convert(s[1])
constrs.append(c)
pr = sys.stdout.write
pr('#include <assert.h>\n')
pr('#include "../../cprover-string-hack.h"\n\n')
pr('int main()\n{\n')
for name in sorted(varmap):
if varmap[name] is not None:
pr(' %s %s;\n' % (to_type(varmap[name]), name))
pr('\n')
pr(' if (%s) {\n assert(0);\n }\n return 0;\n}\n' %
'\n && '.join(constrs))
def handle(self, *args, **options):
if os.access('/tmp/expected_probe_missing_check.lock', os.R_OK):
timestamp = os.path.getmtime('/tmp/expected_probe_missing_check.lock')
created = datetime.datetime.fromtimestamp(timestamp)
if (datetime.datetime.now() - created).total_seconds() > 60 * 60 * 4:
print 'expected_probe_missing_check: Stale lock - removing...'
os.remove('/tmp/expected_probe_missing_check.lock')
else:
return
touch('/tmp/expected_probe_missing_check.lock')
start = timezone.now() - datetime.timedelta(days=START_DAYS)
for device in PurpleRobotDevice.objects.filter(mute_alerts=False).order_by('device_id'):
model = device.last_model()
mfgr = device.last_manufacturer()
config = None
default = PurpleRobotConfiguration.objects.filter(slug='default').first()
if device.configuration is not None:
config = device.configuration
elif device.device_group is not None and device.device_group.configuration is not None:
config = device.device_group.configuration
elif config is None:
config = default
if config is None:
log_alert(message='No configuration associated with ' + device.device_id + '.', severity=2, tags=TAG, user_id=device.hash_key)
else:
config_probes = enabled_probes(loads(config.contents, true='#t', false='#f'))
touch('/tmp/expected_probe_missing_check.lock')
missing_probes = []
for probe in config_probes:
if can_sense(mfgr, model, probe):
found = device.most_recent_reading(probe)
if found is None or found.logged < start:
missing_probes.append(probe.split('.')[-1])
platform = device.last_platform()
if platform is not None and platform.startswith('Android 5'):
if 'ApplicationLaunchProbe' in missing_probes:
missing_probes.remove('ApplicationLaunchProbe')
if 'RunningSoftwareProbe' in missing_probes:
missing_probes.remove('RunningSoftwareProbe')
if len(missing_probes) == 0:
cancel_alert(tags=TAG, user_id=device.hash_key)
else:
missing_probes_str = ', '.join(missing_probes[:4])
if len(missing_probes) > 4:
missing_probes_str = missing_probes_str + ', and ' + str(len(missing_probes) - 4) + ' more'
log_alert(message='Missing data from ' + str(len(missing_probes)) + ' probe(s). Absent probes: ' + missing_probes_str, severity=2, tags=TAG, user_id=device.hash_key)
os.remove('/tmp/expected_probe_missing_check.lock')
for child in connections[id]:
lines += buildConnectionsInCorrectOrder(connections, child)
return lines
# create a DOT file for each sentence parse
for file in glob("parsed_annotated_recipes/*.xml"):
try:
dom = parse(file)
except:
print "The file %s had some parsing error." % file
lineCount = 0
for elt in dom.getElementsByTagName("line"):
try:
parsedSentence = elt.getElementsByTagName("parsed-text")[0].childNodes[0].data
command = elt.getElementsByTagName("annotation")[0].childNodes[0].data
sexp = loads(parsedSentence)
labels = {}
connections = defaultdict(list)
buildGraphModel(sexp, labels, connections)
graphString = buildGraphString(labels, connections, command)
filename = join(join(join(dirname(abspath(__file__)), "syntax_trees"), "dot_files"), basename(file).replace(".xml", "") + "_" + str(lineCount) + ".dot")
if not exists(dirname(filename)):
makedirs(dirname(filename))
output = open(filename, "w")
output.write(graphString)
output.close()
idCount = -1
lineCount += 1
except ExpectClosingBracket as e:
pass
# print "There seem to be too few parens."
def loads(s, **kwargs):
data = sexpdata.loads(s, **kwargs)
return _dictify(data)
def get_sexp(f):
s = "(" + open(f).read() + ")"
s_exp = loads(s)
return s_exp
def receive_message(self):
result = self.client.recv(1024)
self.assertEqual(int(result[:6], 16), len(result[6:]))
return loads(result[6:].decode()) # skip the length part
topLevelDict[topLevelFrame] = 1
else:
topLevelCounts[givenKey] = {topLevelFrame: 1}
parseCommandPairMappings = getParseCommandPairMappingsForTrain()
for filename in parseCommandPairMappings:
pairs = parseCommandPairMappings[filename]
prevPrevIngredients = []
prevIngredients = []
prevPrevTools = []
prevTools = []
for pair in pairs:
command = MILK_parse_command(pair[1])
if command[0] not in ["create_ing", "create_tool"]:
try:
sexp = loads(pair[0])
vps = getVps(sexp)
vpVbPair = getMostLikelyVpVbPair(vps, command[0])
if vpVbPair is not None:
vp, vb = vpVbPair
caseFrame = makeListOfListsHashable(getCaseFrame(vp))
caseFrame = removeWithPredicate(caseFrame, ["PP"], lambda n: n[1][1] in ["until", "for", "at", "by"])
caseFrame = removeWithPredicate(caseFrame, ["CC", "ADVP"], lambda n: True)
inputIngredients = getInputIngredients(command[0], command[1])
tools = getTools(command[0], command[1])
assert(all(re.compile("ing[0-9]+").match(i) for i in inputIngredients))
assert(all(re.compile("t[0-9]+").match(t) for t in tools))
ingredientsPreviouslyUsed = len((set(prevIngredients).union(set(prevPrevIngredients))).intersection(set(inputIngredients))) > 0
toolsPreviouslyUsed = len((set(prevTools).union(set(prevPrevTools))).intersection(set(tools))) > 0
prevPrevIngredients = prevIngredients
prevIngredients = getOutputIngredients(command[0], command[1])
def __init__(self, exp):
self.expression = sexpdata.loads(exp)
def read(self, string):
"""Parse and return a single expression from ``string``."""
return sexpdata.loads(string)
def text_frontend(inp):
return sexp_eval(loads(inp))
return symbol
def get_cache_key(query, connection):
return '___'.join([query, '|'.join(connection)])
if __name__ == "__main__":
connector = Connector()
# justin = 'fb:en.justin_bieber'
# jaxon = 'fb:m.0gxnnwq'
data_file = open(settings.DATASET_PATH)
examples = json.load(data_file)
f1_scores = []
related_entities = RelatedEntities()
for example in examples[100:200]:
query = example['utterance']
target_data = sexpdata.loads(example['targetValue'])
targets = [symbol_to_string(description[1]) for description in target_data[1:]]
best_score = 0.0
for target in targets:
print query, target
result_ids = connector.search(query, target)
print result_ids
results = [related_entities.get_names(e[0]) for e in result_ids]
print results
score = get_f1_score(targets, results)
if score > best_score:
best_score = score
f1_scores.append(best_score)
print "F1 scores:", f1_scores
def unpack_message(bytes):
data = loads(bytes.decode('utf-8'))
return (data[0].value(), data[1], data[2:])
def parse_tree(p):
if "'" in p:
p = "none"
parsed = sexpdata.loads(p)
extracted = extract_parse(parsed)
return extracted
return parse_fun(value_of(elem.value()), s[1])
def parse_fun(function, args):
global vars
tempvars = vars
funvars = {function.args[i]: parse(args[i]) for i in range(len(args))}
vars = funvars
ret = parse(function.func)
vars = tempvars
return ret
while line != 'quit':
if len(sys.argv) > 1:
f = open(sys.argv[1])
program = ''
for line in f.read().split('\n'):
if len(line) > 0 and line[0] != ';':
program += line
sexpr = sex.loads(program)
parse(sexpr)
exit()
line = raw_input('>>')
if line == '!!':
line = last
sexpr = sex.loads(line)
print parse(sexpr)
last = line
# print entry
if type(entry) is dict:
ret_dict.update(entry)
else:
ret_dict.update({"value": entry})
# print ("something is wrong here: {}".format(entry))
return {k.value(): ret_dict}
elif type(data) is Symbol:
return data.value()
else:
return data
if len(sys.argv) != 2:
print("usage: {} filename.net".format(sys.argv[0]))
exit (-1)
fname = sys.argv[1]
data = loads(open(fname).read())
import json
d = to_dict(data)
print json.dumps(d,
sort_keys=False,
indent=4,
separators=(',', ': '))
# print_components(d["export"]["components"])
80: "gis''",
81: "a''",
82: "ais''",
83: "b''",
84: "c'''",
85: "cis'''",
86: "d'''"}
times = ["_","16","8","8.","4","_","4.","_","2","_","_","_","2.","_","_","_","1"]
data = open("choralesmod.lisp", 'r').read().split('&')
notes = ""
for d in data:
d = loads(d)
#transpose the note to keysign 0 (=c) with the lowest possible pitch change
for line in d[1:]:
keysign = line[3][1]
pitch = line[1][1]
if abs((keysign*7)%12) <= 6:
pitch = pitch - ((keysign*7)%12)
else:
pitch = pitch + ((keysign*5)%12)
notes = notes + pitches[pitch] + times[line[2][1]]+' '
output = open("chorales.txt", 'w')
output.write(notes)
def importpddl(pddl):
data = sexpdata.loads(pddl)
stripeddata = stripsexp(data)
return stripeddata
