def extractData(self):
self.training_data, self.training_outputs = parsing.parse(self.training_path, self.input_start_column, self.input_end_column, qualitative = self.qualitative_outputs, output_column = self.output_column)
self.testing_data, self.testing_outputs = parsing.parse(self.testing_path, self.input_start_column, self.input_end_column, qualitative = self.qualitative_outputs, output_column = self.output_column)
self.input_start_column = 0
self.input_end_column = len(self.training_data[0]) - 1
return 1
if self.qualitative_outputs:
if self.testing_path == None:
full_training = np.genfromtxt(self.training_path, delimiter=self.delimitation)
full_size = len(full_training)
random.shuffle(full_training)
self.training_data = full_training[:full_size-self.number_of_tests]
self.testing_data = full_training[full_size-self.number_of_tests:]
else:
self.training_data = np.genfromtxt(self.training_path, delimiter=self.delimitation)
self.testing_data = np.genfromtxt(self.testing_path, delimiter=self.delimitation)
max_output = max(self.getMaxOutput(self.training_data), self.getMaxOutput(self.testing_data))
self.training_outputs, self.training_outputs = self.getQualitativeOutputs(self.training_data, max_output)
self.testing_outputs, self.testing_outputs = self.getQualitativeOutputs(self.testing_data, max_output)
self.training_data = self.training_data[:,[x for x in xrange(self.input_start_column, self.input_end_column+1)]]
self.testing_data = self.testing_data[:,[x for x in xrange(self.input_start_column,self.input_end_column+1)]]
else:
self.training_data = np.genfromtxt(self.training_path, delimiter=self.delimitation)
self.testing_data = np.genfromtxt(self.testing_path, delimiter=self.delimitation)
self.training_outputs = self.training_data[:,[self.output_column]]
self.testing_outputs = self.testing_data[:,[self.output_column]]
self.training_data = self.training_data[:,[x for x in xrange(self.input_start_column, self.input_end_column+1)]]
self.testing_data = self.testing_data[:,[x for x in xrange(self.input_start_column,self.input_end_column+1)]]
def file_chooser():
title = "seleeect"
filetypes = (('Audit files', '*.audit'), ('all files', '*.*'))
global fname
fname = filedialog.askopenfilename(initialdir='.',
title=title,
filetypes=filetypes)
file_name = fname.split('/')[-1]
parse(fname)
update_label(file_name)
def main(argv=None):
try:
if argv is None:
argv = sys.argv[1:]
parser = _mk_options_parser()
options, args = parser.parse_args(argv)
if len(args) == 0:
parser.error("insufficient arguments, expected at least one path.")
return 2
res = parsing.parse(args, options.prune, options.mintime)
if options.interactive or options.output == None:
gui.show(res)
else:
filename = _get_filename(options.output)
res_list = parsing.split_res(res, options.num)
n = 1
for r in res_list:
if len(res_list) == 1:
f = filename + "." + options.format
else:
f = filename + "_" + str(n) + "." + options.format
n = n + 1
batch.render(r, options.format, f)
print "bootchart written to", f
return 0
except parsing.ParseError, ex:
print("Parse error: %s" % ex)
return 2
def get_tests(self, the_makefile, the_dict):
with open(the_makefile) as f:
input = f.read()
output = parse(input)
apps = [each for each in output if 'Command' in str(type(each))]
otb_tests = [each for each in apps if 'ADD_TEST' in each.name.upper()]
return otb_tests
def get_tests(self, the_makefile, the_dict):
with open(the_makefile) as f:
input = f.read()
output = parse(input)
apps = [each for each in output if 'Command' in str(type(each))]
otb_tests = [each for each in apps if 'ADD_TEST' in each.name.upper()]
return otb_tests
def parseData(ticker):
modelsFile = open('models.pkl', 'rb')
m0, m1 = pickle.load(modelsFile)
F, data = dataXY(ticker)
ptable = likTable(F, m0, m1)
res = parse(ptable)
kps = data[data.pivots!=0].Close
idx = kps.index
fig = plt.figure(figsize=(15,10))
ax = fig.add_subplot(211)
bx = fig.add_subplot(212)
kps.plot(style='--o', ax=ax)
ptable.plot(style='o', ax=bx)
#import ipdb; ipdb.set_trace()
for terminal, (xmin, xmax), _, logLik in res:
center = idx[(xmin+xmax)/2]
high = data[idx[xmin]:idx[xmax]].Close.max()
low = data[idx[xmin]:idx[xmax]].Close.min()
bottom, top = ax.get_ylim()
llow = (low-bottom)/float(top-bottom)
hhigh = (high-bottom)/float(top-bottom)
ax.axvspan(idx[xmin], idx[xmax], ymin=llow, ymax=hhigh, alpha=0.3)
ax.annotate("%s"%(terminal), (center, high))
#ax.annotate("%s[%.3f]"%(terminal,logLik), (center, high))
bx.legend(loc=4)
plt.savefig("parsed/%s.png"%(ticker))
def main():
# Parse arguments
parser = argparse.ArgumentParser(
description='Pretty-print CMakeLists files.')
parser.add_argument('files',
type=str,
nargs='*',
help='files to pretty print (default is stdin)')
parser.add_argument('-t',
'--tree',
action='store_true',
help='print out the syntax trees')
args = parser.parse_args()
# Gather files
filenames = args.files
files = [('<stdin>', sys.stdin)]
if filenames:
files = [(name, open(name)) for name in filenames]
# Process files
for (name, file) in files:
with file:
input = file.read()
tree = cmp.parse(input, path=name)
if args.tree:
# Print out AST
print(repr(tree))
else:
# Pretty print
print(str(tree), end='')
def main():
parser = argparse.ArgumentParser(description="Halite II training")
parser.add_argument("--model_name", help="Name of the model")
parser.add_argument("--minibatch_size", type=int, help="Size of the minibatch", default=100)
parser.add_argument("--steps", type=int, help="Number of steps in the training", default=100)
parser.add_argument("--data", help="Data directory or zip file containing uncompressed games")
parser.add_argument("--cache", help="Location of the model we should continue to train")
parser.add_argument("--games_limit", type=int, help="Train on up to games_limit games", default=1000)
parser.add_argument("--seed", type=int, help="Random seed to make the training deterministic")
parser.add_argument("--bot_to_imitate", help="Name of the bot whose strategy we want to learn")
parser.add_argument("--dump_features_location", help="Location of hdf file where the features should be stored")
args = parser.parse_args()
# Make deterministic if needed
if args.seed is not None:
np.random.seed(args.seed)
nn = NeuralNet(cached_model=args.cache, seed=args.seed)
if args.data.endswith('.zip'):
raw_data = fetch_data_zip(args.data, args.games_limit)
else:
raw_data = fetch_data_dir(args.data, args.games_limit)
data_input, data_output = parse(raw_data, args.bot_to_imitate, args.dump_features_location)
data_size = len(data_input)
training_input, training_output = data_input[:int(0.85 * data_size)], data_output[:int(0.85 * data_size)]
validation_input, validation_output = data_input[int(0.85 * data_size):], data_output[int(0.85 * data_size):]
training_data_size = len(training_input)
# randomly permute the data
permutation = np.random.permutation(training_data_size)
training_input, training_output = training_input[permutation], training_output[permutation]
print("Initial, cross validation loss: {}".format(nn.compute_loss(validation_input, validation_output)))
curves = []
for s in range(args.steps):
start = (s * args.minibatch_size) % training_data_size
end = start + args.minibatch_size
training_loss = nn.fit(training_input[start:end], training_output[start:end])
if s % 25 == 0 or s == args.steps - 1:
validation_loss = nn.compute_loss(validation_input, validation_output)
print("Step: {}, cross validation loss: {}, training_loss: {}".format(s, validation_loss, training_loss))
curves.append((s, training_loss, validation_loss))
cf = pd.DataFrame(curves, columns=['step', 'training_loss', 'cv_loss'])
fig = cf.plot(x='step', y=['training_loss', 'cv_loss']).get_figure()
# Save the trained model, so it can be used by the bot
current_directory = os.path.dirname(os.path.abspath(__file__))
model_path = os.path.join(current_directory, os.path.pardir, "models", args.model_name + ".ckpt")
print("Training finished, serializing model to {}".format(model_path))
nn.save(model_path)
print("Model serialized")
curve_path = os.path.join(current_directory, os.path.pardir, "models", args.model_name + "_training_plot.png")
fig.savefig(curve_path)
def compute_predictions(rules,
non_terminals,
unary_rules_proba,
binary_rules_proba,
transition_rules_list,
vocabulary,
embeddings,
test_corpus,
filename='evaluation_data.parser_output'):
"""
Creates prediction file from test corpus (for pyevalb)
"""
n_test = len(test_corpus)
with open(filename, 'w') as f:
for i, elem in enumerate(test_corpus):
print('{} elements have been processed out of {}'.format(
i + 1, n_test))
parsed = parsing.parse(rules,
non_terminals,
unary_rules_proba,
binary_rules_proba,
transition_rules_list,
vocabulary,
embeddings,
tree=elem)
f.write("%s\n" % parsed)
def ep(input, debug=True):
ast = parse(input)
if debug:
print ast
val = evaluate(ast)
if val is not None and debug:
print(_to_scheme_str(val))
def get_apps(self, the_makefile, the_dict):
with open(the_makefile) as f:
input = f.read()
output = parse(input)
apps = [each for each in output if "Command" in str(type(each))]
otb_apps = [each for each in apps if "OTB_TEST_APPLICATION" in each.name.upper()]
return otb_apps
def analyze_macro_application(astnode):
exp = astnode.exp
macro, macro_args = exp[0].exp, astnode.get_exp()
expanded = macro_expand(macro, macro_args)
tostring = str(to_string(expanded))
parsed = analyze(list(parse(tostring))[-1])
return CodeObject(astnode, lambda env: parsed.exec_(env))
def fetch():
display()
parsing.parse()
parsing.new_parse()
T.config(state="normal")
try:
mac_file = open('mac_address.txt', 'r')
T.delete(1.0, END)
T.insert(INSERT, "MAC file found\n\n")
mac_addr = mac_file.readlines()
for mac in mac_addr:
fileMenu.add_command(label=mac.rstrip('\n'), command=partial(sta_data, mac.rstrip('\n')))
except IOError:
T.insert(INSERT, "MAC file not found\n\n")
T.config(state="disabled")
def valid_proof(expressions):
"""
Takes in a list of parse tree expressions [A, B, C, ... Z].
Outputs one parse tree ((A&B&C&...) -> Z).
Last expression is the conclusion, all others are premises.
"""
trees = [parse(e) for e in expressions]
argument = serialize_argument_trees(trees)
return all([row.value for row in truthtable.from_tree(argument)])
def get_apps(self, the_makefile, the_dict):
input = open(the_makefile).read()
output = parse(input)
apps = [each for each in output if 'Command' in unicode(type(each))]
otb_apps = [
each for each in apps
if 'OTB_TEST_APPLICATION' in each.name.upper()
]
return otb_apps
def valid_proof(expressions):
"""
Takes in a list of parse tree expressions [A, B, C, ... Z].
Outputs one parse tree ((A&B&C&...) -> Z).
Last expression is the conclusion, all others are premises.
"""
trees = [parse(e) for e in expressions]
argument = serialize_argument_trees(trees)
return all([row.value for row in truthtable.from_tree(argument)])
def main(argv=None):
try:
if argv is None:
argv = sys.argv[1:]
parser = _mk_options_parser()
options, args = parser.parse_args(argv)
writer = _mk_writer(options)
if len(args) == 0:
print "No path given, trying /var/log/bootchart.tgz"
args = [ "/var/log/bootchart.tgz" ]
res = parsing.parse(writer, args, options.prune,
options.crop_after, options.annotate)
if options.interactive or options.output == None:
gui.show(res, options)
elif options.boottime:
import math
proc_tree = res[3]
if proc_tree.idle:
duration = proc_tree.idle
else:
duration = proc_tree.duration
dur = duration / 100.0
print '%02d:%05.2f' % (math.floor(dur/60), dur - 60 * math.floor(dur/60))
else:
if options.annotate_file:
f = open (options.annotate_file, "w")
try:
for time in res[4]:
if time is not None:
# output as ms
print >> f, time * 10
else:
print >> f
finally:
f.close()
filename = _get_filename(args, options)
def render():
batch.render(writer, res, options, filename)
if options.profile:
import cProfile
import pstats
profile = '%s.prof' % os.path.splitext(filename)[0]
cProfile.runctx('render()', globals(), locals(), profile)
p = pstats.Stats(profile)
p.strip_dirs().sort_stats('time').print_stats(20)
else:
render()
return 0
except parsing.ParseError, ex:
print("Parse error: %s" % ex)
return 2
def assertParseContains(self, location, contains):
# Because the parser is overly greedy and gives many possible
# responses, it keeps the unit tests tidier and less brittle if we
# just list one important parse result that we expect, rather than
# listing every single test result.
try:
actual = [dict(result) for result in parse(location)]
except ParsingError, e:
self.fail(e)
def intro():
answer = askForever(phrasebase.getReply("greetings"))
meaning = parsing.parse(answer)
if "greet" in meaning:
ui.tell("Nice to meet you!")
answer = askForever("You are a human, aren't you?")
meaning = parsing.parse(answer)
if meaning == "yes":
ui.tell("Great! This means you can help me.")
if meaning == "no":
answer = askForever("Are you sure? What is the sum of two and three?")
meaning = parsing.parse(answer)
if meaning == "five":
ui.tell("See, you solved the captcha. You are sufficiently human for my purposes.")
else:
ui.tell("Very funny. You are definitely human.")
ui.tell("My programmers want me to teach you how to be rational, make good decisions and judge situations correctly.")
askForever("Do you want to be more rational?")
meaning = parsing.parse(answer)
if meaning == "yes":
ui.tell("Yeah, that's the spirit!")
if meaning == "no":
answer = askForever("Why would you think that? Rationality is just the ability to make good decisions. Do you want to be able to make good decisions?")
meaning = parsing.parse(answer)
if meaning == "no":
askbreak()
ui.tell("I will just try to ask you some questions, and try to explain to you what you could do better. If I do a bad job at explaning, just ask me, ok? I never taught humans before.")
ui.tell("So, let's see... the first thing I want you to know is that you don't have to be extremely intelligent to be rational. There are very intelligent people who do things that are not at all reasonable. The key to rational decisions is to know when not to follow your gut feelings, but to stop and actually think about the problem.");
answer = askForever("To get used to the whole situation - how about I ask you a test question? Just to make sure I am doing this teaching thing right. ")
meaning = parsing.parse(answer)
if meaning == "yes":
ui.tell("Okay, thank you!")
if meaning == "no":
ui.tell("I would nevertheless like to ask the test question.")
answer = askForever("This is my first question: Do people need to follow their gut feelings to make rational decisions?")
meaning = parsing.parse(answer)
if meaning == "no":
ui.tell("Amazing! I mean, it was easy, I know, but you did it. Very reasonable of you to say this! Now we can start with the actual teaching.")
if meaning == "yes":
ui.tell("Uhm... no. This is a bit awkward. Following you gut feelings means not to think about something, but just go with what feels right. A lot of psychologists have shown that people tend to make a lot of mistakes when they make decisions that way.")
answer = askForever("Do you still want to continue?")
meaning = parsing.parse(answer)
if meaning == "yes":
ui.tell("Okay! Let's start with the actual teaching!")
else :
askbreak()
def generate_operators():
repo_names = getRepoNames()
new_operators = []
output = open('mutation_operators.txt','w')
init.init()
for repo in repo_names:
print("collecting operators form repo : " + repo + " .....")
diff_file = open('../resources/diffs_modified/'+repo+'_diff_modified','r')
minus = None
plus = None
for line in diff_file:
if minus is None:
minus = line.strip()
elif plus is None:
plus = line.strip()
if plus is not None:
#parse both minus and plus
if not (("'''" in minus) or ("'''" in plus) or ('"""' in minus) or ('"""' in plus)):
token_minus = parse(minus[1:])
token_plus = parse(plus[1:])
new_operator = compareTok(token_minus, token_plus)
if (new_operator is not None):
if (new_operator in new_operators):
init.duplicate += 1
else:
output.write(str(new_operator)+ "\n")
new_operators.append(new_operator)
minus = None
plus = None
print("size : " + str(len(new_operators)) );
print("collecting done")
#return new_operators
print("less and equal Than 2 : " + str(init.lessThan2))
print("more and equal than 10 tokens : " + str(init.moreThan10))
print("same afterToken, beforeToken : " + str(init.same))
print("more and equal than 4 identifiers : " + str(init.moreThan4Iden))
print("3 identifers in a row : " + str(init.inArow))
print("unmatched bracket : " + str(init.unmatchedBracket))
print("duplicate : " + str(init.duplicate))
output.close()
def main():
update_id = last_update(get_updates_json(url))['update_id']
while True:
if update_id == last_update(get_updates_json(url))['update_id']:
update_id += 1
if last_update(get_updates_json(url))['message']['text'] == 'bet':
result = parse()
send_mess(get_chat_id(last_update(get_updates_json(url))),
result)
else:
send_mess(get_chat_id(last_update(get_updates_json(url))),
'test')
result = parse()
if (result != "NOW NOTHING"):
send_mess(get_chat_id(last_update(get_updates_json(url))), result)
time.sleep(3)
def startliza:
parsing.setui(ui)
warnings.filterwarnings(module='sklearn*', action='ignore', category=DeprecationWarning)
while(True):
ui.tell("Say something. I will try to understand and parse it.")
answer = ui.listen()
parsed = parsing.parse(answer)
analysis = str(parsed['intent']['name']) + ", with confidence " + str(parsed['intent']['confidence'])
ui.analyze(analysis)
def eval_string(inport):
"Eval string or inport"
x = parse(inport)
if x is eof_object:
return x
val = lisp_eval(x)
if val is None:
return None
return to_string(val)
def __init__(self, **kwargs):
total_start = datetime.now()
test = kwargs['filename'] or input('\nEnter test name: ')
test_rating = parse(os.path.join(TEST_DIR, test + '.test'))
base_rating = parse(os.path.join(TEST_DIR, test + '.base'))
graph = Graph(base_rating=base_rating)
while True:
try:
cluster_number = kwargs['cluster_num'] or int(input('\nEnter number of clusters: '))
component = BoruvkaMST(graph, cluster_number)
break
except (ClusterNumberException, ValueError):
print('Number of clusters must be > 0')
continue
predictor = Predictor(component, base_rating)
from copy import deepcopy
predicted = deepcopy(test_rating)
unpredictables = 0
start = datetime.now()
with open(os.path.join(TEST_DIR, test + '.res'), newline='', mode='w') as csv_file:
data_writer = csv.writer(csv_file, delimiter=' ', quotechar='|')
for user in test_rating.keys():
for movie in test_rating[user].keys():
predicted[user][movie] = None
predicted[user][movie] = predictor.predict(user, movie)
if not predicted[user][movie]:
unpredictables += 1
data_writer.writerow([user, movie, predicted[user][movie]])
self.predicting_time = datetime.now() - start
print('Time for predicting:', self.predicting_time, end='\n\n')
self.total_time = datetime.now() - total_start
print('Total time:', self.total_time)
print('Number of unpredictable users:', unpredictables)
self.evaluator = Evaluator(test_rating, predicted)
def eval(_str, env=THE_GLOBAL_ENV):
"""
Loispy eval (overwrites python eval)
Parse a string, analyze all the expressions in the context of a given
environment, and return the last resulting value
@param str _str
@param Environment env
"""
return [analyze(e, toplevel=True).exec_(env) for e in parse(_str)][-1]
def test_serialize_argtree_simple():
expressions = ["A", "(A>B)", "B"]
trees = [parse(e) for e in expressions]
argument = serialize_argument_trees(trees)
assert_is_instance(argument, IfNode)
assert_is_instance(argument.l, AndNode)
assert_is_instance(argument.r, AtomNode)
assert_is_instance(argument.l.l, AtomNode)
assert_is_instance(argument.l.r, IfNode)
assert_is_instance(argument.l.r.l, AtomNode)
assert_is_instance(argument.l.r.r, AtomNode)
def test_serialize_argtree_simple():
expressions = ["A", "(A>B)", "B"]
trees = [parse(e) for e in expressions]
argument = serialize_argument_trees(trees)
assert_is_instance(argument, IfNode)
assert_is_instance(argument.l, AndNode)
assert_is_instance(argument.r, AtomNode)
assert_is_instance(argument.l.l, AtomNode)
assert_is_instance(argument.l.r, IfNode)
assert_is_instance(argument.l.r.l, AtomNode)
assert_is_instance(argument.l.r.r, AtomNode)
def print_truth_table(exp, verbose, output=True):
"""Outputs the truth table for an expression to stdout."""
try:
tree = parsing.parse(exp)
table = truth_table(exp)
except IOError as e:
print("Parse error: %s" % e)
return
for row in table:
print(row)
if verbose:
print()
table = truth_table(exp)
print_sat_info(table)
def print_truth_table(exp, verbose, output=True):
"""Outputs the truth table for an expression to stdout."""
try:
tree = parsing.parse(exp)
table = truth_table(exp)
except IOError as e:
print("Parse error: %s" % e)
return
for row in table:
print(row)
if verbose:
print()
table = truth_table(exp)
print_sat_info(table)
def __init__(self, report, selector):
self.report_number = report
self.original_report_number = report
self.year = selector.report_list['year'][1]
self.parsed = parse() #for parsing inputs from rows
self.connection = connect() #connects to the DB
self.queries = queries() #query text for all tables
self.agg = agg() #aggregation functions for all tables
self.json_builder = self.JSON_constructor_return()
self.parse_return_string = self.parse_return()
self.aggregation = self.aggregation_return(self.year, report)
self.id_compiler = id_compiler()
self.report_types = {'A':'Aggregate', 'D':'Disclosure', 'N':'National'}
def __init__(self, report, selector):
self.report_number = report
self.original_report_number = report
self.year = selector.report_list['year'][1]
self.parsed = parse() #for parsing inputs from rows
self.connection = connect() #connects to the DB
self.queries = queries() #query text for all tables
self.agg = agg() #aggregation functions for all tables
self.json_builder_return = self.JSON_constructor_return()
self.parse_return_string = self.parse_return()
self.aggregation = self.aggregation_return(self.year, report)
self.id_compiler = id_compiler()
self.report_types = {'A':'Aggregate', 'D':'Disclosure', 'N':'National'}
def run(local, remote, merged, conflict_resolution):
print("TSCN-Merge")
print(f"local/MINE: {local}")
print(f"remote/THEIRS: {remote}")
mine: TscnFile = parse(local)
theirs: TscnFile = parse(remote)
result: TscnFile = TscnFile()
result.gd_scene = mine.gd_scene
resolver = interactive
if conflict_resolution == "mine":
resolver = always_mine
if conflict_resolution == "theirs":
resolver = always_theirs
# Step 1: Merge external resources and consolidate IDs.
merge_ext_resources(mine, theirs, result)
# Step 2: Merge sub resources
merge_sub_resources(mine, theirs, result, resolver)
# Step 3: Merge the node trees
merge_nodes(mine, theirs, result, resolver)
# Step 4: Merge connections
merge_connections(mine, theirs, result)
if not merged:
print(result.to_string())
else:
absolute_path = os.path.abspath(merged)
directory, file_name = os.path.split(absolute_path)
os.makedirs(directory, exist_ok=True)
out_file = open(file_name, "w")
out_file.write(result.to_string())
out_file.close()
print(f"Result written to {absolute_path}.")
def askbreak():
ui.tell("Do you want to continue with this?")
answer = ui.listen()
if (answer == ""):
ui.tell("Do you want to continue? If you don't reply, I have to assume that you are gone...")
answer = ui.listen()
if (answer == ""):
ui.tell("Well, okay... I guess we can't continue then.")
return False
meaning = parsing.parse(answer)
if meaning == "yes":
ui.tell("Okay, great! So let's return to the questions.")
return True
ui.tell("Oh. I am sorry.")
return False
def receive(self, echo = False):
""" Receive messages """
msg=None
while msg == None:
try:
pay = self.__sock.recv(1024)
self.__buffer+=pay
if not pay:
break
except socket.error,e:
break
self.__buffer, msg = p.parse(self.__buffer)
def main():
new_offset = None
today = now.day
hour = now.hour
while True:
# greet_bot.get_updates(new_offset)
last_update = greet_bot.get_last_update()
# last_update_id = last_update['update_id']
# last_chat_text = last_update['message']['text']
last_chat_id = last_update['message']['chat']['id']
# last_chat_name = last_update['message']['chat']['first_name']
result = parse()
greet_bot.send_message(last_chat_id, result)
def add_make(self, previous_context, new_file):
with open(new_file) as f:
input = f.read()
output = parse(input)
apps = [each for each in output if 'Command' in str(type(each))]
setcommands = [each for each in apps if 'SET' in each.name.upper()]
stringcommands = [
each for each in apps if 'STRING' in each.name.upper()
]
environment = previous_context
def mini_clean(item):
if item.startswith('"') and item.endswith('"') and " " not in item:
return item[1:-1]
return item
new_env = {}
for command in setcommands:
key = command.body[0].contents
ct = " ".join([item.contents for item in command.body[1:]])
ct = mini_clean(ct)
if "$" in ct:
values = Template(ct)
else:
values = ct
new_env[key] = values
for stringcommand in stringcommands:
key = stringcommand.body[-1].contents
ct = stringcommand.body[-2].contents
ct = mini_clean(ct.lower())
if "$" in ct:
values = LowerTemplate(ct)
else:
values = ct
new_env[key] = values
resolve_dict(environment, new_env)
environment.update(new_env)
return environment
def __init__(self, expression):
# Parse any raw string expressions.
assert type(expression) is not type(self)
if type(expression) is str:
expression = parse(expression)
# Use immutable data structures for speed & safety.
if type(expression) is list:
expression = tuple(expression)
self._expression = expression
self._length = None
self._string = None
self._length = self._calculate_length()
self._string = self._calculate_string()
self.height = self._calculate_height()
def test_s3_cloudtrail_pasing_and_enrichment(self, lambda_boto3, parsing_boto3):
context = Context()
boto3 = parsing_boto3.client()
boto3.get_object.return_value = {"Body": test_data_gzipped()}
payload = {
"s3": {
"bucket": {
"name": "test-bucket",
},
"object": {
"key": "601427279990_CloudTrail_us-east-1_20210503T0000Z_QrttGEk4ZcBTLwj5.json.gz"
},
}
}
result = parsing.parse({"Records": [payload]}, context)
expected = copy.deepcopy([test_data["Records"][0]])
expected[0].update(
{
"ddsource": "cloudtrail",
"ddsourcecategory": "aws",
"service": "cloudtrail",
"aws": {
"s3": {
"bucket": payload["s3"]["bucket"]["name"],
"key": payload["s3"]["object"]["key"],
},
"function_version": context.function_version,
"invoked_function_arn": context.invoked_function_arn,
},
}
)
# yeah, there are tags, but we don't care to compare them
result[0].pop("ddtags")
# expected parsed result, now testing enrichment
self.assertEqual(expected[0], result[0])
expected[0]["host"] = "i-08014e4f62ccf762d"
self.assertEqual(expected[0], lambda_function.enrich(result)[0])
def get_apps_with_context(self, the_makefile, the_dict):
input = open(the_makefile).read()
output = parse(input)
def is_a_command(item):
return 'Command' in unicode(type(item))
appz = []
context = []
for each in output:
if is_a_command(each):
if 'FOREACH' in each.name and 'ENDFOREACH' not in each.name:
args = [item.contents for item in each.body]
context.append(args)
elif 'ENDFOREACH' in each.name:
context.pop()
elif 'OTB_TEST_APPLICATION' in each.name.upper():
appz.append((each, context[:]))
return appz
def add_make(self, previous_context, new_file):
with open(new_file) as f:
input = f.read()
output = parse(input)
apps = [each for each in output if 'Command' in str(type(each))]
setcommands = [each for each in apps if 'SET' in each.name.upper()]
stringcommands = [each for each in apps if 'STRING' in each.name.upper()]
environment = previous_context
def mini_clean(item):
if item.startswith('"') and item.endswith('"') and " " not in item:
return item[1:-1]
return item
new_env = {}
for command in setcommands:
key = command.body[0].contents
ct = " ".join([item.contents for item in command.body[1:]])
ct = mini_clean(ct)
if "$" in ct:
values = Template(ct)
else:
values = ct
new_env[key] = values
for stringcommand in stringcommands:
key = stringcommand.body[-1].contents
ct = stringcommand.body[-2].contents
ct = mini_clean(ct.lower())
if "$" in ct:
values = LowerTemplate(ct)
else:
values = ct
new_env[key] = values
resolve_dict(environment, new_env)
environment.update(new_env)
return environment
def get_apps_with_context(self, the_makefile, the_dict):
input = open(the_makefile).read()
output = parse(input)
def is_a_command(item):
return 'Command' in str(type(item))
appz = []
context = []
for each in output:
if is_a_command(each):
if 'FOREACH' in each.name and 'ENDFOREACH' not in each.name:
args = [item.contents for item in each.body]
context.append(args)
elif 'ENDFOREACH' in each.name:
context.pop()
elif 'OTB_TEST_APPLICATION' in each.name.upper():
appz.append((each, context[:]))
return appz
def recv(self, echo=False):
"""
Tries to receive one complete message from the buffer.
Returns None if client disconnected or if an error occurred.
"""
msg = None
#continue to receive until at least one message is produced
while msg == None:
try:
pay = self.__sock.recv(1024)
self.__buffer += pay
#client may have disconnected
if not pay: break
except socket.error, e:
self.__logger.error("receive error: %s" % str(e))
break
self.__buffer, msg = p.parse(self.__buffer)
def recv(self, echo = False):
"""
Tries to receive one complete message from the buffer.
Returns None if client disconnected or if an error occurred.
"""
msg=None
#continue to receive until at least one message is produced
while msg==None:
try:
pay=self.__sock.recv(1024)
self.__buffer+=pay
#client may have disconnected
if not pay: break
except socket.error,e:
self.__logger.error("receive error: %s"%str(e))
break
self.__buffer,msg=p.parse(self.__buffer)
def start_composition(visual):
cell = visual.head.cell
while cell.label != '@':
cell = cell.parent
if cell is None:
raise Exception("not implemented")
context = cell.parent.rule.at(cell.parent.index(cell))
if isinstance(context, Context):
rules = context.rules
else:
assert context
rules = [context]
# My attempt to create 'best result first' parser grand failed.
results = list(parsing.parse([c.copy() for c in cell], rules, timeout=0.5))
results.sort(key=lambda x: x.badness)
for result in results:
new_block = result.wrap()
replace(cell, new_block)
visual.setpos(Position.bottom(new_block))
yield None
visual.head, visual.tail = visual.document.undo()
def get_html(url):
"""Retrieve webpage content using provided URL.
:param url: webpage address
:type url: str
:returns: Tuple with HTML content and size of the content
:rtype: namedtuple(content, size)
"""
try:
result = requests.get(url)
logger.info('Fetching webpage: {url}'.format(url=url))
except requests.exceptions.RequestException:
logger.exception('Fetching webpage failed: {url}'.format(url=url))
return None, None
size = result.headers.get('content-length', len(result.content))
Result = namedtuple('Result', ['content', 'size'])
return Result(content=parse(result.content),
size='{:.1f}kb'.format(int(size) / 1024))
def main():
data = fetch(ticker='spy', start=datetime.datetime(2010, 1, 1),
end=datetime.datetime(2012, 10, 1))
kps = data[zig(data)!=0].Close
features = preProcess(kps)
ptables = f2p(features)
ptables.to_csv('test.csv')
buildCSV(ptables)
res = parse(ptables)
idx = kps.index
#
fig = plt.figure(figsize=(10,10))
ax = fig.add_subplot(211)
#data.Close.plot(style='.', ax=ax)
kps.plot(style='--o', ax=ax)
#
bx = fig.add_subplot(212, sharex=ax)
#features.plot(style='--o', ax=bx)
ptables.plot(style='o', ax=bx)
#import ipdb; ipdb.set_trace()
for terminal, (xmin, xmax), _, _ in res:
center = idx[(xmin+xmax)/2]
high = data[idx[xmin]:idx[xmax]].Close.max()
low = data[idx[xmin]:idx[xmax]].Close.min()
bottom, top = ax.get_ylim()
llow = (low-bottom)/float(top-bottom)
hhigh = (high-bottom)/float(top-bottom)
ax.axvspan(idx[xmin], idx[xmax], ymin=llow, ymax=hhigh, alpha=0.3)
ax.annotate(terminal, (center, high))
if terminal in ( Terminal('hs', None, None), Terminal('ihs', None, None)) :
ax.plot([idx[xmin+1], idx[xmax-1]], [kps[xmin+1], kps[xmax-1]], c='r' )
ax.plot([idx[xmin], idx[xmax]], [kps[xmin], kps[xmax]], c='r' )
ax.set_xlim([idx[0]-timedelta(weeks=1), idx[-1]])
plt.show()
def main(argv=None):
try:
if argv is None:
argv = sys.argv[1:]
parser = _mk_options_parser()
options, args = parser.parse_args(argv)
writer = _mk_writer(options)
if len(args) == 0:
print "No path given, trying /var/log/bootchart.tgz"
args = [ "/var/log/bootchart.tgz" ]
res = parsing.parse(writer, args, options.prune)
if options.interactive or options.output == None:
gui.show(res, options)
else:
filename = _get_filename(args, options)
def render():
batch.render(writer, res, options, filename)
if options.profile:
import cProfile
import pstats
profile = '%s.prof' % os.path.splitext(filename)[0]
cProfile.runctx('render()', globals(), locals(), profile)
p = pstats.Stats(profile)
p.strip_dirs().sort_stats('time').print_stats(20)
else:
render()
return 0
except parsing.ParseError, ex:
print("Parse error: %s" % ex)
return 2
def test_CMakelists(self):
provided = {}
provided["OTB_SOURCE_DIR"] = self.root_dir
provided["OTB_BINARY_DIR"] = self.build_dir
provided["OTB_DATA_LARGEINPUT_ROOT"] = os.path.normpath(os.path.join(self.root_dir, "../OTB-Data/Input"))
try:
with open(os.path.join(self.root_dir, "CMakeLists.txt")) as file_input:
content = file_input.read()
output = parse(content)
defined_paths = [each for each in output if 'Command' in str(type(each)) and "FIND_PATH" in each.name]
the_paths = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in defined_paths}
the_sets = [each for each in output if 'Command' in str(type(each)) and "SET" in each.name.upper()]
the_sets = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in the_sets}
the_sets = {key: " ".join(the_sets[key]) for key in the_sets}
the_strings = set([each.body[-1].contents for each in output if 'Command' in str(type(each)) and "STRING" in each.name.upper()])
def mini_clean(item):
if item.startswith('"') and item.endswith('"') and " " not in item:
return item[1:-1]
return item
the_sets = {key: mini_clean(the_sets[key]) for key in the_sets}
def templatize(item):
if "$" in item:
return Template(item)
return item
for key in the_sets:
if key in the_strings:
the_sets[key] = the_sets[key].lower()
the_sets = {key: templatize(the_sets[key]) for key in the_sets}
for path in the_paths:
target_file = the_paths[path][1]
suggested_paths = []
if len(the_paths[path]) > 2:
suggested_paths = the_paths[path][2:]
try:
provided[path] = find_file(target_file)
except Exception as e:
for each in suggested_paths:
st = Template(each)
pac = os.path.abspath(st.safe_substitute(provided))
if os.path.exists(pac):
provided[path] = pac
break
resolve_dict(provided, the_sets)
provided.update(the_sets)
return provided
except Exception as e:
traceback.print_exc()
self.fail(str(e))
def testParseLogDir(self): res = parsing.parse(writer, [self.rootdir], False, None, None) self.assertEqual(6, len(res))
def test_CMakelists(self):
provided = {}
provided["OTB_SOURCE_DIR"] = self.root_dir
provided["OTB_BINARY_DIR"] = self.build_dir
provided["OTB_DATA_LARGEINPUT_ROOT"] = os.path.normpath(os.path.join(self.root_dir, "../OTB-Data/Input"))
try:
with open(os.path.join(self.root_dir, "CMakeLists.txt")) as file_input:
content = file_input.read()
output = parse(content)
defined_paths = [each for each in output if 'Command' in str(type(each)) and "FIND_PATH" in each.name]
the_paths = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in defined_paths}
the_sets = [each for each in output if 'Command' in str(type(each)) and "SET" in each.name.upper()]
the_sets = {key.body[0].contents: [thing.contents for thing in key.body[1:]] for key in the_sets}
the_sets = {key: " ".join(the_sets[key]) for key in the_sets}
the_strings = set([each.body[-1].contents for each in output if 'Command' in str(type(each)) and "STRING" in each.name.upper()])
def mini_clean(item):
if item.startswith('"') and item.endswith('"') and " " not in item:
return item[1:-1]
return item
the_sets = {key: mini_clean(the_sets[key]) for key in the_sets}
def templatize(item):
if "$" in item:
return Template(item)
return item
for key in the_sets:
if key in the_strings:
the_sets[key] = the_sets[key].lower()
the_sets = {key: templatize(the_sets[key]) for key in the_sets}
for path in the_paths:
target_file = the_paths[path][1]
suggested_paths = []
if len(the_paths[path]) > 2:
suggested_paths = the_paths[path][2:]
try:
provided[path] = find_file(target_file)
except Exception as e:
for each in suggested_paths:
st = Template(each)
pac = os.path.abspath(st.safe_substitute(provided))
if os.path.exists(pac):
provided[path] = pac
break
resolve_dict(provided, the_sets)
provided.update(the_sets)
return provided
except Exception as e:
traceback.print_exc()
self.fail(str(e))
def parse(string):
structure = parsing.parse(string)
return collapse_sums(structure)
def transform(text):
output = StringIO()
generate(parse(text), output=output)
return output.getvalue()
def get_apps(self, the_makefile, the_dict):
input = open(the_makefile).read()
output = parse(input)
apps = [each for each in output if 'Command' in str(type(each))]
otb_apps = [each for each in apps if 'OTB_TEST_APPLICATION' in each.name.upper()]
return otb_apps
def get_tests(self, the_makefile, the_dict):
input = open(the_makefile).read()
output = parse(input)
apps = [each for each in output if "Command" in unicode(type(each))]
otb_tests = [each for each in apps if "ADD_TEST" in each.name.upper()]
return otb_tests
def pip(code=None, args=None, interactive=True):
if code or args:
interactive = False
if interactive:
print("=== Welcome to Pip, version %s ===" % VERSION)
print("Enter command-line args, terminated by newline (-h for help):")
args = input()
if args is not None:
# Artificial command-line input was provided
sys.argv = sys.argv[:1]
if type(args) is int:
args = str(args)
if type(args) is list:
# Add list of args to sys.argv, making sure they're all strings
sys.argv.extend(map(str, args))
elif type(args) is str:
# Parse the fake command-line input, simplistically accepting
# single- and double-quoted strings (with no escapes or shell
# expansion)
quote = None
buffer = None
for char in args + " ":
if char in "'\"":
if quote is None:
# Open quote
quote = char
buffer = buffer or ""
elif quote == char:
# Close quote
quote = None
else:
# Already inside the other type of quote
buffer += char
elif char == " " and quote is None:
if buffer is not None:
sys.argv.append(buffer)
buffer = None
else:
buffer = buffer or ""
buffer += char
argparser = argparse.ArgumentParser()
codeSources = argparser.add_mutually_exclusive_group()
listFormats = argparser.add_mutually_exclusive_group()
argparser.add_argument("-d",
"--debug",
help="equivalent to -pvw",
action="store_true")
codeSources.add_argument("-e",
"--execute",
help="execute the given code")
codeSources.add_argument("-f",
"--file",
help="execute code from the given file")
codeSources.add_argument("-i",
"--stdin",
help="execute code read from stdin",
action="store_true")
listFormats.add_argument("-l",
"--lines",
help=("output list items on separate lines, "
"concatenated"),
action="store_true")
listFormats.add_argument("-n",
"--newline",
help="concatenate lists on newline",
action="store_true")
listFormats.add_argument("-p",
"--repr",
help="print lists in repr form",
action="store_true")
listFormats.add_argument("-P",
"--reprlines",
help=("output list items on separate lines, "
"repr'd"),
action="store_true")
argparser.add_argument("-r",
"--readlines",
help="read args from lines of stdin",
action="store_true")
listFormats.add_argument("-s",
"--space",
help="concatenate lists on space",
action="store_true")
listFormats.add_argument("-S",
"--spacelines",
help=("output list items on separate lines, "
"space-concatenated"),
action="store_true")
argparser.add_argument("-v",
"--verbose",
help="show extra messages",
action="store_true")
argparser.add_argument("-V",
"--version",
help="display version info and quit",
action="store_true")
argparser.add_argument("-w",
"--warnings",
help="show nonfatal warning messages",
action="store_true")
argparser.add_argument("args",
help="arguments to main function",
nargs="*")
options = argparser.parse_args()
#!print(options)
if options.version:
print("Pip %s" % VERSION)
return
if options.debug:
options.warnings = options.verbose = options.repr = True
listFormat = ("p" if options.repr else
"P" if options.reprlines else
"s" if options.space else
"S" if options.spacelines else
"n" if options.newline else
"l" if options.lines else
None)
if not (code or options.execute or options.file or options.stdin):
if interactive:
options.stdin = True
print("Enter your program, terminated by Ctrl-D or Ctrl-Z:")
elif options.args:
# Treat first non-option arg as name of code file
options.file = options.args.pop(0)
else:
print("Type {} -h for usage information.".format(sys.argv[0]))
return
if code:
# Code is passed into function
program = code + "\n"
elif options.execute:
# Code is given as command-line argument
program = options.execute + "\n"
elif options.file:
# Get code from specified file
if interactive:
print("Reading", options.file)
try:
with open(options.file) as f:
program = f.read() + "\n"
except:
print("Could not read from file", options.file, file=sys.stderr)
return
elif options.stdin:
# Get code from stdin, stopping at EOF
program = "\n"
try:
while True:
program += input() + "\n"
except EOFError:
pass
try:
tkns = scan(program)
except FatalError:
print("Fatal error while scanning, execution aborted.",
file=sys.stderr)
return
if options.verbose:
print(addSpaces(tkns))
print()
try:
tree = parse(tkns)
except FatalError:
print("Fatal error while parsing, execution aborted.",
file=sys.stderr)
return
if options.verbose:
pprint.pprint(tree)
print()
state = ProgramState(listFormat, options.warnings)
if options.readlines:
args = []
try:
while True:
args.append(input())
except EOFError:
pass
else:
args = options.args
if interactive:
print("Executing...")
try:
state.executeProgram(tree, args)
except FatalError:
print("Fatal error during execution, program terminated.",
file=sys.stderr)
except KeyboardInterrupt:
print("Program terminated by user.", file=sys.stderr)
except RuntimeError as e:
# Probably exceeded Python's max recursion depth
print("Fatal error:", e, file=sys.stderr)
neighbors_with_movie.append(x)
except KeyError:
continue
bottom = 0
for x in neighbors_with_movie:
bottom += abs(tree.w[user][x])
try:
predicted = mean + top / bottom
except ZeroDivisionError:
predicted = None
return predicted
if __name__ == '__main__':
from parsing import parse
import pickle
import sys
test = sys.argv[1]
G = pickle.load(open(os.path.join(BASE_DIR, test + ".graph"), "rb"))
with open(os.path.join(BASE_DIR, test + "forest"), "rb") as f:
component = pickle.load(f)
test_rating = parse(os.path.join(BASE_DIR, test + '.test'))
base_rating = parse(os.path.join(BASE_DIR, test + '.base'))
predictor = Predictor(component, base_rating)
test_user = test_rating.keys()[0]
test_movie = test_rating[test_user].keys()[0]
predictor.predict(test_user, test_movie)
def send_text(message):
track_details = parsing.parse(message.text)
bot.send_message(message.chat.id, track_details)