def train():
cleanup.cleanup()
c.save(c.work_dir)
data_loader = TextLoader(c.work_dir, c.batch_size, c.seq_length)
with open(os.path.join(c.work_dir, 'chars_vocab.pkl'), 'wb') as f:
cPickle.dump((data_loader.chars, data_loader.vocab), f)
model = Model(c.rnn_size, c.num_layers, len(data_loader.chars), c.grad_clip, c.batch_size, c.seq_length)
with tf.Session() as sess:
tf.initialize_all_variables().run()
saver = tf.train.Saver(tf.all_variables())
for e in range(c.num_epochs):
sess.run(tf.assign(model.lr, c.learning_rate * (c.decay_rate ** e)))
data_loader.reset_batch_pointer()
state = model.initial_state.eval()
for b in range(data_loader.num_batches):
start = time.time()
x, y = data_loader.next_batch()
feed = {model.input_data: x, model.targets: y, model.initial_state: state}
train_loss, state, _ = sess.run([model.cost, model.final_state, model.train_op], feed)
end = time.time()
print("{}/{} (epoch {}), train_loss = {:.3f}, time/batch = {:.3f}"
.format(e * data_loader.num_batches + b,
c.num_epochs * data_loader.num_batches,
e, train_loss, end - start))
if (e * data_loader.num_batches + b) % c.save_every == 0:
checkpoint_path = os.path.join(c.work_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=e * data_loader.num_batches + b)
print("model saved to {}".format(checkpoint_path))
def main(args):
"""Main program to run preprocessing of the font Arguments:
font-file
--hinting=(False|True) ,default is false
"""
options = Options()
args = options.parse_opts(args, ignore_unknown=True)
if len(args) < 1:
print('usage: ./pyprepfnt font-file [--option=value]...', file=sys.stderr)
sys.exit(1)
fontfile = args[0]
args = args[1:]
filename, extension = os.path.splitext(fontfile)
cleanfile = filename + '_clean' + extension
cleanup.cleanup(fontfile, False, cleanfile)
closure.dump_closure_map(cleanfile, '.')
preprocess = Preprocess(cleanfile, '.')
preprocess.base_font()
preprocess.cmap_dump()
preprocess.serial_glyphs()
def main():
parser = argparse.ArgumentParser(description='Search video in channel')
parser.add_argument('--id', type=str)
parser.add_argument('--max', '-m', type=int, default=10)
parser.add_argument('--cleanup', action='store_true')
args = parser.parse_args()
config = config_factory.load()
master = master_site(config, merge_small_groups=False)
if args.cleanup:
cleanup(master)
return
if args.id is None:
skip_lang = ['fr']
langs = config.site_config.languages
for lang in langs:
if lang in skip_lang:
print("Skip fetching %s" % lang)
continue
print("==== Fetching %s ====" % lang)
master = fetch_all(config, master, lang, max_result=args.max)
cleanup(master)
else:
fetch_single(config, master, args.id)
def test_cleanup(self):
cleanupDir = "cleanup_test"
os.mkdir(cleanupDir)
for i in ["a", "b_tmp", "c_tmp", "_tmpd"]:
open(os.path.join(cleanupDir, i), "a").close()
cleanup.cleanup(cleanupDir)
self.assertEqual(sorted(["a", "_tmpd"]),
sorted(os.listdir(cleanupDir)))
shutil.rmtree(cleanupDir)
def run_all():
cleanup()
download_data()
unpack_data()
crop_data()
vectorize_data()
create_dataframe()
# cleanup()
print("Done!")
def log(self, s):
global colorize
if not self.silence:
if colorize:
print("\33[" + self.color + "m" + s + "\33[0m")
else:
print(s)
if self.should_exit:
cleanup.cleanup()
exit(-1)
def test_cleanup(self):
self.ch_openstack_utils.is_unit_paused_set.return_value = False
with self.assertRaises(actions.UnitNotPaused):
actions.cleanup([])
self.ch_openstack_utils.is_unit_paused_set.return_value = True
self.ch_core.hookenv.action_get.return_value = True
actions.cleanup([])
self.remove_patch_ports.assert_called_once_with('br-int')
self.ch_ovs.del_bridge.assert_called_once_with('br-tun')
self.remove_per_bridge_controllers.assert_called_once_with()
self.neutron_netns_cleanup.assert_called_once_with()
self.neutron_ipset_cleanup.assert_called_once_with()
def main(args):
"""Main program to run preprocessing of the font and dump the base parts
Arguments:
font-file
--output= Output folder of the files, default is current folder
--hinting=(False|True) ,default is false
"""
parser = argparse.ArgumentParser(prog='pyprepfnt')
parser.add_argument('fontfile', help='Input font file')
parser.add_argument('--changefont',
default=False,
action='store_true',
help='Font structure has changed, default is True')
parser.add_argument('--changebase',
default=False,
action='store_true',
help='Base structure has changed, default is True')
parser.add_argument(
'--hinting',
default=False,
action='store_true',
help='Enable hinting if specified, no hinting if not present')
parser.add_argument('--output',
default='.',
help='Output folder, default is current folder')
cmd_args = parser.parse_args(args)
fontfile = cmd_args.fontfile
# TODO(bstell) use Logger
print('preprocess {0}'.format(cmd_args.fontfile))
basename = os.path.basename(fontfile)
filename, extension = os.path.splitext(basename)
output_folder = cmd_args.output + '/' + filename
try:
os.makedirs(output_folder)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
cleanfile = output_folder + '/' + filename + '_clean' + extension
is_clean = os.path.isfile(cleanfile)
if not is_clean:
cleanup(fontfile, cmd_args.hinting, cleanfile)
dump_tables(cleanfile, output_folder)
print('done')
def main():
parser = optparse.OptionParser("%prog - I diff repo manifests")
parser.add_option("--filter", dest="filters", action="append")
parser.add_option("--diff", "-d", dest="diff", action="store_false", default=True)
parser.add_option("--format", dest="out_format", default="report")
parser.add_option("--output", dest="output", default=None)
parser.add_option("--root", dest="root", default=os.getcwd())
options, args = parser.parse_args()
if not options.output:
output = sys.stdout
else:
output = options.output
if os.path.exists(output):
print >> sys.stderr, "ERROR: Output file already exists"
exit(1)
if len(args) == 0:
print "Choose a command: diff, cleanup, filter"
exit(1)
elif len(args) > 1:
cmd_args = args[1:]
else:
cmd_args = None
cmd = args[0]
if cmd == 'diff':
if len(cmd_args) != 2:
print >> sys.stderr, "ERROR: must specify exactly two arguments (left and right)"
exit(1)
diff(cmd_args[0], cmd_args[1], output=output, output_format=options.out_format, filters=options.filters)
elif cmd == 'freeze':
freeze(cmd_args[0], output, options.root,
gaia_branch='v1-train',
gecko_branch='gecko-18',
moz_remotes=['b2g'],
moz_branch='v1-train')
elif cmd == 'cleanup':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
cleanup(cmd_args[0], output, options.filters)
elif cmd == 'filter':
if len(cmd_args) != 1:
print >> sys.stderr, "ERROR: you can only filter one file at a time"
exit(1)
if options.filters == None:
print >> sys.stderr, "ERROR: you must specify filters for the filter command"
exit(1)
filter(cmd_args[0], output, options.filters)
def upload_file():
# check if the post request has the file part
if 'file' not in request.files:
resp = make_response('No file provided', 400)
return resp
file = request.files['file']
# if user does not select file, browser also
# submit an empty part without filename
if file.filename == '':
resp = make_response('No file provided', 400)
return resp
if file and file.filename.lower().endswith('.docx'):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['upload_folder'], filename))
# Try to convert the file; redirect to success/fail page
try:
filename = convert_file(filename)
filename = cleanup(filename)
return redirect(url_for('converted_file', filename=filename))
except Exception as e:
return redirect(url_for('conversion_failure', error=e))
else:
resp = make_response(
f'Неправильный тип файла (требуется .docx): {file.filename}', 400)
return resp
def histogram(self, source_text):
'''Takes source text and counts frequency of each word used. Returns Dict()'''
source_text = cleanup(source_text)
word_frequency = dict()
for word in source_text:
# Adds to word if it's already in the dictionary
if word in word_frequency.keys():
word_frequency[word] += 1
pass
elif word == '':
pass
elif word not in word_frequency.keys():
# Removes periods from word then adds it to dict
if "." in word:
word = word.strip(".")
word_frequency.update({word: 1})
pass
# Adds word to dict
else:
word_frequency.update({word: 1})
pass
else:
pass
return word_frequency
def img_to_latex(name, folder, ext):
cleanName = cleanup.cleanup(name)
prefix = {"graph": "gr", "img": "img"}
f = open("latex/" + folder + "/" + cleanName + ".tex", "w")
try:
caption = open("raw/" + folder + "/" + cleanName + ".txt", "r").read()
except FileNotFoundError:
caption = cleanName.replace("_", " ")
if not cleanup.flags["sub"]:
f.write("\\begin{figure}[" +
("H" if cleanup.flags["nofloat"] else "h") + "]\n")
else:
f.write("\\begin{subfigure}{0.45\\textwidth}\n")
f.write("\\centering\n")
if ext == ".tex":
f.write("\\resizebox{0.9\\textwidth}{!}{\\input{../raw/" + folder +
"/" + cleanName + ext + "}}\n")
else:
f.write("\\includegraphics[width=0.9\\textwidth]{../raw/" + folder +
"/" + cleanName + ext + "}\n")
f.write("\\caption{" + caption + "}\n")
f.write("\\label{" + prefix[folder] + ":" + cleanName + "}\n")
if not cleanup.flags["sub"]:
f.write("\\end{figure}")
else:
f.write("\\end{subfigure}")
f.close()
return 0
def generator():
source_text = 'corpus.txt'
clean_text = cleanup(source_text)
tokenize_text = tokenize(clean_text)
markov = markov_chain(tokenize_text)
sentence = generate_sentence(markov, 4)
return render_template('index.html', sentence=sentence)
def main ():
manager = Manager()
# Range Sensor Distance
distance = manager.list([0, 0, 0, 0, 0, 0])
# DataBase
db = manager.list()
# Command from Keyboard
keyboard_cmd = manager.dict()
keyboard_cmd['vld'] = 0
keyboard_cmd['cmd'] = ''
# GoBack Ready
goback_cmd = manager.dict()
goback_cmd['vld'] = 0
#Step 1: Initialzation
initialization.initialize()
#Step 2: Departure
operations.take_off()
#Step 3: Multi threading start
to_thr = Process(target = timeout, args = (runtime, ))
key_thr = Process(target = keyboard_thread.keyboard_thread_wrapper_function, args = (keyboard_cmd, ))
cmd_thr = Process(target = command_thread.command_thread_wrapper_function, args = (distance, keyboard_cmd, db, goback_cmd, ))
rs_thr = Process(target = range_sensor_thread.range_sensor_wrapper_function, args = (distance, ))
gb_thr = Process(target = goback_thread.goback_wrapper_function, args = (goback_cmd, ))
cmd_thr.start()
rs_thr.start()
to_thr.start()
key_thr.start()
gb_thr.start()
#Step 4: Multi threading end
to_thr.join()
cmd_thr.terminate()
rs_thr.terminate()
key_thr.terminate()
gb_thr.terminate()
operations.landing()
time.sleep(1)
cleanup.cleanup()
def cleandata():
Session = sessionmaker(bind=engine)
s = Session()
options = []
query = s.query(ExerciseTopic)
# {"id":"q11","groupid":"q1","desc":"Hive Schema mismatch error","run":"","result":""}
for row in query:
dict = json.loads(row.property)
for key in dict.keys():
print(
key,
dict[key],
)
cleanup.cleanup(key, dict[key])
#session.delete(row)
options.append(dict)
return render_template('exercises/removesuccess.html', options=options)
def net(s, l):
global switch_to_root_eth0_name
"Create a network with custom topo"
topo = CustomTopo()
topo.create_spine_leaf(s, l)
info('*** Starting network ***\n')
#if not flowpaths_file:
# print "No flows file. Creating a network with default controller"
# net = Mininet(topo=topo, controller=None, host=CPULimitedHost, link=TCLink)
#else:
net = Mininet(topo=topo)
# create a node in root namespace and link to switch 0
routes = ['10.0.0.0/8']
ip = '10.123.123.1/32'
print switch_to_root_eth0_name
rootnode = connectToRootNS(net, net[switch_to_root_eth0_name], ip, routes)
# Generate and add flows
#generate_flows(flowpaths_file, topo)
# now run sshd on each host
cmd = '/usr/sbin/sshd'
opts = '-D -o UseDNS=no -u0'
for host in net.hosts:
print 'at host:', host
host.cmd(cmd + ' ' + opts + '&')
#add_root_eth0_flows(rootnode, switch_to_root_eth0_name, ip)
#os.system("./flows-tmp")
# print interface - port number mappings
print_port_numbers(net)
print
print "*** Hosts are running sshd at the following addresses:"
for host in net.hosts:
print host.name, host.IP()
print
print "*** Type 'exit' or control-D to shut down network"
CLI(net)
info('*** Stopping network ***')
net.stop()
from cleanup import cleanup
cleanup()
def shutdown(self, hard=False):
'''
Shutsdown the topology, kills any underlying processes and removes any virtual interfaces and bridges that were created
if hard is set to True, a hard shutdown is performed. All virtual devices created by the emulator are removed and all processes are killed.
'''
# for node in self.interNodeObjects:
# node.shutdown()
# if (node.is_physical == True):
# ## Remove virtual interfaces
# subprocess.call(utils.get_cmd_command("sudo ip link del {0}".format(node.vethInName)))
# subprocess.call(utils.get_cmd_command("sudo ip link del {0}".format(node.vethOutName)))
try:
self.coreemu.shutdown()
except CoreCommandError as e:
print(e)
global interface_names
cleanup(interface_names)
if hard:
subprocess.call(['core-cleanup'])
def tab_to_latex(name):
cleanName = cleanup.cleanup(name)
f = open("latex/table/" + cleanName + ".tex", "w")
fin = open("raw/table/" + name + ".dat", "r")
table_model = Template(open("aux/table/table_model.tpl").read())
tbody_sep = json.load(open("aux/table/tbody_model.json"))
try:
caption = open("raw/table/" + cleanName + ".txt", "r").read()
except FileNotFoundError:
caption = cleanName.replace("_", " ")
#fileLines = fin.readlines()
title = fin.readline().strip()
titlefields = re.split(r"\t+", title)
cols = ""
write_fields = []
for field in titlefields:
if field == "|":
cols += "|"
else:
cols += "c"
if field == "$empty":
write_fields.append("")
else:
write_fields.append(field)
write_title = compose_title(write_fields)
lines = []
for line in fin:
if line.startswith("#"):
continue
line = line.strip()
fields = re.split(r"\t+", line)
lines.append(compose_line(fields))
tablebody = compose_tbody(lines)
table = table_model.substitute(titleline = write_title, tbody = tablebody, cols = cols)
f.write("\\begin{table}[" + ("H" if cleanup.flags["nofloat"] else "h") + "]\n")
f.write(table)
f.write("\\caption{" + caption + "}\n")
f.write("\\label{tab:" + cleanName + "}\n")
f.write("\\end{table}")
f.close()
return 0
def main ():
loop = 0
manager = Manager()
distance = manager.list([0, 0, 0, 0, 0, 0])
# distance2 = manager.list([0, 0, 0, 0, 0, 0])
#Step 1: Initialzation
initialization.initialize()
#Step 2: Departure
operations.take_off()
#Step 3: Multi threading start
timeout_thread = Process(target = timeout, args = (runtime, ))
p1 = Process(target = command_thread.command_thread_wrapper_function, args = (distance, ))
p2 = Process(target = range_sensor_thread.range_sensor_wrapper_function, args = (distance, ))
p1.start()
p2.start()
timeout_thread.start()
#Step 4: Multi threading end
timeout_thread.join()
p1.terminate()
p2.terminate()
time.sleep(1)
cleanup.cleanup()
def connect(address, exit, connecting):
#set variable to track connection
connection = False
#retry connection until it is successful
while not connection:
try:
# connect only if another attempt is not in progress
if connecting.empty():
#log a current connection is in progress
connecting.put(1)
#disconnect previous connections and connect to peripheral
cleanup(address)
device = Peripheral(address, 'random')
#print and log successful connection and remove current connection attempt
print address, "connected"
connection = True
connecting.get()
#if keyboard interrupt is pressed exit program
except KeyboardInterrupt:
exit.put(1)
quit()
#log errors
except Exception as err:
method = 'connect()'
log_error(method,err)
print 'error at connect():',err
#remove connection attempt and wait to allow other peripherals to connect
connecting.get()
sleep(3)
#after connection enable notifications from peripheral
sleep(2)
enable_notifications(device)
return device
def main ():
loop = 0
#Step 1: Initialzation
initialization.initialize()
#Step 2: Departure
operations.take_off()
#while True :
while (loop < 300): #time for a loop: 0.15s
loop += 1
#Step 2: Detect Range
# for i in range(0, range_sensor.num_directions):
# distance[i] = range_sensor.detect_range(i)
distance[4] = range_sensor.detect_range(4)
time.sleep(0.05)
#Step 3: Analyze Range
# if (distance[4] > 180):
# #operations.hover()
# operations.move_backward()
# elif (distance[4] < 180):
# #operations.hover()
# operations.move_backward()
# else:
# #operations.hover()
# operations.move_backward()
#Step 4: Generate Next Command
if (loop < 50):
operations.hover()
elif (loop < 100):
operations.move_backward()
elif (loop < 150):
operations.hover()
elif (loop < 200):
operations.move_backward()
elif (loop < 250):
operations.drop()
else :
operations.drop()
cleanup.cleanup()
def main(args):
"""Main program to run preprocessing of the font and dump the base parts
Arguments:
font-file
--output= Output folder of the files, default is current folder
--hinting=(False|True) ,default is false
"""
parser = argparse.ArgumentParser(prog='pyprepfnt')
parser.add_argument('fontfile',help='Input font file')
parser.add_argument('--changefont', default=False , action='store_true', help='Font structure has changed, default is True')
parser.add_argument('--changebase', default=False , action='store_true', help='Base structure has changed, default is True')
parser.add_argument('--hinting',default=False, action='store_true', help='Enable hinting if specified, no hinting if not present')
parser.add_argument('--output', default='.' , help='Output folder, default is current folder')
cmd_args = parser.parse_args(args)
fontfile = cmd_args.fontfile
# TODO(bstell) use Logger
print('preprocess {0}'.format(cmd_args.fontfile))
basename = os.path.basename(fontfile)
filename, extension = os.path.splitext(basename)
output_folder = cmd_args.output+'/'+filename
try:
os.makedirs(output_folder)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
cleanfile = output_folder+'/'+filename + '_clean' + extension
is_clean = os.path.isfile(cleanfile)
if not is_clean:
cleanup(fontfile, cmd_args.hinting, cleanfile)
dump_tables(cleanfile, output_folder)
print('done')
def list_histogram(self, source_text):
'''Takes source text and returns frequency of each word used in a list'''
source_text = cleanup(source_text)
listo = []
words = []
for word in source_text:
if word in listo:
count = source_text.count(word)
listo.insert(listo.index(word) + 1, [word, count])
words.append(word)
listo.pop(listo.index(word))
elif word in words:
pass
elif word not in listo:
listo.append(word)
return listo
for file in tqdm(files):
file_tif = file[:-4] + '.tif'
string = file[:-4]
dat = string.split('_')
strain = dat[0]
series = dat[1]
age = dat[2]
name = dat[3]
gaussian_sigma = dat[4]
#algorithm = dat[5]
#decon = dat[6]
if clean == True:
cleanup.cleanup(infilename=n_infolder + file,
outfilename=n_outfolder_cleanedtrees + file,
neurontype=neurontype,
scale=scale,
visualize=False)
else:
n_outfolder_cleanedtrees = n_infolder
tree_lengths, tree_classes, tree_mean_radii, tree_max_radii, mainbranch, annotated_mainbranch = classify.classify(
infilename=n_outfolder_cleanedtrees + file,
outfilename_tree=n_outfolder_classifiedtrees + file,
outfilename_mainbranch=n_outfolder_mainbranch + file,
neurontype=neurontype,
length_threshold=length_threshold,
scale=scale)
kinks_count, angle_threshold, fully_annotated_mainbranch = wavyness.wavyness(
infilename_or_mainbranch=annotated_mainbranch,
}
directory = ''
if ADDRESS_STYLE == 'PARENT':
folder = input(colored('Input folder name: ', 'blue'))
if folder != '':
folder = '/' + folder + '/'
path = Path(__file__).resolve().parents[1].as_posix()
directory = path + folder
elif ADDRESS_STYLE == 'CURRENT':
folder = input(colored('Input folder name: ', 'blue'))
if folder != '':
folder = '/' + folder + '/'
path = Path(__file__).resolve().parents[0].as_posix()
directory = path + folder
elif ADDRESS_STYLE == 'ABSOLUTE':
directory = input(colored('Input folder\'s absolute path: ', 'blue'))
if directory[-1] != '/':
directory += '/'
if directory != '':
config = settings.open_settings(default_settings=DEFAULT_SETTINGS, directory=directory)
files = find_file.find_files(directory=directory)
if not build.build(files, directory=directory, settings=config):
run_tests.run(directory=directory, settings=config)
if config['style_check']:
style_check.check(files, directory=directory)
if config['cleanup']:
cleanup.cleanup(directory=directory, settings=config)
def calc_appstats_results(middleware):
if middleware.recorder:
total_call_count = 0
total_time = 0
calls = []
service_totals_dict = {}
likely_dupes = False
end_offset_last = 0
requests_set = set()
appstats_key = long(middleware.recorder.start_timestamp * 1000)
for trace in middleware.recorder.traces:
total_call_count += 1
total_time += trace.duration_milliseconds()
# Don't accumulate total RPC time for traces that overlap asynchronously
if trace.start_offset_milliseconds() < end_offset_last:
total_time -= (end_offset_last -
trace.start_offset_milliseconds())
end_offset_last = trace.start_offset_milliseconds(
) + trace.duration_milliseconds()
service_prefix = trace.service_call_name()
if "." in service_prefix:
service_prefix = service_prefix[:service_prefix.find(".")]
if service_prefix not in service_totals_dict:
service_totals_dict[service_prefix] = {
"total_call_count": 0,
"total_time": 0,
"total_misses": 0,
}
service_totals_dict[service_prefix]["total_call_count"] += 1
service_totals_dict[service_prefix][
"total_time"] += trace.duration_milliseconds()
stack_frames_desc = []
for frame in trace.call_stack_:
stack_frames_desc.append(
"%s:%s %s" % (RequestStats.short_rpc_file_fmt(
frame.class_or_file_name()), frame.line_number(),
frame.function_name()))
request = trace.request_data_summary()
response = trace.response_data_summary()
likely_dupe = request in requests_set
likely_dupes = likely_dupes or likely_dupe
requests_set.add(request)
request_short = request_pretty = None
response_short = response_pretty = None
miss = 0
try:
request_object = unformatter.unformat(request)
response_object = unformatter.unformat(response)
request_short, response_short, miss = cleanup.cleanup(
request_object, response_object)
request_pretty = pformat(request_object)
response_pretty = pformat(response_object)
except Exception, e:
logging.warning("Prettifying RPC calls failed.\n%s", e)
service_totals_dict[service_prefix]["total_misses"] += miss
calls.append({
"service":
trace.service_call_name(),
"start_offset":
RequestStats.milliseconds_fmt(
trace.start_offset_milliseconds()),
"total_time":
RequestStats.milliseconds_fmt(
trace.duration_milliseconds()),
"request":
request_pretty or request,
"response":
response_pretty or response,
"request_short":
request_short or cleanup.truncate(request),
"response_short":
response_short or cleanup.truncate(response),
"stack_frames_desc":
stack_frames_desc,
"likely_dupe":
likely_dupe,
})
service_totals = []
for service_prefix in service_totals_dict:
service_totals.append({
"service_prefix":
service_prefix,
"total_call_count":
service_totals_dict[service_prefix]["total_call_count"],
"total_misses":
service_totals_dict[service_prefix]["total_misses"],
"total_time":
RequestStats.milliseconds_fmt(
service_totals_dict[service_prefix]["total_time"]),
})
service_totals = sorted(
service_totals,
reverse=True,
key=lambda service_total: float(service_total["total_time"]))
return {
"total_call_count": total_call_count,
"total_time": RequestStats.milliseconds_fmt(total_time),
"calls": calls,
"service_totals": service_totals,
"likely_dupes": likely_dupes,
"appstats_key": appstats_key,
}
def run_tweets(event='', msg_counter=[]):
# Reset output
reset_counter = 0
for msg in msg_counter:
msg_counter[reset_counter].destroy()
reset_counter += 1
msg_counter.clear()
# Catch any errors that might come from the input
try:
# Check if the entry is a twitter url and post specific tweet if so
if re.search('^(https:\/\/twitter\.com\/)',
user_input.get()) and re.search('\/(\d+)$',
user_input.get()):
# Pulls the tweet ID out of the url
url = re.search('(\d+)$', user_input.get()).group(0)
# Output original tweet
valid_text = api.get_status(url, tweet_mode="extended").full_text
valid_text = ''.join(c for c in valid_text if c <= '\uFFFF')
orig_message = tk.Message(root,
text=valid_text,
background='antiquewhite1',
width='400')
orig_message.grid(column=0, row=3, sticky='W' + 'E')
msg_counter.append(orig_message)
#Post changed tweet only if there were changes
if valid_text == cleanup(valid_text):
clean_message = tk.Message(root,
text="No changes were made",
background='antiquewhite1',
width='400')
clean_message.grid(column=1, row=3, sticky='W' + 'E')
msg_counter.append(clean_message)
else:
clean_message = tk.Message(root,
text=cleanup(valid_text),
background='antiquewhite1',
width='400')
clean_message.grid(column=1, row=3, sticky='W' + 'E')
msg_counter.append(clean_message)
# Search up to 10 tweets matching inputted term
else:
loop_pos = 0
for status in tweepy.Cursor(api.search,
q=user_input.get() +
' -filter:retweets',
tweet_mode='extended',
result_type='mixed').items(100):
if loop_pos < 10:
# Clean up emoji and other invalid characters
valid_text = status.full_text
valid_text = ''.join(c for c in valid_text
if c <= '\uFFFF')
# Check if there were any changes to each post
# Alternate colors per line and output original/changed text
if cleanup(valid_text) != valid_text:
if (loop_pos % 2) == 0:
alternate_colors = 'antiquewhite1'
else:
alternate_colors = root.cget('bg')
# Output original and cleaned up tweets into two columns
orig_message = tk.Message(root,
text=valid_text,
background=alternate_colors,
width='400')
orig_message.grid(column=0,
row=loop_pos + 3,
sticky='W' + 'E')
msg_counter.append(orig_message)
clean_message = tk.Message(root,
text=cleanup(valid_text),
background=alternate_colors,
width='400')
clean_message.grid(column=1,
row=loop_pos + 3,
sticky='W' + 'E')
msg_counter.append(clean_message)
loop_pos += 1
if loop_pos == 0:
orig_message = tk.Message(
root,
text="No fitting messages were found",
background='antiquewhite1',
width='400')
orig_message.grid(column=0, row=loop_pos + 3, sticky='W' + 'E')
msg_counter.append(orig_message)
except:
clean_message = tk.Message(root,
text=cleanup(valid_text),
background=alternate_colors,
width='400')
clean_message.grid(column=1, row=loop_pos + 3, sticky='W' + 'E')
msg_counter.append(clean_message)
{'active':True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
errorinrun += router.rundispatcher(command,use_routestorun)
if userscript and hasattr(userscript,'post' + command):
botslib.runscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
#in acceptance tests: run a user script that can do checks on results of acceptance test.******************************
#this is before the cleanup, but this is not critical
if acceptance_userscript:
if hasattr(acceptance_userscript,'posttest'):
#no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
try:
botslib.runscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
except Exception,msg:
print str(msg)
cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
except Exception,msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
if __name__ == '__main__':
start()
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = ['--new','--retry','--retransmit','--cleanup','--crashrecovery','--retrycommunication','--automaticretrycommunication']
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".',key,value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py.'))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(minutes=botsglobal.ini.getint('settings','maxruntime',60))
for row in botslib.query('''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',{'vanaf':vanaf}):
warn = _(u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(_(u'Database is locked, but "maxruntime" has not been exceeded.'))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.')
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'),warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--crashrecovery',stuff2evaluate)
else:
botsglobal.logger.info(_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript,'postcrashrecovery'):
botslib.runscript(userscript,scriptname,'postcrashrecovery',routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postretrycommunication'):
botslib.runscript(userscript,scriptname,'postretrycommunication',routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,'--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--automaticretrycommunication',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run automatic recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript,'postautomaticretrycommunication'):
botslib.runscript(userscript,scriptname,'postautomaticretrycommunication',routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun,'--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retry',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript,'postretry'):
botslib.runscript(userscript,scriptname,'postretry',routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate('--retransmit',stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript,'postretransmit'):
botslib.runscript(userscript,scriptname,'postretransmit',routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate('--new',stuff2evaluate)
if userscript and hasattr(userscript,'postnewrun'):
botslib.runscript(userscript,scriptname,'postnewrun',routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get('settings','whencleanup','always')=='always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception,e:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%s')%(e)) #of course this 'should' not happen.
sys.exit(1)
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
'''%{'name':os.path.basename(sys.argv[0]),'version':botsglobal.version}
configdir = 'config'
commandspossible = ['--automaticretrycommunication','--resend','--rereceive','--new']
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?",'-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [command[2:] for command in commandspossible if command in commandstorun] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories','botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key,value in botslib.botsinfo(): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".',{'key':key,'value':value})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'),{'msg':msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript,scriptname = botslib.botsimport('routescripts','botsengine')
except ImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance','runacceptancetest',False):
botsglobal.logger.info(_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'))
try:
acceptance_userscript,acceptance_scriptname = botslib.botsimport('routescripts','bots_acceptancetest')
except ImportError:
botsglobal.logger.info(_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is damaged'),warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(_(u'[Bots severe error]Database is locked'),warn)
commandstorun.insert(0,'crashrecovery') #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'pretest',routestorun=routestorun)
botslib.tryrunscript(userscript,scriptname,'pre',commandstorun=commandstorun,routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),{'command':command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(_(u'Run routes from command line: "%(routes)s".'),{'routes':str(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''',
{'active':True,'notindefaultrun':False}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database that are in default run: "%(routes)s".'),{'routes':str(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query('''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active':True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run all active routes from database: "%(routes)s".'),{'routes':str(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,scriptname,'pre' + command,routestorun=use_routestorun)
errorinrun += router.rundispatcher(command,use_routestorun)
botslib.tryrunscript(userscript,scriptname,'post' + command,routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,scriptname,'post',commandstorun=commandstorun,routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,acceptance_scriptname,'posttest',routestorun=use_routestorun)
except Exception as msg:
print str(msg)
cleanup.cleanup(do_cleanup_parameter,userscript,scriptname)
except Exception as msg:
botsglobal.logger.exception(_(u'Severe error in bots system:\n%(msg)s'),{'msg':str(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
def getUniqueRows(filename_to_read, col_to_lookup):
import cleanup
return len(cleanup.cleanup(filename_to_read, col_to_lookup))
#Set routes in each namespace will try to automate making these dictionaries later. Already have some code to kinda do this
routers = [r1, r2, r3, r4, r5, r6]
all_routes = {'201':[{'dest':'192.168.1.1' , 'intf':'201-eth0', 'src':'192.168.1.2'},{'dest':'192.168.1.5', 'intf':'201-eth1', 'src':'192.168.1.3'},{'dest':'192.168.1.11' , 'intf':'201-eth2', 'src':'192.168.1.4'}],
'202':[{'dest':'192.168.1.3' , 'intf':'202-eth0', 'src':'192.168.1.5'},{'dest':'192.168.1.8' , 'intf':'202-eth1', 'src':'192.168.1.6'}],
'203':[{'dest':'192.168.1.6' , 'intf':'203-eth1', 'src':'192.168.1.8'},{'dest':'192.168.1.9' , 'intf':'203-eth0', 'src':'192.168.1.7'},{'dest':'192.168.1.12' , 'intf':'203-eth2', 'src':'192.168.1.10'}],
'204':[{'dest':'192.168.1.4' , 'intf':'204-eth0', 'src':'192.168.1.11'},{'dest':'192.168.1.10' , 'intf':'204-eth1', 'src':'192.168.1.12'},{'dest':'192.168.1.14' , 'intf':'204-eth2', 'src':'192.168.1.13'}],
'205':[{'dest':'192.168.1.13' , 'intf':'205-eth0', 'src':'192.168.1.14'},{'dest':'192.168.1.16' , 'intf':'205-eth1', 'src':'192.168.1.15'}],
'206':[{'dest':'192.168.1.15' , 'intf':'206-eth0', 'src':'192.168.1.16'},{'dest':'192.168.1.19' , 'intf':'206-eth1', 'src':'192.168.1.17'},{'dest':'192.168.1.20' , 'intf':'206-eth2', 'src':'192.168.1.18'}]}
routers = [r1, r2, r3, r4, r5, r6]
for router in routers:
routes = all_routes[str(router)]
for route in routes:
router.cmd( 'ip route add {} via {} dev {} proto {} scope {} src {}'.format(route['dest'], route['src'], route['intf'],'kernel', 'link', route['src']))
for router in routers:
ifconfig_parse(router.cmd( 'ifconfig' ), str(router))
#Build network
info('Building Network\n')
net.build()
CLI( net )
if __name__ == '__main__':
logging.basicConfig(filename='debug_logs/network_debug.log', level=logging.INFO)
example_network_2()
cleanup()
__author__ = 'tianlan'
import time
import initialization
import range_sensor
import operations
import cleanup
import atexit
atexit.register(cleanup.cleanup())
distance = [0, 0, 0, 0, 0, 0]
hover_thr = 1500
def main ():
loop = 0
#Step 1: Initialzation
initialization.initialize()
#Step 2: Departure
operations.take_off()
#while True :
while (loop < 300): #time for a loop: 0.15s
loop += 1
#Step 2: Detect Range
# for i in range(0, range_sensor.num_directions):
# distance[i] = range_sensor.detect_range(i)
distance[4] = range_sensor.detect_range(4)
time.sleep(0.05)
#Step 3: Analyze Range
def calc_appstats_results(middleware):
if middleware.recorder:
total_call_count = 0
total_time = 0
calls = []
service_totals_dict = {}
likely_dupes = False
end_offset_last = 0
requests_set = set()
appstats_key = long(middleware.recorder.start_timestamp * 1000)
for trace in middleware.recorder.traces:
total_call_count += 1
total_time += trace.duration_milliseconds()
# Don't accumulate total RPC time for traces that overlap asynchronously
if trace.start_offset_milliseconds() < end_offset_last:
total_time -= end_offset_last - trace.start_offset_milliseconds()
end_offset_last = trace.start_offset_milliseconds() + trace.duration_milliseconds()
service_prefix = trace.service_call_name()
if "." in service_prefix:
service_prefix = service_prefix[: service_prefix.find(".")]
if service_prefix not in service_totals_dict:
service_totals_dict[service_prefix] = {"total_call_count": 0, "total_time": 0, "total_misses": 0}
service_totals_dict[service_prefix]["total_call_count"] += 1
service_totals_dict[service_prefix]["total_time"] += trace.duration_milliseconds()
stack_frames_desc = []
for frame in trace.call_stack_list():
stack_frames_desc.append(
"%s:%s %s"
% (
RequestStats.short_rpc_file_fmt(frame.class_or_file_name()),
frame.line_number(),
frame.function_name(),
)
)
request = trace.request_data_summary()
response = trace.response_data_summary()
likely_dupe = request in requests_set
likely_dupes = likely_dupes or likely_dupe
requests_set.add(request)
request_short = request_pretty = None
response_short = response_pretty = None
miss = 0
try:
request_object = unformatter.unformat(request)
response_object = unformatter.unformat(response)
request_short, response_short, miss = cleanup.cleanup(request_object, response_object)
request_pretty = pformat(request_object)
response_pretty = pformat(response_object)
except Exception, e:
logging.warning("Prettifying RPC calls failed.\n%s", e)
service_totals_dict[service_prefix]["total_misses"] += miss
calls.append(
{
"service": trace.service_call_name(),
"start_offset": RequestStats.milliseconds_fmt(trace.start_offset_milliseconds()),
"total_time": RequestStats.milliseconds_fmt(trace.duration_milliseconds()),
"request": request_pretty or request,
"response": response_pretty or response,
"request_short": request_short or cleanup.truncate(request),
"response_short": response_short or cleanup.truncate(response),
"stack_frames_desc": stack_frames_desc,
"likely_dupe": likely_dupe,
}
)
service_totals = []
for service_prefix in service_totals_dict:
service_totals.append(
{
"service_prefix": service_prefix,
"total_call_count": service_totals_dict[service_prefix]["total_call_count"],
"total_misses": service_totals_dict[service_prefix]["total_misses"],
"total_time": RequestStats.milliseconds_fmt(service_totals_dict[service_prefix]["total_time"]),
}
)
service_totals = sorted(
service_totals, reverse=True, key=lambda service_total: float(service_total["total_time"])
)
return {
"appstats_available": True,
"total_call_count": total_call_count,
"total_time": RequestStats.milliseconds_fmt(total_time),
"calls": calls,
"service_totals": service_totals,
"likely_dupes": likely_dupes,
"appstats_key": appstats_key,
}
def results(self):
"""Return appstats results in a dictionary for template context."""
if not self.recorder:
# If appstats fails to initialize for any reason, return an empty
# set of results.
logging.warn("Missing recorder for appstats profiler.")
return {
"calls": [],
"total_time": 0,
}
total_call_count = 0
total_time = 0
calls = []
service_totals_dict = {}
likely_dupes = False
end_offset_last = 0
requests_set = set()
appstats_key = long(self.recorder.start_timestamp * 1000)
for trace in self.recorder.traces:
total_call_count += 1
total_time += trace.duration_milliseconds()
# Don't accumulate total RPC time for traces that overlap asynchronously
if trace.start_offset_milliseconds() < end_offset_last:
total_time -= (end_offset_last - trace.start_offset_milliseconds())
end_offset_last = trace.start_offset_milliseconds() + trace.duration_milliseconds()
service_prefix = trace.service_call_name()
if "." in service_prefix:
service_prefix = service_prefix[:service_prefix.find(".")]
if service_prefix not in service_totals_dict:
service_totals_dict[service_prefix] = {
"total_call_count": 0,
"total_time": 0,
"total_misses": 0,
}
service_totals_dict[service_prefix]["total_call_count"] += 1
service_totals_dict[service_prefix]["total_time"] += trace.duration_milliseconds()
stack_frames_desc = []
for frame in trace.call_stack_list():
stack_frames_desc.append("%s:%s %s" %
(util.short_rpc_file_fmt(frame.class_or_file_name()),
frame.line_number(),
frame.function_name()))
request = trace.request_data_summary()
response = trace.response_data_summary()
likely_dupe = request in requests_set
likely_dupes = likely_dupes or likely_dupe
requests_set.add(request)
request_short = request_pretty = None
response_short = response_pretty = None
miss = 0
try:
request_object = unformatter.unformat(request)
response_object = unformatter.unformat(response)
request_short, response_short, miss = cleanup.cleanup(request_object, response_object)
request_pretty = pformat(request_object)
response_pretty = pformat(response_object)
except Exception, e:
pass
# enable this if you want to improve prettification
# logging.warning("Prettifying RPC calls failed.\n%s\nRequest: %s\nResponse: %s",
# e, request, response, exc_info=True)
service_totals_dict[service_prefix]["total_misses"] += miss
calls.append({
"service": trace.service_call_name(),
"start_offset": util.milliseconds_fmt(trace.start_offset_milliseconds()),
"total_time": util.milliseconds_fmt(trace.duration_milliseconds()),
"request": request_pretty or request,
"response": response_pretty or response,
"request_short": request_short or cleanup.truncate(request),
"response_short": response_short or cleanup.truncate(response),
"stack_frames_desc": stack_frames_desc,
"likely_dupe": likely_dupe,
})
parser.get("DEFAULT", "container_name"), parser.get("DEFAULT", "device")
)
)
mainLogger.info(
"NOTE: this could take a while, depending on how fast your USB-Stick and your PC is. So please be patient, stay calm and drink a cup of coffee after you entered your password twice if you know you're not running 21st century hardware..."
)
#
# Exit if the container already exists
#
if os.path.exists(parser.get("DEFAULT", "container_path")):
mainLogger.info(
"The Container given ({}) already exists. Exiting...".format(parser.get("DEFAULT", "container_path"))
)
cleanup(mountpoint, tc_mountpoint, tempdir, args.device)
exit()
#
# Exit if there is not enough free diskspace
#
s = os.statvfs(mountpoint)
totalSize = s.f_bavail * s.f_frsize
if totalSize < int(parser.get("truecrypting", "size")):
mainLogger.info("Insufficient Diskpace on your Device: {}".format(parser.get("DEFAULT", "device")))
cleanup_failed(mountpoint, tc_mountpoint, tempdir, args.device, parser.get("DEFAULT", "container_name"))
exit()
#
# Create Container
def terminate(force):
print('terminating all instances and loadbalancer...')
succ = _get_service_data() if not force else True
if succ:
cleanup()
def main(args):
"""Preprocess a font for use as a TachyFont.
Args:
args: list, command line arguments.
Raises:
ValueError: if build directory cannot be created
Returns:
Status of the operation.
"""
parser = argparse.ArgumentParser(prog='pyprepfnt')
parser.add_argument('fontfile', help='Input font file')
parser.add_argument('output_dir', help='Output directory')
parser.add_argument('--hinting', default=False, action='store_true',
help='Retain hinting if set, else strip hinting')
parser.add_argument('--log', default='WARNING',
help='Set the logging level; eg, --log=INFO')
cmd_args = parser.parse_args(args)
loglevel = getattr(logging, cmd_args.log.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % loglevel)
log = logging.getLogger()
logging_handler = logging.StreamHandler(sys.stdout)
logging_handler.setLevel(loglevel)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - '
'%(message)s')
logging_handler.setFormatter(formatter)
log.addHandler(logging_handler)
log.setLevel(loglevel)
verbose = loglevel < logging.DEBUG
fontfile = cmd_args.fontfile
# TODO(bstell) use Logger
log.info('preprocess ' + cmd_args.fontfile)
basename = os.path.basename(fontfile)
filename, extension = os.path.splitext(basename)
cur_time = datetime.datetime.now()
build_dir = ('tmp-%s-%04d-%02d-%02d-%02d-%02d-%02d.%d' %
(filename, cur_time.year, cur_time.month, cur_time.day,
cur_time.hour, cur_time.minute, cur_time.second, os.getpid()))
output_dir = cmd_args.output_dir
log.info('put results in ' + output_dir)
try:
os.makedirs(build_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
log.error('failed to create build_dir (' + build_dir + ')')
raise
cleanfile = filename + '_clean' + extension
cleanfilepath = build_dir + '/' + cleanfile
log.debug('make cleaned up version: ' + cleanfilepath)
cleanup.cleanup(fontfile, cmd_args.hinting, cleanfilepath)
log.info(basename + '=' + str(os.path.getsize(fontfile)) + ', ' +
cleanfilepath + '=' + str(os.path.getsize(cleanfilepath)))
closure.dump_closure_map(cleanfilepath, build_dir)
log.debug('start proprocess')
preprocess = Preprocess(cleanfilepath, build_dir, verbose)
log.debug('build base')
preprocess.base_font()
log.debug('dump cmap')
preprocess.cmap_dump()
log.debug('build glyph data')
preprocess.serial_glyphs()
log.debug('create jar file')
tachyfont_file = filename + '.TachyFont.jar'
sub_files = ('base closure_data closure_idx codepoints gids glyph_data '
'glyph_table')
jar_cmd = 'cd %s; jar cf %s %s' % (build_dir, tachyfont_file, sub_files)
log.debug('jar_cmd: ' + jar_cmd)
status = os.system(jar_cmd)
log.debug('jar command status: ' + str(status))
if status:
log.error('jar command status: ' + str(status))
return status
log.debug('move the files to the output directory')
mv_cmd = ('cd %s; mv %s %s %s' %
(build_dir, tachyfont_file, cleanfile, output_dir))
log.debug('mv_cmd: ' + mv_cmd)
status = os.system(mv_cmd)
log.debug('mv status ' + str(status))
if status:
log.error('mv status = ' + str(status))
return status
log.debug('cleanup the build directory')
rm_cmd = ('rm -rf %s' % build_dir)
log.debug('rm_cmd: ' + rm_cmd)
status = os.system(rm_cmd)
log.debug('rm status ' + str(status))
if status:
log.error('rm status = ' + str(status))
return status
log.info('command status = ' + str(status))
return status
def main():
if version_info.major != 3:
raise VersionMismatch("Must be using Python3")
if len(argv) == 1:
print(f"Jpm {version} - help:")
with open("/usr/local/Jac/Data/jpm-help.txt") as help_file:
print(help_file.read(), end='')
return
arg = argv[1]
args = argv[2:]
if arg == "install":
from install import install
from checkForRepositoryConnection import checkConnection, InternetConnectionError
from upgrader import checkForJaclangUpdate
checkForJaclangUpdate()
try:
checkConnection()
except InternetConnectionError as e_:
printException(e_)
install(set(args))
elif arg == "remove":
from remove import removePackages, PackageError
from upgrader import checkForJaclangUpdate
checkForJaclangUpdate()
try:
removePackages(set(args))
except PackageError as e_:
printException(e_)
elif arg == "list":
from listPackages import listPackages
listPackages()
elif arg == "cleanup":
from cleanup import cleanup
from upgrader import checkForJaclangUpdate
checkForJaclangUpdate()
cleanup()
elif arg == "upgrade":
from upgrader import upgrade
from checkForRepositoryConnection import checkConnection, InternetConnectionError
try:
checkConnection()
except InternetConnectionError as e_:
printException(e_)
upgrade()
elif arg == "listall":
from listAll import listall
from upgrader import checkForJaclangUpdate
checkForJaclangUpdate()
listall()
elif arg == "uninstall":
from uninstall import uninstall
uninstall()
else:
raise ArgumentError(f"Unknown argument: {arg}")
def main(args):
"""Preprocess a font for use as a TachyFont.
Args:
args: list, command line arguments.
Raises:
ValueError: if build directory cannot be created
Returns:
Status of the operation.
"""
parser = argparse.ArgumentParser(prog='pyprepfnt')
parser.add_argument('fontfile', help='Input font file')
parser.add_argument('output_dir', help='Output directory')
parser.add_argument(
'--force',
default=False,
action='store_true',
help='Force preprocessing even if the timestamps indicate'
' it is not necessary')
parser.add_argument(
'--hinting',
default=False,
action='store_true',
help='Retain hinting if set, else strip hinting')
parser.add_argument(
'--reuse_clean',
default=False,
action='store_true',
help='Reuse the "clean" file if possible')
parser.add_argument(
'--log',
default='WARNING',
help='Set the logging level; eg, --log=INFO')
parser.add_argument(
'--verbose',
default=False,
action='store_true',
help='Report internal operations')
cmd_args = parser.parse_args(args)
loglevel = getattr(logging, cmd_args.log.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % loglevel)
log = logging.getLogger()
logging_handler = logging.StreamHandler(sys.stdout)
logging_handler.setLevel(loglevel)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
logging_handler.setFormatter(formatter)
log.addHandler(logging_handler)
log.setLevel(loglevel)
verbose = cmd_args.verbose
force_preprocessing = cmd_args.force
log.debug('force_preprocessing = ' + str(force_preprocessing))
fontfile = cmd_args.fontfile
fonttime = os.path.getmtime(fontfile)
# TODO(bstell) use Logger
basename = os.path.basename(fontfile)
log.info('preprocess %s = %d bytes' % (cmd_args.fontfile,
os.path.getsize(cmd_args.fontfile)))
filename, extension = os.path.splitext(basename)
cur_time = datetime.datetime.now()
build_dir = 'tmp-%s' % filename
if not cmd_args.reuse_clean:
build_dir = ('%s-%04d-%02d-%02d-%02d-%02d-%02d.%d' % (
build_dir, cur_time.year, cur_time.month, cur_time.day,
cur_time.hour, cur_time.minute, cur_time.second, os.getpid()))
output_dir = cmd_args.output_dir
log.debug('JAR file: ' + output_dir)
try:
os.makedirs(build_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
log.error('failed to create build_dir (' + build_dir + ')')
raise
log.debug(
'if reuse_clean then we should compare the source font and final jar')
cleanfile = filename + '_clean' + extension
cleanfilepath = build_dir + '/' + cleanfile
# Decide if we are building the cleaned up version of the font.
rebuild_clean = not cmd_args.reuse_clean
cleanfile_exists = os.path.isfile(cleanfilepath)
if force_preprocessing or not cleanfile_exists:
rebuild_clean = True
else:
cleantime = os.path.getmtime(cleanfilepath)
if cleantime <= fonttime:
rebuild_clean = True
log.debug('rebuild_clean = ' + str(rebuild_clean))
if rebuild_clean:
log.debug('cleaned version: ' + cleanfilepath)
cleanup.cleanup(fontfile, cmd_args.hinting, cleanfilepath, verbose)
closure.dump_closure_map(cleanfilepath, build_dir)
else:
log.debug('reuse cleaned up version: ' + cleanfilepath)
# Get the latest cleaned up font timestamp.
cleantime = os.path.getmtime(cleanfilepath)
# Decide if we are rebuilding the jar file.
tachyfont_file = filename + '.TachyFont.jar'
jarfilepath = build_dir + '/' + tachyfont_file
rebuild_jar = False
jarfile_exists = os.path.isfile(jarfilepath)
log.debug('file %s exists: %s' % (jarfilepath, jarfile_exists))
if force_preprocessing or not jarfile_exists:
rebuild_jar = True
else:
jartime = os.path.getmtime(jarfilepath)
if jartime <= cleantime:
rebuild_jar = True
log.debug('rebuild_jar = ' + str(rebuild_jar))
if rebuild_jar:
log.debug('start proprocess')
preprocess = Preprocess(cleanfilepath, build_dir, verbose)
log.debug('build base')
preprocess.base_font()
log.debug('dump cmap')
preprocess.cmap_dump()
log.debug('build glyph data')
preprocess.serial_glyphs()
log.debug('write sha-1 fingerprint')
preprocess.sha1_fingerprint()
log.debug('create jar file')
sub_files = (
'base closure_data closure_idx codepoints gids glyph_data '
'glyph_table sha1_fingerprint')
jar_cmd = 'cd %s; jar cf %s %s' % (build_dir, tachyfont_file,
sub_files)
log.debug('jar_cmd: ' + jar_cmd)
status = os.system(jar_cmd)
log.debug('jar command status: ' + str(status))
if status:
log.error('jar command status: ' + str(status))
return status
else:
log.debug('no need to rebuild intermediate jar file: ' + jarfilepath)
# Get the latest cleaned up jar timestamp.
jartime = os.path.getmtime(jarfilepath)
# Decide if we are copying over the jar file.
copy_jar = False
jarcopy_filepath = output_dir + '/' + tachyfont_file
jarcopy_exists = os.path.isfile(jarcopy_filepath)
if force_preprocessing or not jarcopy_exists:
copy_jar = True
else:
jarcopytime = os.path.getmtime(jarcopy_filepath)
if jarcopytime <= jartime:
copy_jar = True
log.debug('copy_jar = ' + str(copy_jar))
if copy_jar:
log.debug('cp the files to the output directory')
log.info(
'cleaned: %s = %d' % (cleanfile, os.path.getsize(cleanfilepath)))
log.info('Jar: %s/%s' % (output_dir, tachyfont_file))
cp_cmd = ('cp %s/%s %s/%s %s' % (build_dir, tachyfont_file, build_dir,
cleanfile, output_dir))
log.debug('cp_cmd: ' + cp_cmd)
status = os.system(cp_cmd)
log.debug('cp status ' + str(status))
if status:
log.error('cp status = ' + str(status))
return status
else:
log.debug('the existing jar file is up to date: ' + jarfilepath)
if cmd_args.reuse_clean:
log.debug('leaving the build directory: ' + build_dir)
status = 0
else:
log.debug('cleanup the build directory')
rm_cmd = ('rm -rf %s' % build_dir)
log.debug('rm_cmd: ' + rm_cmd)
status = os.system(rm_cmd)
log.debug('rm status ' + str(status))
if status:
log.error('rm status = ' + str(status))
return status
log.debug('command status = ' + str(status))
if status != 0:
log.info('preprocessing FAILED')
return status
)
# loadup
loadup.loadup()
# -------
# Loopit
exe_loop = True
while exe_loop:
# initial pull
repo_pull(home_dir, main_dir)
# run clean up
# looking for .DS_Store nuggets
cleanup.cleanup(main_dir, cur_branch_name)
# ==================
# Determine To-Do's
# ==================
# current branch
print('\nBRANCH: %s\n' % cur_branch_name)
time.sleep(.1)
# get projects
proj_path = '%s/%s' % (main_dir, cur_branch_name)
try:
proj_list = next(os.walk(proj_path))[1]
except StopIteration:
proj_list = []
def start():
#exit codes:
# 0: OK, no errors
# 1: (system) errors
# 2: bots ran OK, but there are errors/process errors in the run
# 3: Database is locked, but "maxruntime" has not been exceeded.
#********command line arguments**************************
commandspossible = [
'--new', '--retry', '--retransmit', '--cleanup', '--crashrecovery',
'--retrycommunication', '--automaticretrycommunication'
]
commandstorun = []
routestorun = [] #list with routes to run
configdir = 'config'
for arg in sys.argv[1:]:
if not arg:
continue
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg in ["?", "/?"] or arg.startswith('-'):
showusage()
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun: #if no command on command line, use new (default)
commandstorun = ['--new']
#**************init general: find locating of bots, configfiles, init paths etc.****************
botsinit.generalinit(configdir)
#set current working directory to botspath
#~ old_current_directory = os.getcwdu()
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************initialise logging******************************
try:
botsinit.initenginelogging()
except:
print _('Error in initialising logging system.')
traceback.print_exc()
sys.exit(1)
else:
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(): #log start info
botsglobal.logger.info(u'%s: "%s".', key, value)
#**************connect to database**********************************
try:
botsinit.connect()
except:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py.'
))
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#initialise user exits for the whole bots-engine (this script file)
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except ImportError:
userscript = scriptname = None
#**************handle database lock****************************************
#try to set a lock on the database; if this is not possible, the database is already locked. Either:
#1 another instance bots bots-engine is (still) running
#2 or bots-engine had a severe crash.
#What to do?
#first: check ts of database lock. If below a certain value (set in bots.ini) we assume an other instance is running. Exit quietly - no errors, no logging.
# else: Warn user, give advise on what to do. gather data: nr files in, errors.
#next: warn with report & logging. advise a crashrecovery.
if not botslib.set_database_lock():
if '--crashrecovery' in commandstorun: #user starts recovery operation; the databaselock is ignored; the databaselock is unlocked when routes have run.
commandstorun = ['--crashrecovery'] #is an exclusive option!
else:
#when scheduling bots it is possible that the last run is still running. Check if maxruntime has passed:
vanaf = datetime.datetime.today() - datetime.timedelta(
minutes=botsglobal.ini.getint('settings', 'maxruntime', 60))
for row in botslib.query(
'''SELECT ts FROM mutex WHERE ts < %(vanaf)s ''',
{'vanaf': vanaf}):
warn = _(
u'!Bots database is locked!\nBots-engine has ended in an unexpected way during the last run.\nThis happens, but is very very rare.\nPossible causes: bots-engine terminated by user, system crash, power-down, etc.\nA forced retry of the last run is advised; bots will (try to) repair the last run.'
)
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
#add: count errors etc.
sys.exit(1)
else: #maxruntime has not passed. Exit silently, nothing reported
botsglobal.logger.info(
_(u'Database is locked, but "maxruntime" has not been exceeded.'
))
sys.exit(3)
else:
if '--crashrecovery' in commandstorun: #user starts recovery operation but there is no databaselock.
warn = _(
u'User started a forced retry of the last run.\nOnly use this when the database is locked.\nThe database was not locked (database is OK).\nSo Bots has done nothing now.'
)
botsglobal.logger.error(warn)
botslib.sendbotserrorreport(
_(u'[Bots Error Report] User started a forced retry of last run, but this was not needed'
), warn)
botslib.remove_database_lock()
sys.exit(1)
#*************get list of routes to run****************************************
#~ raise Exception('locked database') #for testing database lock: abort, database will be locked
if routestorun:
botsglobal.logger.info(u'Run routes from command line: "%s".',
str(routestorun))
else: # no routes from command line parameters: fetch all active routes from database
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
routestorun.append(row['idroute'])
botsglobal.logger.info(_(u'Run active routes from database: "%s".'),
str(routestorun))
#routestorun is now either a list with routes from commandline, or the list of active routes for the routes table in the db.
#**************run the routes for retry, retransmit and new runs*************************************
try:
#commandstorun determines the type(s) of run
#routes to run is a listof the routes that are runs (for each command to run
#botsglobal.incommunicate is used to control if there is communication in; only 'new' incommunicates.
#botsglobal.minta4query controls which ta's are queried by the routes.
#stuff2evaluate controls what is evaluated in automatic maintenance.
errorinrun = 0 #detect if there has been some error. Only used for good exit() code
botsglobal.incommunicate = False
if '--crashrecovery' in commandstorun:
botsglobal.logger.info(_(u'Run crash recovery.'))
stuff2evaluate = botslib.set_minta4query_crashrecovery()
if stuff2evaluate:
router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--crashrecovery', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'No retry of the last run - there was no last run.'))
if userscript and hasattr(userscript, 'postcrashrecovery'):
botslib.runscript(userscript,
scriptname,
'postcrashrecovery',
routestorun=routestorun)
if '--retrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run communication retry.'))
stuff2evaluate = router.routedispatcher(routestorun,
'--retrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run recommunicate: nothing to recommunicate.'))
if userscript and hasattr(userscript, 'postretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postretrycommunication',
routestorun=routestorun)
if '--automaticretrycommunication' in commandstorun:
botsglobal.logger.info(_(u'Run automatic communication retry.'))
stuff2evaluate = router.routedispatcher(
routestorun, '--automaticretrycommunication')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--automaticretrycommunication', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run automatic recommunicate: nothing to recommunicate.'
))
if userscript and hasattr(userscript,
'postautomaticretrycommunication'):
botslib.runscript(userscript,
scriptname,
'postautomaticretrycommunication',
routestorun=routestorun)
if '--retry' in commandstorun:
botsglobal.logger.info(u'Run retry.')
stuff2evaluate = router.routedispatcher(routestorun, '--retry')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retry', stuff2evaluate)
else:
botsglobal.logger.info(_(u'Run retry: nothing to retry.'))
if userscript and hasattr(userscript, 'postretry'):
botslib.runscript(userscript,
scriptname,
'postretry',
routestorun=routestorun)
if '--retransmit' in commandstorun:
botsglobal.logger.info(u'Run retransmit.')
stuff2evaluate = router.routedispatcher(routestorun,
'--retransmit')
if stuff2evaluate:
errorinrun += automaticmaintenance.evaluate(
'--retransmit', stuff2evaluate)
else:
botsglobal.logger.info(
_(u'Run retransmit: nothing to retransmit.'))
if userscript and hasattr(userscript, 'postretransmit'):
botslib.runscript(userscript,
scriptname,
'postretransmit',
routestorun=routestorun)
if '--new' in commandstorun:
botsglobal.logger.info('Run new.')
botsglobal.incommunicate = True
botsglobal.minta4query = 0 #meaning: reset. the actual value is set later (in routedispatcher)
stuff2evaluate = router.routedispatcher(routestorun)
errorinrun += automaticmaintenance.evaluate(
'--new', stuff2evaluate)
if userscript and hasattr(userscript, 'postnewrun'):
botslib.runscript(userscript,
scriptname,
'postnewrun',
routestorun=routestorun)
if '--cleanup' in commandstorun or botsglobal.ini.get(
'settings', 'whencleanup', 'always') == 'always':
botsglobal.logger.debug(u'Do cleanup.')
cleanup.cleanup()
botslib.remove_database_lock()
except Exception, e:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%s') %
(e)) #of course this 'should' not happen.
sys.exit(1)
def start():
''' sysexit codes:
0: OK, no errors
1: (system) errors incl parsing of command line arguments
2: bots ran OK, but there are errors/process errors in the run
3: Database is locked, but "maxruntime" has not been exceeded.
'''
#NOTE: bots directory should always be on PYTHONPATH - otherwise it will not start.
#********command line arguments**************************
usage = '''
This is "%(name)s" version %(version)s, part of Bots open source edi translator (http://bots.sourceforge.net).
Does the actual translations and communications; it's the workhorse. It does not have a fancy interface.
Usage:
%(name)s [run-options] [config-option] [routes]
Run-options (can be combined):
--new receive new edi files (default: if no run-option given: run as new).
--resend resend as indicated by user.
--rereceive rereceive as indicated by user.
--automaticretrycommunication - automatically retry outgoing communication.
--cleanup remove older data from database.
Config-option:
-c<directory> directory for configuration files (default: config).
Routes: list of routes to run. Default: all active routes (in the database)
''' % {
'name': os.path.basename(sys.argv[0]),
'version': botsglobal.version
}
configdir = 'config'
commandspossible = [
'--automaticretrycommunication', '--resend', '--rereceive', '--new'
]
commandstorun = []
routestorun = [] #list with routes to run
do_cleanup_parameter = False
for arg in sys.argv[1:]:
if arg.startswith('-c'):
configdir = arg[2:]
if not configdir:
print 'Error: configuration directory indicated, but no directory name.'
sys.exit(1)
elif arg in commandspossible:
commandstorun.append(arg)
elif arg == '--cleanup':
do_cleanup_parameter = True
elif arg in ["?", "/?", '-h', '--help'] or arg.startswith('-'):
print usage
sys.exit(0)
else: #pick up names of routes to run
routestorun.append(arg)
if not commandstorun and not do_cleanup_parameter: #if no command on command line, use new (default)
commandstorun = ['--new']
commandstorun = [
command[2:] for command in commandspossible if command in commandstorun
] #sort commands
#***********end handling command line arguments**************************
botsinit.generalinit(
configdir) #find locating of bots, configfiles, init paths etc.
#set working directory to bots installation. advantage: when using relative paths it is clear that this point paths within bots installation.
os.chdir(botsglobal.ini.get('directories', 'botspath'))
#**************check if another instance of bots-engine is running/if port is free******************************
try:
engine_socket = botslib.check_if_other_engine_is_running()
except socket.error:
sys.exit(3)
else:
atexit.register(engine_socket.close)
#**************initialise logging******************************
process_name = 'engine'
botsglobal.logger = botsinit.initenginelogging(process_name)
atexit.register(logging.shutdown)
for key, value in botslib.botsinfo(
): #log info about environement, versions, etc
botsglobal.logger.info(u'%(key)s: "%(value)s".', {
'key': key,
'value': value
})
#**************connect to database**********************************
try:
botsinit.connect()
except Exception as msg:
botsglobal.logger.exception(
_(u'Could not connect to database. Database settings are in bots/config/settings.py. Error: "%(msg)s".'
), {'msg': msg})
sys.exit(1)
else:
botsglobal.logger.info(_(u'Connected to database.'))
atexit.register(botsglobal.db.close)
#************initialise user exits for the whole bots-engine*************************
try:
userscript, scriptname = botslib.botsimport('routescripts',
'botsengine')
except botslib.BotsImportError: #userscript is not there; other errors like syntax errors are not catched
userscript = scriptname = None
#***acceptance tests: initialiase acceptance user script******************************
acceptance_userscript = acceptance_scriptname = None
if botsglobal.ini.getboolean('acceptance', 'runacceptancetest', False):
botsglobal.logger.info(
_(u'This run is an acceptance test - as indicated in option "runacceptancetest" in bots.ini.'
))
try:
acceptance_userscript, acceptance_scriptname = botslib.botsimport(
'routescripts', 'bots_acceptancetest')
except botslib.BotsImportError:
botsglobal.logger.info(
_(u'In acceptance test there is no script file "bots_acceptancetest.py" to check the results of the acceptance test.'
))
#**************handle database lock****************************************
#set a lock on the database; if not possible, the database is locked: an earlier instance of bots-engine was terminated unexpectedly.
if not botslib.set_database_lock():
#for SQLite: do a integrity check on the database
if botsglobal.settings.DATABASES['default'][
'ENGINE'] == 'django.db.backends.sqlite3':
cursor = botsglobal.db.execute('''PRAGMA integrity_check''')
result = cursor.fetchone()
if result[0] != u'ok':
warn = _(u'!Bots database is locked!\n'\
'Bots did an integrity check on the database, but database was not OK.\n'\
'Manual action is needed!\n'\
'Bots has stopped processing EDI files.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is damaged'), warn)
sys.exit(1)
warn = _(u'!Bots database is locked!\n'\
'Bots-engine has ended in an unexpected way during the last run.\n'\
'Most likely causes: sudden power-down, system crash, problems with disk I/O, bots-engine terminated by user, etc.\n'
'Bots will do an automatic crash recovery now.')
botsglobal.logger.critical(warn)
botslib.sendbotserrorreport(
_(u'[Bots severe error]Database is locked'), warn)
commandstorun.insert(
0, 'crashrecovery'
) #there is a database lock. Add a crashrecovery as first command to run.
atexit.register(botslib.remove_database_lock)
warnings.simplefilter('error', UnicodeWarning)
#**************run the routes**********************************************
#commandstorun determines the type(s) of run. eg: ['automaticretrycommunication','new']
try:
botslib.prepare_confirmrules()
#in acceptance tests: run a user script before running eg to clean output directories******************************
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'pretest',
routestorun=routestorun)
botslib.tryrunscript(userscript,
scriptname,
'pre',
commandstorun=commandstorun,
routestorun=routestorun)
errorinrun = 0 #detect if there has been some error. Only used for correct exit() code
first_command_2_run = True
for command in commandstorun:
#if multiple commands in run: reports etc are based on timestamp; so there needs to be at least one second between these runs.
if first_command_2_run:
first_command_2_run = False
else:
time.sleep(1)
botsglobal.logger.info(_(u'Run "%(command)s".'),
{'command': command})
#************get list of routes to run*******************************
if routestorun:
use_routestorun = routestorun[:]
botsglobal.logger.info(
_(u'Run routes from command line: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
elif command == 'new': #fetch all active routes from database unless 'not in default run' or not active.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
AND (notindefaultrun=%(notindefaultrun)s OR notindefaultrun IS NULL)
ORDER BY idroute ''', {
'active': True,
'notindefaultrun': False
}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run active routes from database that are in default run: "%(routes)s".'
), {'routes': unicode(use_routestorun)})
else: #for command other than 'new': use all active routes.
use_routestorun = []
for row in botslib.query(
'''SELECT DISTINCT idroute
FROM routes
WHERE active=%(active)s
ORDER BY idroute ''',
{'active': True}):
use_routestorun.append(row['idroute'])
botsglobal.logger.info(
_(u'Run all active routes from database: "%(routes)s".'),
{'routes': unicode(use_routestorun)})
#************run routes for this command******************************
botslib.tryrunscript(userscript,
scriptname,
'pre' + command,
routestorun=use_routestorun)
errorinrun += router.rundispatcher(command, use_routestorun)
botslib.tryrunscript(userscript,
scriptname,
'post' + command,
routestorun=use_routestorun)
#*********finished running routes for this command****************************
#*********finished all commands****************************************
botslib.tryrunscript(userscript,
scriptname,
'post',
commandstorun=commandstorun,
routestorun=routestorun)
try: #in acceptance tests: run a user script. no good reporting of errors/results in post-test script. Reason: this is after automaticmaintence.
botslib.tryrunscript(acceptance_userscript,
acceptance_scriptname,
'posttest',
routestorun=use_routestorun)
except Exception as msg:
print unicode(msg)
cleanup.cleanup(do_cleanup_parameter, userscript, scriptname)
except Exception as msg:
botsglobal.logger.exception(
_(u'Severe error in bots system:\n%(msg)s'),
{'msg': unicode(msg)}) #of course this 'should' not happen.
sys.exit(1)
else:
if errorinrun:
sys.exit(2) #indicate: error(s) in run(s)
else:
sys.exit(0) #OK
if len(pending_devices) == 0:
break
for pnd_device in pending_devices:
if pnd_device['device_model'] != device_info["device_model"]:
main_logger.error(
"model mismatch between discovered device and attempted to discover devices. Found model:"
+ str(pnd_device['device_model']))
raise Exception("model mismatch of discovered devices")
return pending_devices # we only process the first building
recheck_count -= 1
time.sleep(5)
main_logger.error("No devices found to have been discovered")
raise Exception("Devices could not be discovered")
if __name__ == "__main__":
cleanup.cleanup()
try:
device_info = test_settings.DEVICE_INFO["RTH8580WF"]
token = getToken.login(test_settings.testusername,
test_settings.testuserpassword)
discDevices = discoverDevice(token, device_info)
print(discDevices)
print(len(discDevices))
except:
print(getErrorInfo())
pass
cleanup.cleanup()
def main(args):
"""Preprocess a font for use as a TachyFont.
Args:
args: list, command line arguments.
Raises:
ValueError: if build directory cannot be created
Returns:
Status of the operation.
"""
parser = argparse.ArgumentParser(prog='pyprepfnt')
parser.add_argument('fontfile', help='Input font file')
parser.add_argument('output_dir', help='Output directory')
parser.add_argument(
'--force',
default=False,
action='store_true',
help='Force preprocessing even if the timestamps indicate'
' it is not necessary')
parser.add_argument('--hinting',
default=False,
action='store_true',
help='Retain hinting if set, else strip hinting')
parser.add_argument('--reuse_clean',
default=False,
action='store_true',
help='Reuse the "clean" file if possible')
parser.add_argument('--log',
default='WARNING',
help='Set the logging level; eg, --log=INFO')
parser.add_argument('--verbose',
default=False,
action='store_true',
help='Report internal operations')
cmd_args = parser.parse_args(args)
loglevel = getattr(logging, cmd_args.log.upper(), None)
if not isinstance(loglevel, int):
raise ValueError('Invalid log level: %s' % loglevel)
log = logging.getLogger()
logging_handler = logging.StreamHandler(sys.stdout)
logging_handler.setLevel(loglevel)
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
logging_handler.setFormatter(formatter)
log.addHandler(logging_handler)
log.setLevel(loglevel)
verbose = cmd_args.verbose
force_preprocessing = cmd_args.force
log.debug('force_preprocessing = ' + str(force_preprocessing))
fontfile = cmd_args.fontfile
fonttime = os.path.getmtime(fontfile)
# TODO(bstell) use Logger
basename = os.path.basename(fontfile)
log.info('preprocess %s = %d bytes' %
(cmd_args.fontfile, os.path.getsize(cmd_args.fontfile)))
filename, extension = os.path.splitext(basename)
cur_time = datetime.datetime.now()
build_dir = 'tmp-%s' % filename
if not cmd_args.reuse_clean:
build_dir = (
'%s-%04d-%02d-%02d-%02d-%02d-%02d.%d' %
(build_dir, cur_time.year, cur_time.month, cur_time.day,
cur_time.hour, cur_time.minute, cur_time.second, os.getpid()))
output_dir = cmd_args.output_dir
log.debug('TAR file: ' + output_dir)
try:
os.makedirs(build_dir)
except OSError as exception:
if exception.errno != errno.EEXIST:
log.error('failed to create build_dir (' + build_dir + ')')
raise
log.debug(
'if reuse_clean then we should compare the source font and final tar')
cleanfile = filename + '_clean' + extension
cleanfilepath = build_dir + '/' + cleanfile
# Decide if we are building the cleaned up version of the font.
rebuild_clean = not cmd_args.reuse_clean
cleanfile_exists = os.path.isfile(cleanfilepath)
if force_preprocessing or not cleanfile_exists:
rebuild_clean = True
else:
cleantime = os.path.getmtime(cleanfilepath)
if cleantime <= fonttime:
rebuild_clean = True
log.debug('rebuild_clean = ' + str(rebuild_clean))
if rebuild_clean:
log.debug('cleaned version: ' + cleanfilepath)
cleanup.cleanup(fontfile, cmd_args.hinting, cleanfilepath, verbose)
closure.dump_closure_map(cleanfilepath, build_dir)
else:
log.debug('reuse cleaned up version: ' + cleanfilepath)
# Get the latest cleaned up font timestamp.
cleantime = os.path.getmtime(cleanfilepath)
# Decide if we are rebuilding the tar file.
tachyfont_file = filename + '.TachyFont.tar'
tarfilepath = build_dir + '/' + tachyfont_file
rebuild_tar = False
tarfile_exists = os.path.isfile(tarfilepath)
log.debug('file %s exists: %s' % (tarfilepath, tarfile_exists))
if force_preprocessing or not tarfile_exists:
rebuild_tar = True
else:
tartime = os.path.getmtime(tarfilepath)
if tartime <= cleantime:
rebuild_tar = True
log.debug('rebuild_tar = ' + str(rebuild_tar))
if rebuild_tar:
log.debug('start proprocess')
preprocess = Preprocess(cleanfilepath, build_dir, verbose)
log.debug('build base')
preprocess.base_font()
log.debug('dump cmap')
preprocess.cmap_dump()
log.debug('build glyph data')
preprocess.serial_glyphs()
log.debug('write sha-1 fingerprint')
preprocess.sha1_fingerprint()
log.debug('create tar file')
sub_files = (
'base closure_data closure_idx codepoints gids glyph_data '
'glyph_table sha1_fingerprint')
tar_cmd = 'cd %s; tar cf %s %s' % (build_dir, tachyfont_file,
sub_files)
log.debug('tar_cmd: ' + tar_cmd)
status = os.system(tar_cmd)
log.debug('tar command status: ' + str(status))
if status:
log.error('tar command status: ' + str(status))
return status
else:
log.debug('no need to rebuild intermediate tar file: ' + tarfilepath)
# Get the latest cleaned up tar timestamp.
tartime = os.path.getmtime(tarfilepath)
# Decide if we are copying over the tar file.
copy_tar = False
tarcopy_filepath = output_dir + '/' + tachyfont_file
tarcopy_exists = os.path.isfile(tarcopy_filepath)
if force_preprocessing or not tarcopy_exists:
copy_tar = True
else:
tarcopytime = os.path.getmtime(tarcopy_filepath)
if tarcopytime <= tartime:
copy_tar = True
log.debug('copy_tar = ' + str(copy_tar))
if copy_tar:
log.debug('cp the files to the output directory')
log.info('cleaned: %s = %d' %
(cleanfile, os.path.getsize(cleanfilepath)))
log.info('Tar: %s/%s' % (output_dir, tachyfont_file))
cp_cmd = (
'cp %s/%s %s/%s %s' %
(build_dir, tachyfont_file, build_dir, cleanfile, output_dir))
log.debug('cp_cmd: ' + cp_cmd)
status = os.system(cp_cmd)
log.debug('cp status ' + str(status))
if status:
log.error('cp status = ' + str(status))
return status
else:
log.debug('the existing tar file is up to date: ' + tarfilepath)
if cmd_args.reuse_clean:
log.debug('leaving the build directory: ' + build_dir)
status = 0
else:
log.debug('cleanup the build directory')
rm_cmd = ('rm -rf %s' % build_dir)
log.debug('rm_cmd: ' + rm_cmd)
status = os.system(rm_cmd)
log.debug('rm status ' + str(status))
if status:
log.error('rm status = ' + str(status))
return status
log.debug('command status = ' + str(status))
if status != 0:
log.info('preprocessing FAILED')
return status
'''
from init import init
from renameRef import renameRef
from copy_sch import copy_sch
from copy_pcb import copy
from cleanup import cleanup
from init import tempdir as tmpDir
projectPath= "/Users/mldelibero/case/masters/hightemp/board/siggen/"
#projectPath= "/Users/mldelibero/case/masters/stressCap/buckDemo/board/"
#projectPath= "./ex1/"
schematic = "temp.sch"
layout = "temp.pcb"
sub_schematic = "siggen.sch"
sub_layout = "gen.pcb"
numCop = 4 # Total number of instances desired in the overall project
def getinfo():
"""Return the variables initialized above"""
return tuple([projectPath,schematic,layout,sub_schematic,sub_layout,numCop])
if __name__ == "__main__":
tempPath = init(schematic,layout,sub_schematic,sub_layout,numCop,projectPath)
renameRef(sub_schematic,sub_layout,numCop,tmpDir)
copy_sch(schematic,sub_schematic,numCop,projectPath,tmpDir)
copy(layout,sub_layout,numCop,projectPath,tmpDir)
cleanup(tempPath)
from flask import Flask, render_template
from markov import MarkovChain
from cleanup import cleanup
app = Flask(__name__)
source_text = cleanup("trump.txt")
markov = MarkovChain(source_text)
@app.route('/')
def index():
return render_template("index.html", markov=markov)
def results(self):
"""Return appstats results in a dictionary for template context."""
if not self.recorder:
# If appstats fails to initialize for any reason, return an empty
# set of results.
logging.warn("Missing recorder for appstats profiler.")
return {"calls": [], "total_time": 0}
total_call_count = 0
total_time = 0
calls = []
service_totals_dict = {}
likely_dupes = False
end_offset_last = 0
requests_set = set()
appstats_key = long(self.recorder.start_timestamp * 1000)
for trace in self.recorder.traces:
total_call_count += 1
total_time += trace.duration_milliseconds()
# Don't accumulate total RPC time for traces that overlap asynchronously
if trace.start_offset_milliseconds() < end_offset_last:
total_time -= end_offset_last - trace.start_offset_milliseconds()
end_offset_last = trace.start_offset_milliseconds() + trace.duration_milliseconds()
service_prefix = trace.service_call_name()
if "." in service_prefix:
service_prefix = service_prefix[: service_prefix.find(".")]
if service_prefix not in service_totals_dict:
service_totals_dict[service_prefix] = {"total_call_count": 0, "total_time": 0, "total_misses": 0}
service_totals_dict[service_prefix]["total_call_count"] += 1
service_totals_dict[service_prefix]["total_time"] += trace.duration_milliseconds()
stack_frames_desc = []
for frame in trace.call_stack_list():
stack_frames_desc.append(
"%s:%s %s"
% (util.short_rpc_file_fmt(frame.class_or_file_name()), frame.line_number(), frame.function_name())
)
request = trace.request_data_summary()
response = trace.response_data_summary()
likely_dupe = request in requests_set
likely_dupes = likely_dupes or likely_dupe
requests_set.add(request)
request_short = request_pretty = None
response_short = response_pretty = None
miss = 0
try:
request_object = unformatter.unformat(request)
response_object = unformatter.unformat(response)
request_short, response_short, miss = cleanup.cleanup(request_object, response_object)
request_pretty = pformat(request_object)
response_pretty = pformat(response_object)
except Exception, e:
logging.warning(
"Prettifying RPC calls failed.\n%s\nRequest: %s\nResponse: %s", e, request, response, exc_info=True
)
service_totals_dict[service_prefix]["total_misses"] += miss
calls.append(
{
"service": trace.service_call_name(),
"start_offset": util.milliseconds_fmt(trace.start_offset_milliseconds()),
"total_time": util.milliseconds_fmt(trace.duration_milliseconds()),
"request": request_pretty or request,
"response": response_pretty or response,
"request_short": request_short or cleanup.truncate(request),
"response_short": response_short or cleanup.truncate(response),
"stack_frames_desc": stack_frames_desc,
"likely_dupe": likely_dupe,
}
)
import siRnaPredict as si import cleanup import subprocess peakFName = 'small.peak.data' smallUpdateFN = 'small.degradome' cleanFN = 'small.degradome.small' exonUpdateFN = 'small.degradome.small.clean' mismatchFN = 'small.degradome.small.clean.degOverlap' clean2FN = 'small.degradome.small.clean.degOverlap.mismatch' sortFN = 'small.degradome.small.clean.degOverlap.mismatch.clean' smallDirectory = '/home/chrisgre/smallLibs/siRNA/small/sortedChroms' print 'Getting highest level, offset, middle level' si.predictSiRNA(peakFName, 'siDegradome.conf') print 'Getting small expression level' si.updateSmallExpression(smallUpdateFN, 'siPeaks.conf') print 'cleaning' cleanup.cleanup(cleanFN) print 'Getting transcript level' si.exonOverlap(exonUpdateFN) print 'Getting mismatch levels' si.markMismatches(mismatchFN, smallDirectory) print 'Cleaning up again, and sorting' cleanup.cleanup2(clean2FN) subprocess.Popen(['./sortIt.sh', sortFN], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
