def receive_names_in_path(_path):
all_names = []
for tree in fetch_trees_from_path(_path):
all_names.extend(find_all_names_in_tree(tree))
return flat([split_snake_case_to_words(name)
for name in all_names
if not is_magic_name(name)])
async def main(username, password, start, end, output, njobs):
tasks = []
data = []
async with ClientSession(connector=TCPConnector(ssl=False)) as session:
extractor = LivetexExtractor(
username,
password,
start,
end,
session,
concurrency_level=njobs,
)
logging.info('Logging in with the given credentials...')
await extractor.login(session)
logging.info('Successful login!')
logging.info('Fetching employee list...')
await extractor.get_employees(session)
logging.info('Employee list received!')
topics_short = await extractor.get_dialogs_short()
if not topics_short:
return
topics_count = len(topics_short)
logging.info('Fetching %i dialogs...', topics_count)
for topic in topics_short:
task = asyncio.create_task(
extractor.get_dialog_info(topic['topicId']))
tasks.append(task)
try:
for res in tqdm(asyncio.as_completed(tasks), total=topics_count):
data.append(await res)
finally:
data = flat(data)
dict_writer = csv.DictWriter(output, data[0].keys())
dict_writer.writeheader()
dict_writer.writerows(
sorted(data, key=lambda x: x['dialog_start'] + x['timestamp']))
nets += [(net, net_create)]
print("Name: ", net_create.__name__)
print("Number of Neurons (relus): ", net.neuronCount())
print("Number of Parameters: ",
sum([h.product(s.size()) for s in net.parameters()]))
print("Depth (relu layers): ", net.depth())
print()
net.showNet()
print()
if args.domain == []:
models = [createModel(net, goals.Box(args.width), "Box") for net in nets]
else:
models = h.flat([[
createModel(net, h.parseValues(d, goals, scheduling), h.catStrs(d))
for net in nets
] for d in args.domain])
patience = 30
last_best_origin = 0
best_origin = 1e10
last_best = 0
best = 1e10
decay = True
with h.mopen(args.dont_write, os.path.join(out_dir, "log.txt"), "w") as f:
startTime = timer()
for epoch in range(1, args.epochs + 1):
if f is not None:
f.flush()
if (epoch - 1) % args.test_freq == 0 and (epoch > 1
or args.test_first):
nets += [getattr(models, n) for n in args.net]
nets = [(n(num_classes).infer(input_dims), n) for n in nets]
for net, net_create in nets:
print("Name: ", net_create.__name__)
print("Number of Neurons (relus): ", net.neuronCount())
print("Number of Parameters: ",
sum([h.product(s.size()) for s in net.parameters()]))
print()
if args.domain == []:
models = [createModel(net, domains.Box, "Box") for net in nets]
else:
models = h.flat(
[[createModel(net, getattr(domains, d), d) for net in nets]
for d in args.domain])
def adjust_learning_rate(optimizer, epoch):
lr = args.lr
if epoch >= 0.85 * args.epochs:
lr = args.lr * 0.01
elif epoch >= 0.7 * args.epochs:
lr = args.lr * 0.1
for param_group in optimizer.param_groups:
param_group['lr'] = lr
with open(os.path.join(out_dir, "log.txt"), "w") as f:
m = getattr(models,n)
net_create = (lambda m: lambda: buildNet(m))(m) # why doesn't python do scoping right? This is a thunk. It is bad.
net_create.__name__ = n
net = buildNet(m)
net.__name__ = n
nets += [ (net, net_create) ]
print("Name: ", net_create.__name__)
print("Number of Neurons (relus): ", net.neuronCount())
print("Number of Parameters: ", sum([h.product(s.size()) for s in net.parameters()]))
print()
if args.domain == []:
models = [ createModel(net, domains.Box(args.width), "Box") for net in nets]
else:
models = h.flat([[createModel(net, h.parseValues(domains,d), h.catStrs(d)) for net in nets] for d in args.domain])
with h.mopen(args.dont_write, os.path.join(out_dir, "log.txt"), "w") as f:
startTime = timer()
for epoch in range(1, args.epochs + 1):
if (epoch - 1) % args.test_freq == 0:
with Timer("test before epoch "+str(epoch),"sample", 10000):
test(models, epoch, f)
h.printBoth("Elapsed-Time: {:.2f}s\n".format(timer() - startTime), f = f)
if args.epochs <= args.test_freq:
break
with Timer("train","sample", 60000):
train(epoch, models)
def receive_function_verbs_in_path(_path):
return flat([extract_verbs_from_snake_case(function_name)
for function_name
in receive_function_names_in_path(_path)])