def main(data_txt_path, label_txt_path, stride=25,
images_folder='roadC621/'):
"""
Train a neural network with patches of patch_size x patch_size
(as given via the module network_path).
Parameters
----------
network_path : str
Path to a Python script with a function generate_nnet(feats) which
returns a neural network
image_batch_size : int
stride : int
"""
assert image_batch_size >= 1
assert stride >= 1
features, labels = load_data_raw_images(train_images_folder=images_folder)
mem_size = (sys.getsizeof(42) * len(features) * features[0].size +
sys.getsizeof(42) * len(labels) * labels[0].size)
logging.info("Loaded %i data images with their labels (approx %s)",
len(features),
utils.sizeof_fmt(mem_size))
nn_params = {'training': {'image_batch_size': image_batch_size,
'stride': stride}}
logging.info("## Network: %s", network_path)
network = imp.load_source('sst.network', network_path)
logging.info("Fully network: %s", str(network.fully))
nn_params['code'] = inspect.getsource(network)
nn_params['fully'] = network.fully
nn_params['patch_size'] = network.patch_size
assert nn_params['patch_size'] > 0
labeled_patches = get_patches(features[:1],
labels[:1],
nn_params=nn_params)
feats, _ = get_features(labeled_patches, fully=nn_params['fully'])
net1 = network.generate_nnet(feats)
for block in range(0, len(features), image_batch_size):
from_img = block
to_img = block + image_batch_size
logging.info("Training on batch %i - %i of %i total",
from_img,
to_img,
len(features))
labeled_patches = get_patches(features[from_img:to_img],
labels[from_img:to_img],
nn_params=nn_params,
stride=stride)
logging.info(("labeled_patches[0].shape: %s , "
"labeled_patches[1].shape: %s"),
labeled_patches[0].shape,
labeled_patches[1].shape)
net1 = train_nnet(labeled_patches, net1, fully=nn_params['fully'])
model_pickle_name = 'nnet1-trained.pickle'
utils.serialize_model(net1,
filename=model_pickle_name,
parameters=nn_params)
def create_contact_person():
xero_tenant_id = get_xero_tenant_id()
accounting_api = AccountingApi(api_client)
contact_person = ContactPerson(
first_name="John",
last_name="Smith",
email_address="*****@*****.**",
include_in_emails=True,
)
contact = Contact(
name="FooBar",
first_name="Foo",
last_name="Bar",
email_address="*****@*****.**",
contact_persons=[contact_person],
)
contacts = Contacts(contacts=[contact])
try:
created_contacts = accounting_api.create_contacts(
xero_tenant_id, contacts=contacts) # type: Contacts
except AccountingBadRequestException as exception:
sub_title = "Error: " + exception.reason
code = jsonify(exception.error_data)
else:
sub_title = "Contact {} created.".format(
getvalue(created_contacts, "contacts.0.name", ""))
code = serialize_model(created_contacts)
return render_template("code.html",
title="Create Contacts",
code=code,
sub_title=sub_title)
def get_invoices():
xero_tenant_id = get_xero_tenant_id()
accounting_api = AccountingApi(api_client)
invoices = accounting_api.get_invoices(xero_tenant_id,
statuses=["DRAFT", "SUBMITTED"])
code = serialize_model(invoices)
sub_title = "Total invoices found: {}".format(len(invoices.invoices))
return render_template("code.html",
title="Invoices",
code=code,
sub_title=sub_title)
async def run_network_instances(args):
num_instances = NUMBER_NETWORK_INSTANCES
activ = 'relu'
if 'instances' in args:
num_instances = args['instances']
if 'activation' in args:
activ = args['activation']
await spawn_network_instances(num_instances)
dpinfo = await pulsar.send('data_provider', 'data_provider_info')
input_format = 'hot_vector'
output_format = 'categorical'
input_length = dpinfo['model_params']['input_length'][input_format]
output_length = dpinfo['model_params']['output_length'][output_format]
if NETWORK_STATE['model'] is None:
NETWORK_STATE['input_length'] = input_length
NETWORK_STATE['output_length'] = output_length
NETWORK_STATE['activation'] = activ
model = create_model(input_length, output_length, activation=activ)
logging.info(
'NETWORK SUPER-MODEL INIT (size = {}, X:{}->Y:{})'.format(model_sizeMB(serialize_model(model)),
input_length, output_length))
NETWORK_STATE['model'] = serialize_model(model)
instance_args = dict(args, **{'model': NETWORK_STATE['model'], 'in_len': input_length,
'out_len': output_length, 'activation': activ,
'input_format': input_format, 'output_format': output_format})
#logging.warning(instance_args)
for (aid, inst) in NETWORK_STATE['instances'].items():
asyncio.ensure_future(
pulsar.send(aid, 'run', run_network_instance, args=instance_args)
)
def journals_list():
xero_tenant_id = get_xero_tenant_id()
accounting_api = AccountingApi(api_client)
journals = accounting_api.get_journals(
xero_tenant_id=xero_tenant_id,
if_modified_since=datetime.datetime.now() - relativedelta(days=90))
return render_template(
"code.html",
title="Journals",
code=serialize_model(journals),
sub_title="For Past 90 days",
)
def members_list():
xero_tenant_id = get_xero_tenant_id()
accounting_api = AccountingApi(api_client)
sources = defaultdict(list)
member_transactions = defaultdict(list)
# Walk the full journal for the past 90 days for membership items
# and split them based on which type they are, i.e. 'CASHREC' / 'ACCREC'
for journal in get_journals(datetime.datetime.now() -
relativedelta(days=90)):
if is_membership_journal(journal):
sources[journal.source_type].append(journal.source_id)
transactions = []
for source, source_ids in sources.items():
if source == 'CASHREC':
for s in source_ids:
transactions.extend(
# get_bank_transactions has to be done individually but there's usually not many of them
accounting_api.get_bank_transaction(xero_tenant_id,
s).bank_transactions)
elif source == 'ACCREC':
transactions.extend(
accounting_api.get_invoices(xero_tenant_id,
i_ds=source_ids).invoices)
else:
raise ValueError(source)
for transaction in transactions:
c = transaction.contact.contact_id
member_transactions[c].append(transaction)
members = accounting_api.get_contacts(
xero_tenant_id, i_ds=list(member_transactions.keys())).contacts
contact_sheet = []
for m in members:
contact_sheet.append({
**fix_contact(m), 'transactions':
len(member_transactions[m.contact_id])
})
return render_template(
"code.html",
title="Members",
code=serialize_model(contact_sheet),
sub_title="For Past 90 days",
)
def create_multiple_contacts():
xero_tenant_id = get_xero_tenant_id()
accounting_api = AccountingApi(api_client)
contact = Contact(
name="George Jetson",
first_name="George",
last_name="Jetson",
email_address="*****@*****.**",
)
# Add the same contact twice - the first one will succeed, but the
# second contact will fail with a validation error which we'll show.
contacts = Contacts(contacts=[contact, contact])
try:
created_contacts = accounting_api.create_contacts(
xero_tenant_id, contacts=contacts,
summarize_errors=False) # type: Contacts
except AccountingBadRequestException as exception:
sub_title = "Error: " + exception.reason
result_list = None
code = jsonify(exception.error_data)
else:
sub_title = ""
result_list = []
for contact in created_contacts.contacts:
if contact.has_validation_errors:
error = getvalue(contact.validation_errors, "0.message", "")
result_list.append("Error: {}".format(error))
else:
result_list.append("Contact {} created.".format(contact.name))
code = serialize_model(created_contacts)
return render_template(
"code.html",
title="Create Multiple Contacts",
code=code,
result_list=result_list,
sub_title=sub_title,
)
def run(self, training_data, testing_data, fold_num):
"""Método principal de execução do multilayer perceptron"""
fold_num = fold_num + 1
files = ('config', 'error')
for file in files:
file_command = '{output}{file}-{fold}.txt'.format(file=file, fold=fold_num,
output=self.output_directory)
if file == 'config':
self.config_f = open(file_command, "w")
self.config_write()
elif file == 'error':
self.error_f = open(file_command, "w")
# Torna aleatória a lista de arquivos para treinamento e teste
random.shuffle(training_data)
random.shuffle(testing_data)
self.start_fold = datetime.now()
self.error_f.write("Execucao em {} \n\n".format(time.strftime("%d/%m/%Y %H:%M")))
print ("\nK-Fold with max {} epochs started at: {}\n".format(self.epochs,
self.start_fold.strftime("%Y-%m-%d %H:%M:%S")))
for epoch_current in range(self.epochs):
# u.print_title_epoch(epoch_current + 1, fold_num, 'training',
# self.part_2, self.descriptor)
# treinamento de 4/5 do fold
for image_i, image in enumerate(training_data):
self.training(image, image_i, epoch_current + 1, fold_num)
# erro médio de treinamento
self.error_training_avg = self.error_training_avg / self.training_number
# serialização dos pesos desta época (model.dat)
u.serialize_model(fold_num, self.weights_0, self.weights_1, self.output_directory)
# teste de 1/5 do fold
# u.print_title_epoch(epoch_current + 1, fold_num,
# 'testing', self.part_2, self.descriptor)
for image_i, image in enumerate(testing_data):
self.testing(image, image_i)
# erro médio de teste
self.error_test_avg = self.error_test_avg / self.test_number
# salva o erro quadratico médio desta época
self.errors_test_avg_list.append(self.error_test_avg)
# atualização da lista de erros de teste
self.errors_test_list.append(self.error_test_avg)
# u.error_list_update(self.error_test_avg, self.errors_list)
# atualiza'ccão da lista de erros de treinamento
self.errors_training_list.append(self.error_training_avg)
# gravação dos erros quadráticos médios
self.error_f.write("{};{};{}\n".format(epoch_current, self.error_training_avg,
self.error_test_avg))
# reinicialização das médias de erros quadráticos com 0 para a próxima época
self.error_training_avg = 0
self.error_test_avg = 0
self.test_number = 0
self.training_number = 0
# atualização da taxa de aprendizado e condição de parada por taxa de aprendizado
if self.alpha > 0.001:
self.alpha = 0.9 * self.alpha
# condicao de parada por erro
stop_condition = u.stop_condition(self.errors_test_list, epoch_current, self.alpha)
if stop_condition['result']:
break
self.get_confusion_matrix_and_accuracy(fold_num)
# média total
mean_total = np.mean(self.errors_test_avg_list)
# desvio padrão
std_dev = np.std(self.errors_test_avg_list)
self.config_f.write("epoca_final: {}\n".format(stop_condition['message']))
self.config_f.write("epoca_final: {}\n".format(epoch_current))
self.config_f.write("media_total: {}\n".format(mean_total))
self.config_f.write("desvio_padrao: {}\n".format(std_dev))
self.end_fold = datetime.now()
print ("\nK-Fold {}/5:\tMax Epoch (s): \t{}\tStart Time:\t{}".format(fold_num, self.epochs,
self.start_fold.strftime("%Y-%m-%d %H:%M:%S")))
print ("K-Fold {}/5:\tTotal Epoch (s):\t{}\tEnd Time:\t{}".format(fold_num,
epoch_current + 1,
self.end_fold.strftime("%Y-%m-%d %H:%M:%S")))
print ("K-Fold {}/5:\t\t\t\t\tRun. Time:\t{}\n".format(fold_num,
self.end_fold - self.start_fold))
self.config_f.close()
self.error_f.close()
u.plot_graph(fold_num, self.errors_test_list, self.errors_training_list, self.output_directory)
spark, parquet_path=args.data_dir)
spark.stop()
# Do not use GPU for the session creation.
tf.config.experimental.set_visible_devices([], 'GPU')
model = get_model()
# 2. Horovod: add Distributed Optimizer.
opt = tf.keras.optimizers.Adam(lr=args.learning_rate, epsilon=1e-3)
opt = hvd.DistributedOptimizer(opt)
model.compile(opt,
loss=tf.keras.losses.MeanSquaredError(),
metrics=['mse'])
model_bytes = serialize_model(model)
# Create Spark session for training.
conf = SparkConf().setAppName('training')
# if args.training_master:
# conf.setMaster(args.training_master)
conf = set_gpu_conf(conf)
spark = SparkSession.builder.config(conf=conf).getOrCreate()
history, best_model_bytes = horovod.spark.run(
train_fn,
args=(model_bytes, args.batch_size, args.epochs, train_rows, val_rows),
num_proc=args.num_proc,
verbose=2)[0]
print(f"Best MSE: {min(history['val_mean_squared_error'])}")
