def main(data_txt_path, label_txt_path, stride=25, images_folder='roadC621/'): """ Train a neural network with patches of patch_size x patch_size (as given via the module network_path). Parameters ---------- network_path : str Path to a Python script with a function generate_nnet(feats) which returns a neural network image_batch_size : int stride : int """ assert image_batch_size >= 1 assert stride >= 1 features, labels = load_data_raw_images(train_images_folder=images_folder) mem_size = (sys.getsizeof(42) * len(features) * features[0].size + sys.getsizeof(42) * len(labels) * labels[0].size) logging.info("Loaded %i data images with their labels (approx %s)", len(features), utils.sizeof_fmt(mem_size)) nn_params = {'training': {'image_batch_size': image_batch_size, 'stride': stride}} logging.info("## Network: %s", network_path) network = imp.load_source('sst.network', network_path) logging.info("Fully network: %s", str(network.fully)) nn_params['code'] = inspect.getsource(network) nn_params['fully'] = network.fully nn_params['patch_size'] = network.patch_size assert nn_params['patch_size'] > 0 labeled_patches = get_patches(features[:1], labels[:1], nn_params=nn_params) feats, _ = get_features(labeled_patches, fully=nn_params['fully']) net1 = network.generate_nnet(feats) for block in range(0, len(features), image_batch_size): from_img = block to_img = block + image_batch_size logging.info("Training on batch %i - %i of %i total", from_img, to_img, len(features)) labeled_patches = get_patches(features[from_img:to_img], labels[from_img:to_img], nn_params=nn_params, stride=stride) logging.info(("labeled_patches[0].shape: %s , " "labeled_patches[1].shape: %s"), labeled_patches[0].shape, labeled_patches[1].shape) net1 = train_nnet(labeled_patches, net1, fully=nn_params['fully']) model_pickle_name = 'nnet1-trained.pickle' utils.serialize_model(net1, filename=model_pickle_name, parameters=nn_params)
def create_contact_person(): xero_tenant_id = get_xero_tenant_id() accounting_api = AccountingApi(api_client) contact_person = ContactPerson( first_name="John", last_name="Smith", email_address="*****@*****.**", include_in_emails=True, ) contact = Contact( name="FooBar", first_name="Foo", last_name="Bar", email_address="*****@*****.**", contact_persons=[contact_person], ) contacts = Contacts(contacts=[contact]) try: created_contacts = accounting_api.create_contacts( xero_tenant_id, contacts=contacts) # type: Contacts except AccountingBadRequestException as exception: sub_title = "Error: " + exception.reason code = jsonify(exception.error_data) else: sub_title = "Contact {} created.".format( getvalue(created_contacts, "contacts.0.name", "")) code = serialize_model(created_contacts) return render_template("code.html", title="Create Contacts", code=code, sub_title=sub_title)
def get_invoices(): xero_tenant_id = get_xero_tenant_id() accounting_api = AccountingApi(api_client) invoices = accounting_api.get_invoices(xero_tenant_id, statuses=["DRAFT", "SUBMITTED"]) code = serialize_model(invoices) sub_title = "Total invoices found: {}".format(len(invoices.invoices)) return render_template("code.html", title="Invoices", code=code, sub_title=sub_title)
async def run_network_instances(args): num_instances = NUMBER_NETWORK_INSTANCES activ = 'relu' if 'instances' in args: num_instances = args['instances'] if 'activation' in args: activ = args['activation'] await spawn_network_instances(num_instances) dpinfo = await pulsar.send('data_provider', 'data_provider_info') input_format = 'hot_vector' output_format = 'categorical' input_length = dpinfo['model_params']['input_length'][input_format] output_length = dpinfo['model_params']['output_length'][output_format] if NETWORK_STATE['model'] is None: NETWORK_STATE['input_length'] = input_length NETWORK_STATE['output_length'] = output_length NETWORK_STATE['activation'] = activ model = create_model(input_length, output_length, activation=activ) logging.info( 'NETWORK SUPER-MODEL INIT (size = {}, X:{}->Y:{})'.format(model_sizeMB(serialize_model(model)), input_length, output_length)) NETWORK_STATE['model'] = serialize_model(model) instance_args = dict(args, **{'model': NETWORK_STATE['model'], 'in_len': input_length, 'out_len': output_length, 'activation': activ, 'input_format': input_format, 'output_format': output_format}) #logging.warning(instance_args) for (aid, inst) in NETWORK_STATE['instances'].items(): asyncio.ensure_future( pulsar.send(aid, 'run', run_network_instance, args=instance_args) )
def journals_list(): xero_tenant_id = get_xero_tenant_id() accounting_api = AccountingApi(api_client) journals = accounting_api.get_journals( xero_tenant_id=xero_tenant_id, if_modified_since=datetime.datetime.now() - relativedelta(days=90)) return render_template( "code.html", title="Journals", code=serialize_model(journals), sub_title="For Past 90 days", )
def members_list(): xero_tenant_id = get_xero_tenant_id() accounting_api = AccountingApi(api_client) sources = defaultdict(list) member_transactions = defaultdict(list) # Walk the full journal for the past 90 days for membership items # and split them based on which type they are, i.e. 'CASHREC' / 'ACCREC' for journal in get_journals(datetime.datetime.now() - relativedelta(days=90)): if is_membership_journal(journal): sources[journal.source_type].append(journal.source_id) transactions = [] for source, source_ids in sources.items(): if source == 'CASHREC': for s in source_ids: transactions.extend( # get_bank_transactions has to be done individually but there's usually not many of them accounting_api.get_bank_transaction(xero_tenant_id, s).bank_transactions) elif source == 'ACCREC': transactions.extend( accounting_api.get_invoices(xero_tenant_id, i_ds=source_ids).invoices) else: raise ValueError(source) for transaction in transactions: c = transaction.contact.contact_id member_transactions[c].append(transaction) members = accounting_api.get_contacts( xero_tenant_id, i_ds=list(member_transactions.keys())).contacts contact_sheet = [] for m in members: contact_sheet.append({ **fix_contact(m), 'transactions': len(member_transactions[m.contact_id]) }) return render_template( "code.html", title="Members", code=serialize_model(contact_sheet), sub_title="For Past 90 days", )
def create_multiple_contacts(): xero_tenant_id = get_xero_tenant_id() accounting_api = AccountingApi(api_client) contact = Contact( name="George Jetson", first_name="George", last_name="Jetson", email_address="*****@*****.**", ) # Add the same contact twice - the first one will succeed, but the # second contact will fail with a validation error which we'll show. contacts = Contacts(contacts=[contact, contact]) try: created_contacts = accounting_api.create_contacts( xero_tenant_id, contacts=contacts, summarize_errors=False) # type: Contacts except AccountingBadRequestException as exception: sub_title = "Error: " + exception.reason result_list = None code = jsonify(exception.error_data) else: sub_title = "" result_list = [] for contact in created_contacts.contacts: if contact.has_validation_errors: error = getvalue(contact.validation_errors, "0.message", "") result_list.append("Error: {}".format(error)) else: result_list.append("Contact {} created.".format(contact.name)) code = serialize_model(created_contacts) return render_template( "code.html", title="Create Multiple Contacts", code=code, result_list=result_list, sub_title=sub_title, )
def run(self, training_data, testing_data, fold_num): """Método principal de execução do multilayer perceptron""" fold_num = fold_num + 1 files = ('config', 'error') for file in files: file_command = '{output}{file}-{fold}.txt'.format(file=file, fold=fold_num, output=self.output_directory) if file == 'config': self.config_f = open(file_command, "w") self.config_write() elif file == 'error': self.error_f = open(file_command, "w") # Torna aleatória a lista de arquivos para treinamento e teste random.shuffle(training_data) random.shuffle(testing_data) self.start_fold = datetime.now() self.error_f.write("Execucao em {} \n\n".format(time.strftime("%d/%m/%Y %H:%M"))) print ("\nK-Fold with max {} epochs started at: {}\n".format(self.epochs, self.start_fold.strftime("%Y-%m-%d %H:%M:%S"))) for epoch_current in range(self.epochs): # u.print_title_epoch(epoch_current + 1, fold_num, 'training', # self.part_2, self.descriptor) # treinamento de 4/5 do fold for image_i, image in enumerate(training_data): self.training(image, image_i, epoch_current + 1, fold_num) # erro médio de treinamento self.error_training_avg = self.error_training_avg / self.training_number # serialização dos pesos desta época (model.dat) u.serialize_model(fold_num, self.weights_0, self.weights_1, self.output_directory) # teste de 1/5 do fold # u.print_title_epoch(epoch_current + 1, fold_num, # 'testing', self.part_2, self.descriptor) for image_i, image in enumerate(testing_data): self.testing(image, image_i) # erro médio de teste self.error_test_avg = self.error_test_avg / self.test_number # salva o erro quadratico médio desta época self.errors_test_avg_list.append(self.error_test_avg) # atualização da lista de erros de teste self.errors_test_list.append(self.error_test_avg) # u.error_list_update(self.error_test_avg, self.errors_list) # atualiza'ccão da lista de erros de treinamento self.errors_training_list.append(self.error_training_avg) # gravação dos erros quadráticos médios self.error_f.write("{};{};{}\n".format(epoch_current, self.error_training_avg, self.error_test_avg)) # reinicialização das médias de erros quadráticos com 0 para a próxima época self.error_training_avg = 0 self.error_test_avg = 0 self.test_number = 0 self.training_number = 0 # atualização da taxa de aprendizado e condição de parada por taxa de aprendizado if self.alpha > 0.001: self.alpha = 0.9 * self.alpha # condicao de parada por erro stop_condition = u.stop_condition(self.errors_test_list, epoch_current, self.alpha) if stop_condition['result']: break self.get_confusion_matrix_and_accuracy(fold_num) # média total mean_total = np.mean(self.errors_test_avg_list) # desvio padrão std_dev = np.std(self.errors_test_avg_list) self.config_f.write("epoca_final: {}\n".format(stop_condition['message'])) self.config_f.write("epoca_final: {}\n".format(epoch_current)) self.config_f.write("media_total: {}\n".format(mean_total)) self.config_f.write("desvio_padrao: {}\n".format(std_dev)) self.end_fold = datetime.now() print ("\nK-Fold {}/5:\tMax Epoch (s): \t{}\tStart Time:\t{}".format(fold_num, self.epochs, self.start_fold.strftime("%Y-%m-%d %H:%M:%S"))) print ("K-Fold {}/5:\tTotal Epoch (s):\t{}\tEnd Time:\t{}".format(fold_num, epoch_current + 1, self.end_fold.strftime("%Y-%m-%d %H:%M:%S"))) print ("K-Fold {}/5:\t\t\t\t\tRun. Time:\t{}\n".format(fold_num, self.end_fold - self.start_fold)) self.config_f.close() self.error_f.close() u.plot_graph(fold_num, self.errors_test_list, self.errors_training_list, self.output_directory)
spark, parquet_path=args.data_dir) spark.stop() # Do not use GPU for the session creation. tf.config.experimental.set_visible_devices([], 'GPU') model = get_model() # 2. Horovod: add Distributed Optimizer. opt = tf.keras.optimizers.Adam(lr=args.learning_rate, epsilon=1e-3) opt = hvd.DistributedOptimizer(opt) model.compile(opt, loss=tf.keras.losses.MeanSquaredError(), metrics=['mse']) model_bytes = serialize_model(model) # Create Spark session for training. conf = SparkConf().setAppName('training') # if args.training_master: # conf.setMaster(args.training_master) conf = set_gpu_conf(conf) spark = SparkSession.builder.config(conf=conf).getOrCreate() history, best_model_bytes = horovod.spark.run( train_fn, args=(model_bytes, args.batch_size, args.epochs, train_rows, val_rows), num_proc=args.num_proc, verbose=2)[0] print(f"Best MSE: {min(history['val_mean_squared_error'])}")