def main(): # env env_path = find_dotenv() load_dotenv(dotenv_path=env_path, verbose=True) processed_p = Path(os.environ.get('PATH_PROCESSED')).resolve() models_p = Path(os.environ.get('PATH_MODELS')).resolve() img_h = int(os.environ.get('IMAGE_HEIGHT')) img_w = int(os.environ.get('IMAGE_WIDTH')) batch_size = int(os.environ.get('BATCH_SIZE')) downsample_factor = int(os.environ.get('DOWNSAMPLE_FACTOR')) min_lr = float(os.environ.get('MIN_LEARNING_RATE')) max_lr = float(os.environ.get('MAX_LEARNING_RATE')) # according how Keras' multi_gpu_mode() handles mini-batches # logging logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) logger.info('TensorFlow version: ' + tf.__version__) logger.info('Keras version: ' + tf.keras.__version__) # parameters train_p = processed_p.joinpath('train') assert train_p.exists() # generators logger.info('loading data') train_gen = DataGenerator(train_p, img_w, img_h, batch_size, downsample_factor) max_text_len = train_gen.max_text_len logger.info('alphabet: \'' + str(train_gen.alphabet) + '\'') logger.info('alphabet size: ' + str(len(train_gen.alphabet))) logger.info('max text length: ' + str(max_text_len)) logger.info('image shape: height=' + str(img_h) + ' width=' + str(img_w)) logger.info('batch size: ' + str(batch_size)) logger.info('output size: ' + str(train_gen.output_size)) logger.info('training samples: ' + str(train_gen.n)) logger.info('train steps per epoch: ' + str(len(train_gen))) logger.info('min. learning-rate: ' + str(min_lr)) logger.info('max. learning-rate: ' + str(max_lr)) # create model model = OCRNet(train_gen.output_size, img_w, img_h, max_text_len) model.summary() # find best learning rate # initialize optimizer adam = Adam(lr=min_lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # compile model # the loss calc occurs elsewhere, so use a dummy lambda func for the loss model.compile(loss={'ctc': lambda y_true, y_pred: y_pred}, optimizer=adam, metrics=['accuracy']) lrf = LRFinder(model) lrf.find(train_gen, min_lr, max_lr, stepsPerEpoch=len(train_gen), batchSize=batch_size) # plot the loss for the various learning rates and save the # resulting plot to disk if not models_p.exists(): models_p.mkdir() lrf.plot_loss(models_p.joinpath('loss_plot.png'), title='loss') lrf.plot_loss_change(models_p.joinpath('loss_change_plot.png'), title='loss change') # in the config and then train the network for our full set of logger.info('learning rate finder complete') logger.info('best LR: %f' % lrf.get_best_lr())
def save_examples_to_folder(output_folder, images_count=1000, dataset="tf_flowers"): logging.basicConfig(level=logging.INFO) logging.getLogger("tensorflow").handlers = [] ds, ds_info = tfds.load(dataset, split="train", with_info=True) Path(output_folder).mkdir(parents=True, exist_ok=True) save_examples(ds_info, ds, images_count, output_folder)
def main(): # env env_path = find_dotenv() load_dotenv(dotenv_path=env_path, verbose=True) processed_p = Path(os.environ.get('PATH_PROCESSED')).resolve() models_p = Path(os.environ.get('PATH_MODELS')).resolve() img_h = int(os.environ.get('IMAGE_HEIGHT')) img_w = int(os.environ.get('IMAGE_WIDTH')) batch_size = int(os.environ.get('BATCH_SIZE')) downsample_factor = int(os.environ.get('DOWNSAMPLE_FACTOR')) lr = float(os.environ.get('LEARNING_RATE')) # logging logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) logger.info('TensorFlow version: ' + tf.__version__) logger.info('Keras version: ' + tf.keras.__version__) # parameters test_p = processed_p.joinpath('test') assert test_p.exists() logger.info('load data') test_gen = DataGenerator(test_p, img_w, img_h, batch_size, downsample_factor) alphabet = test_gen.alphabet logger.info('image shape: height=' + str(img_h) + ' width=' + str(img_w)) logger.info('batch size: ' + str(batch_size)) logger.info('test samples: ' + str(test_gen.n)) logger.info('test steps per epoch: ' + str(len(test_gen))) logger.info('learning rate: ' + str(lr)) # model checkpoint_p = models_p.joinpath('model.h5') assert checkpoint_p.exists() model = load_model(str(checkpoint_p), compile=False) model.summary() logger.info('model loaded') # optimizer adam = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # the loss calc occurs elsewhere, so use a dummy lambda func for the loss model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=adam, metrics=['accuracy']) logger.info('model compiled') # test data score = model.evaluate_generator(generator=test_gen, steps=len(test_gen), verbose=1) logger.info('loss %.3f accuracy: %.3f' % (score[0], score[1]))
def init_logging(): logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False logging.basicConfig(stream=sys.stderr, level=logging.INFO)
from image2seq.models.eda_xu_mlp import EDAXUMLP from image2seq.models.eda_xu_mlp_exp_loss import EDAXUMLPEXPLOSS from image2seq.models.drake_concat import DRAKECONCAT from image2seq.models.drake_concat_mlp import DRAKECONCATMLP from image2seq.models.drake_detections2 import DRAKEDETECTIONS2 from image2seq.models.drake_nested_single_lstm import DRAKENESTEDSINGLELSTM from image2seq.models.drake_parallel import DRAKEPARALLEL # Logging options ##########################################################' logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False date = pd.datetime.now().date() hour = pd.datetime.now().hour minute = pd.datetime.now().minute logging.basicConfig( level=logging.INFO, format='%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s', filename="image2seq/logs/train_log_{}_{}{}.txt".format(date, hour, minute)) console = logging.StreamHandler() console.setLevel(logging.INFO) formatter = logging.Formatter( '%(asctime)-15s %(name)-5s %(levelname)-8s %(message)s') console.setFormatter(formatter) logging.getLogger("").addHandler(console) ############################################################################# # Model Setup # ############################################################################# logging.info("MODEL SETUP - Tensorflow version".format(tf.__version__)) logging.info("MODEL SETUP - Validation Script - train_full.py") from tensorflow.python.client import device_lib
def train(net, train_data, train_dataset, val_data, eval_metric, ctx, save_prefix, start_epoch, num_samples): """Training pipeline""" net.collect_params().reset_ctx(ctx) if FLAGS.no_wd: for k, v in net.collect_params('.*beta|.*gamma|.*bias').items(): v.wd_mult = 0.0 if FLAGS.label_smooth: net._target_generator._label_smooth = True if FLAGS.lr_decay_period > 0: lr_decay_epoch = list(range(FLAGS.lr_decay_period, FLAGS.epochs, FLAGS.lr_decay_period)) else: lr_decay_epoch = FLAGS.lr_decay_epoch # for handling reloading from past epoch lr_decay_epoch_tmp = list() for e in lr_decay_epoch: if int(e) <= start_epoch: FLAGS.lr = FLAGS.lr * FLAGS.lr_decay else: lr_decay_epoch_tmp.append(int(e) - start_epoch - FLAGS.warmup_epochs) lr_decay_epoch = lr_decay_epoch_tmp num_batches = num_samples // FLAGS.batch_size lr_scheduler = LRSequential([ LRScheduler('linear', base_lr=0, target_lr=FLAGS.lr, nepochs=FLAGS.warmup_epochs, iters_per_epoch=num_batches), LRScheduler(FLAGS.lr_mode, base_lr=FLAGS.lr, nepochs=FLAGS.epochs - FLAGS.warmup_epochs - start_epoch, iters_per_epoch=num_batches, step_epoch=lr_decay_epoch, step_factor=FLAGS.lr_decay, power=2), ]) trainer = gluon.Trainer( net.collect_params(), 'sgd', {'wd': FLAGS.wd, 'momentum': FLAGS.momentum, 'lr_scheduler': lr_scheduler}, kvstore='local') # targets sigmoid_ce = gluon.loss.SigmoidBinaryCrossEntropyLoss(from_sigmoid=False) l1_loss = gluon.loss.L1Loss() # metrics obj_metrics = mx.metric.Loss('ObjLoss') center_metrics = mx.metric.Loss('BoxCenterLoss') scale_metrics = mx.metric.Loss('BoxScaleLoss') cls_metrics = mx.metric.Loss('ClassLoss') # set up logger logging.basicConfig() logger = logging.getLogger() logger.setLevel(logging.INFO) log_file_path = save_prefix + '_train.log' log_dir = os.path.dirname(log_file_path) if log_dir and not os.path.exists(log_dir): os.makedirs(log_dir) fh = logging.FileHandler(log_file_path) logger.addHandler(fh) # logger.info(FLAGS) # set up tensorboard summary writer tb_sw = SummaryWriter(log_dir=os.path.join(log_dir, 'tb'), comment=FLAGS.save_prefix) # Check if wanting to resume logger.info('Start training from [Epoch {}]'.format(start_epoch)) if FLAGS.resume.strip() and os.path.exists(save_prefix+'_best_map.log'): with open(save_prefix+'_best_map.log', 'r') as f: lines = [line.split()[1] for line in f.readlines()] best_map = [float(lines[-1])] else: best_map = [0] # Training loop num_batches = int(len(train_dataset)/FLAGS.batch_size) for epoch in range(start_epoch, FLAGS.epochs+1): st = time.time() if FLAGS.mixup: # TODO(zhreshold): more elegant way to control mixup during runtime try: train_data._dataset.set_mixup(np.random.beta, 1.5, 1.5) except AttributeError: train_data._dataset._data.set_mixup(np.random.beta, 1.5, 1.5) if epoch >= FLAGS.epochs - FLAGS.no_mixup_epochs: try: train_data._dataset.set_mixup(None) except AttributeError: train_data._dataset._data.set_mixup(None) tic = time.time() btic = time.time() if not FLAGS.nd_only: net.hybridize() for i, batch in enumerate(train_data): batch_size = batch[0].shape[0] if FLAGS.max_epoch_time > 0 and (time.time()-st)/60 > FLAGS.max_epoch_time: logger.info('Max epoch time of %d minutes reached after completing %d%% of epoch. ' 'Moving on to next epoch' % (FLAGS.max_epoch_time, int(100*(i/num_batches)))) break if FLAGS.features_dir is not None: f1 = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0) f2 = gluon.utils.split_and_load(batch[1], ctx_list=ctx, batch_axis=0) f3 = gluon.utils.split_and_load(batch[2], ctx_list=ctx, batch_axis=0) # objectness, center_targets, scale_targets, weights, class_targets fixed_targets = [gluon.utils.split_and_load(batch[it], ctx_list=ctx, batch_axis=0) for it in range(3, 8)] gt_boxes = gluon.utils.split_and_load(batch[8], ctx_list=ctx, batch_axis=0) else: data = gluon.utils.split_and_load(batch[0], ctx_list=ctx, batch_axis=0) # objectness, center_targets, scale_targets, weights, class_targets fixed_targets = [gluon.utils.split_and_load(batch[it], ctx_list=ctx, batch_axis=0) for it in range(1, 6)] gt_boxes = gluon.utils.split_and_load(batch[6], ctx_list=ctx, batch_axis=0) sum_losses = [] obj_losses = [] center_losses = [] scale_losses = [] cls_losses = [] if FLAGS.features_dir is not None: with autograd.record(): for ix, (x1, x2, x3) in enumerate(zip(f1, f2, f3)): obj_loss, center_loss, scale_loss, cls_loss = net(x1, x2, x3, gt_boxes[ix], *[ft[ix] for ft in fixed_targets]) sum_losses.append(obj_loss + center_loss + scale_loss + cls_loss) obj_losses.append(obj_loss) center_losses.append(center_loss) scale_losses.append(scale_loss) cls_losses.append(cls_loss) autograd.backward(sum_losses) else: with autograd.record(): for ix, x in enumerate(data): obj_loss, center_loss, scale_loss, cls_loss = net(x, gt_boxes[ix], *[ft[ix] for ft in fixed_targets]) sum_losses.append(obj_loss + center_loss + scale_loss + cls_loss) obj_losses.append(obj_loss) center_losses.append(center_loss) scale_losses.append(scale_loss) cls_losses.append(cls_loss) autograd.backward(sum_losses) if FLAGS.motion_stream is None: trainer.step(batch_size) else: trainer.step(batch_size, ignore_stale_grad=True) # we don't use all layers of each stream obj_metrics.update(0, obj_losses) center_metrics.update(0, center_losses) scale_metrics.update(0, scale_losses) cls_metrics.update(0, cls_losses) if FLAGS.log_interval and not (i + 1) % FLAGS.log_interval: name1, loss1 = obj_metrics.get() name2, loss2 = center_metrics.get() name3, loss3 = scale_metrics.get() name4, loss4 = cls_metrics.get() logger.info('[Epoch {}][Batch {}/{}], LR: {:.2E}, Speed: {:.3f} samples/sec, {}={:.3f}, {}={:.3f}, ' '{}={:.3f}, {}={:.3f}'.format(epoch, i, num_batches, trainer.learning_rate, batch_size/(time.time()-btic), name1, loss1, name2, loss2, name3, loss3, name4, loss4)) tb_sw.add_scalar(tag='Training_' + name1, scalar_value=loss1, global_step=(epoch * len(train_data) + i)) tb_sw.add_scalar(tag='Training_' + name2, scalar_value=loss2, global_step=(epoch * len(train_data) + i)) tb_sw.add_scalar(tag='Training_' + name3, scalar_value=loss3, global_step=(epoch * len(train_data) + i)) tb_sw.add_scalar(tag='Training_' + name4, scalar_value=loss4, global_step=(epoch * len(train_data) + i)) btic = time.time() name1, loss1 = obj_metrics.get() name2, loss2 = center_metrics.get() name3, loss3 = scale_metrics.get() name4, loss4 = cls_metrics.get() logger.info('[Epoch {}] Training cost: {:.3f}, {}={:.3f}, {}={:.3f}, {}={:.3f}, {}={:.3f}'.format( epoch, (time.time()-tic), name1, loss1, name2, loss2, name3, loss3, name4, loss4)) if not (epoch + 1) % FLAGS.val_interval: # consider reduce the frequency of validation to save time logger.info('End Epoch {}: # samples: {}, seconds: {}, samples/sec: {:.2f}'.format( epoch, len(train_data)*batch_size, time.time() - st, (len(train_data)*batch_size)/(time.time() - st))) st = time.time() map_name, mean_ap = validate(net, val_data, ctx, eval_metric) logger.info('End Val: # samples: {}, seconds: {}, samples/sec: {:.2f}'.format( len(val_data)*batch_size, time.time() - st, (len(val_data) * batch_size)/(time.time() - st))) val_msg = '\n'.join(['{}={}'.format(k, v) for k, v in zip(map_name, mean_ap)]) tb_sw.add_scalar(tag='Validation_mAP', scalar_value=float(mean_ap[-1]), global_step=(epoch * len(train_data) + i)) logger.info('[Epoch {}] Validation: \n{}'.format(epoch, val_msg)) current_map = float(mean_ap[-1]) else: current_map = 0. save_params(net, best_map, current_map, epoch, FLAGS.save_interval, save_prefix)
from datasets.imgnetvid import ImageNetVidDetection from datasets.combined import CombinedDetection from metrics.pascalvoc import VOCMApMetric, VOCMApMetricTemporal from metrics.mscoco import COCODetectionMetric from models.definitions.yolo.wrappers import yolo3_darknet53, yolo3_no_backbone, yolo3_3ddarknet from models.definitions.yolo.transforms import YOLO3DefaultTrainTransform, YOLO3DefaultInferenceTransform, \ YOLO3VideoTrainTransform, YOLO3VideoInferenceTransform, YOLO3NBVideoTrainTransform, YOLO3NBVideoInferenceTransform from utils.general import as_numpy # disable autotune os.environ['MXNET_CUDNN_AUTOTUNE_DEFAULT'] = '0' logging.basicConfig(level=logging.INFO) flags.DEFINE_string('network', 'darknet53', 'Base network name: darknet53') flags.DEFINE_list('dataset', ['voc'], 'Datasets to train on.') flags.DEFINE_list('dataset_val', [], 'Datasets to test on.') flags.DEFINE_string('trained_on', '', 'Used for finetuning, specify the dataset the original model was trained on.') flags.DEFINE_string('save_prefix', '0001', 'Model save prefix.') flags.DEFINE_integer('log_interval', 100, 'Logging mini-batch interval.') flags.DEFINE_integer('save_interval', -10, 'Saving parameters epoch interval, best model will always be saved. '
def main(jpg_p, outdir_p): # env env_path = find_dotenv() load_dotenv(dotenv_path=env_path, verbose=True) processed_p = Path(os.environ.get('PATH_PROCESSED')).resolve() models_p = Path(os.environ.get('PATH_MODELS')).resolve() img_h = int(os.environ.get('IMAGE_HEIGHT')) img_w = int(os.environ.get('IMAGE_WIDTH')) batch_size = int(os.environ.get('BATCH_SIZE')) downsample_factor = int(os.environ.get('DOWNSAMPLE_FACTOR')) ncores = int(os.environ.get('NCORES')) lr = float(os.environ.get('LEARNING_RATE')) # logging logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) logger.info('TensorFlow version: ' + tf.__version__) logger.info('Keras version: ' + tf.keras.__version__) # preparation train_p = processed_p.joinpath('train') assert train_p.exists() assert jpg_p.exists() if not outdir_p.exists(): outdir_p.mkdir() split(jpg_p, outdir_p) pad_and_resize(outdir_pi, img_h, img_w) # model model = build_model(ncores, lr, models_p) # process train_gen = DataGenerator(train_p, img_w, img_h, batch_size, downsample_factor) stem = jpg_p.stem with open(outdir_p.joinpath(stem + '.csv'), 'w+') as csv_f: writer = csv.writer(csv_f, delimiter=';') # CSV header writer.writerow( ('Symbol', 'High', 'Low', 'Now', 'Sell Target Potential', 'Worst-Case Drawdowns', 'Range Index', 'Win Odds/100', '% Payoff', 'Days Held', 'Annual Rate of Return', 'Sample Size', '', 'Creadible Ratio', 'Rwd~Rsk Ratio', 'Wghted')) for r in range(1, 21): row = [] for c in range(1, 17): if 'channels_first' == K.image_data_format(): X = np.ones([1, 1, img_w, img_h]) else: X = np.ones([1, img_w, img_h, 1]) img_p = outdir_p.joinpath('%s-%d-%d.jpg' % (stem, r, c)) assert img_p.exists() img = cv2.imread(str(img_p)) img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) img = cv2.resize(img, (img_w, img_h), interpolation=cv2.INTER_AREA) img = img.astype(np.float32) img /= 255 img = img.T if 'channels_first' == K.image_data_format(): img = np.expand_dims(img, 0) else: img = np.expand_dims(img, -1) X[0] = img y = np.zeros([1, train_gen.max_text_len]) input_length = np.ones( (1, 1)) * (img_w // downsample_factor - 2) label_length = np.zeros((1, 1)) inputs = { 'the_input': X, 'the_labels': y, 'input_length': input_length, 'label_length': label_length, 'the_sources': img_p.name, } net_out_value = model.predict(inputs) pred_texts = decode_batch(net_out_value, train_gen.alphabet) row.append(pred_texts[0]) writer.writerow(row)
def main(unused_argv): if FLAGS.verbose: logging.basicConfig(level=logging.INFO) if FLAGS.debug: logging.basicConfig(level=logging.DEBUG) logging.debug( 'binary: %s\noptimize: %d\nbase_directory: %s\n' 'policy_file: %s\nrendered_acl_directory: %s', str(sys.argv[0]), int(FLAGS.optimize), str(FLAGS.base_directory), str(FLAGS.policy_file), str(FLAGS.output_directory)) definitions = None try: definitions = naming.Naming(FLAGS.definitions_directory) except naming.NoDefinitionsError: err_msg = 'bad definitions directory: %s' % FLAGS.definitions_directory logging.fatal(err_msg) # thead-safe list for storing files to write manager = multiprocessing.Manager() write_files = manager.list() with_errors = False if FLAGS.policy_file: # render just one file logging.info('rendering one file') RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions, FLAGS.exp_info, write_files) else: # render all files in parallel logging.info('finding policies...') pols = [] pols.extend( DescendRecursively(FLAGS.base_directory, FLAGS.output_directory, definitions)) pool = multiprocessing.Pool(processes=FLAGS.max_renderers) results = [] for x in pols: results.append( pool.apply_async(RenderFile, args=(x.get('in_file'), x.get('out_dir'), definitions, FLAGS.exp_info, write_files))) pool.close() pool.join() for result in results: try: result.get() except (ACLParserError, ACLGeneratorError) as e: with_errors = True logging.warn( '\n\nerror encountered in rendering process:\n%s\n\n', e) # actually write files to disk WriteFiles(write_files) if with_errors: logging.warn('done, with errors.') sys.exit(1) else: logging.info('done.')
def main(): # env env_path = find_dotenv() load_dotenv(dotenv_path=env_path, verbose=True) processed_p = Path(os.environ.get('PATH_PROCESSED')).resolve() models_p = Path(os.environ.get('PATH_MODELS')).resolve() img_h = int(os.environ.get('IMAGE_HEIGHT')) img_w = int(os.environ.get('IMAGE_WIDTH')) batch_size = int(os.environ.get('BATCH_SIZE')) epochs = int(os.environ.get('EPOCHS')) ngpus = int(os.environ.get('NGPUS')) downsample_factor = int(os.environ.get('DOWNSAMPLE_FACTOR')) lr = float(os.environ.get('LEARNING_RATE')) # calculate batchsize according to strategy strategy = tf.distribute.MirroredStrategy( ) if 1 < ngpus else tf.distribute.OneDeviceStrategy(device="/gpu:1") batch_size = batch_size * strategy.num_replicas_in_sync # logging logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') logger = logging.getLogger(__name__) logger.info('TensorFlow version: ' + tf.__version__) logger.info('Keras version: ' + tf.keras.__version__) # parameters val_p = processed_p.joinpath('val') assert val_p.exists() train_p = processed_p.joinpath('train') assert train_p.exists() # generators logger.info('load data') train_gen = DataGenerator(train_p, img_w, img_h, batch_size, downsample_factor) val_gen = DataGenerator(val_p, img_w, img_h, batch_size, downsample_factor) assert train_gen.alphabet == val_gen.alphabet assert train_gen.output_size == val_gen.output_size max_text_len = max(train_gen.max_text_len, val_gen.max_text_len) logger.info('alphabet: \'' + str(train_gen.alphabet) + '\'') logger.info('alphabet size: ' + str(len(train_gen.alphabet))) logger.info('max text length: ' + str(max_text_len)) logger.info('image shape: height=' + str(img_h) + ' width=' + str(img_w)) logger.info('batch size: ' + str(batch_size)) logger.info('output size: ' + str(train_gen.output_size)) logger.info('training samples: ' + str(train_gen.n)) logger.info('validation samples: ' + str(val_gen.n)) logger.info('epochs: ' + str(epochs)) logger.info('train steps per epoch: ' + str(len(train_gen))) logger.info('validation steps per epoch: ' + str(len(val_gen))) logger.info('learning rate: ' + str(lr)) with strategy.scope(): # create model model = OCRNet(train_gen.output_size, img_w, img_h, max_text_len) model.summary() # initialize optimizer adam = Adam(lr=lr, beta_1=0.9, beta_2=0.999, epsilon=1e-08) # compile model # the loss calc occurs elsewhere, so use a dummy lambda func for the loss model.compile(loss={ 'ctc': lambda y_true, y_pred: y_pred }, optimizer=adam, metrics=['accuracy']) # callbacks log_p = models_p.joinpath('logs') log_p.mkdir(exist_ok=True) callbacks = [ EarlyStopping(patience=2, monitor='val_loss'), ModelCheckpoint(str(models_p.joinpath('model.h5'))) ] # model training logger.info('start fitting model') start = time.perf_counter() model.fit(train_gen, validation_data=val_gen, epochs=epochs, shuffle=False, use_multiprocessing=True, workers=6, callbacks=callbacks) elapsed = time.perf_counter() - start logger.info('elapsed: {:0.3f}'.format(elapsed))
self._world.reset() joint_state = gazebo.JointState(1) joint_state.set_positions([self.noise * random.random()]) joint_state.set_velocities([0.0]) self._agent.set_joint_state("cartpole::cartpole::cart_to_pole", joint_state) return self._get_state() def main(): env = CartPole() env.reset() env._world.info() for _ in range(100): env.reset() total_rewards = 0 while True: obs, reward, done, info = env.step((random.random() - 0.5) * 0.5) total_rewards += reward env.render() if done: print("total reward " + str(total_rewards)) break if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) main()
import absl.logging from flask import Flask, render_template, redirect from werkzeug.utils import secure_filename import pickledb from core.modelcomponents import LAYERS, NORMALIZATION_METHODS from core.projectmanager import ProjectManager from core.runtimemanager import RuntimeManager from core.usermanager import UserManager from core.validation import * # Global variables DATABASE_NAME = 'Kerasuite.db' # Enable logging logging.basicConfig(level=logging.INFO) # Default logging level # Remove Tensorflow logging bug logging.root.removeHandler(absl.logging._absl_handler) absl.logging._warn_preinit_stderr = False # Create Flask app app = Flask(__name__) app.secret_key = urandom(80) app.config['UPLOAD_FOLDER'] = path.join(path.dirname(path.realpath(__file__)), 'data') # Check if the database exists if DATABASE_NAME not in listdir('.'): # Initialise the database with a default user & password (admin - Kerasuite) logging.warning('The database does not exist, initialising now ...') database = pickledb.load(DATABASE_NAME, auto_dump=True)
def main(): # load arguments args = parse_args() # define logging level and format level = logging.INFO absl.logging.set_stderrthreshold('info') absl.logging.set_verbosity('info') if args.debug: absl.logging.set_stderrthreshold('debug') absl.logging.set_verbosity('debug') level = logging.DEBUG logging.basicConfig(format='%(asctime)s %(levelname)s:%(message)s', datefmt='%Y-%m-%d %H:%M:%S', level=level) # check output file and create folder if not args.output.endswith('.h5'): logging.error('Output file should end with .h5. Extiting ...') return 0 if not os.path.exists(os.path.dirname(args.output)): os.makedirs(os.path.dirname(args.output)) if os.path.isfile(args.output): mode = 'r+' else: mode = 'a' # create model depending on model type FE = None if 'facenet' in args.model: logging.info( 'Building network for embedding based on person verification ...') FE = FacialFeatureExtractor(model_path=args.model) elif 'location' in args.model: logging.info( 'Building network for embedding based on geolocation estimation ...' ) FE = GeoEstimator(model_path=args.model, use_cpu=args.use_cpu) elif 'scene' in args.model: logging.info( 'Building network for embedding based on scene classification ...') FE = SceneClassificator(model_path=args.model) if not FE: logging.error('Unknown model. Exiting ...') return 0 if 'scene' not in args.model and args.logits: logging.error( 'Please specify a scene classification model to create scene logits.' ) return 0 # read dataset if args.type == 'news': dataset = utils.read_jsonl(args.input, dict_key='id') else: dataset = utils.read_jsonl(args.input, dict_key='wd_id') logging.info(f'{len(dataset.keys())} dataset entries to process') # create embeddings with h5py.File(args.output, mode) as output_file: for entry in dataset.values(): images = [] if args.type == 'news': fname = os.path.join(args.directory, entry['id']) images.append({'fname': fname, 'search_engine': None}) else: # entity for image in entry['image_urls']: fname, _ = os.path.splitext(image['filename']) fname = os.path.join(args.directory, entry['wd_id'], fname) images.append({ 'fname': fname, 'search_engine': image['search_engine'] }) for image in images: image_files = glob.glob(image['fname'] + '.*') # get image_file with extension if len(image_files) == 0: logging.info( f"Cannot find image file {image['fname']}.jpg") continue else: image_file = image_files[0] if args.type == 'news': h5_key = entry['id'] else: # entity h5_key = f"{entry['wd_id']}/{image['search_engine']}/{os.path.basename(image['fname'])}" if h5_key in output_file: logging.info( f'Embedding for {h5_key} already computed ...') continue logging.info(f'Generate embedding for {h5_key} ...') img_embeddings = [] if args.logits: img_emb = FE.get_logits(image_file) else: img_emb = FE.get_img_embedding(image_file) for e in img_emb: img_embeddings.append(e) if len(img_embeddings) == 0: logging.debug(f'No embedding found for {h5_key} ...') output_file[h5_key] = [] else: # convert to np array and store to output file id_img_embs = np.asarray(img_embeddings, dtype=np.float32) output_file[h5_key] = id_img_embs return 0
def main(args): FLAGS(args) if FLAGS.verbose: logging.basicConfig(level=logging.INFO) if FLAGS.debug: logging.basicConfig(level=logging.DEBUG) logging.debug('binary: %s\noptimize: %d\nbase_directory: %s\n' 'policy_file: %s\nrendered_acl_directory: %s', str(sys.argv[0]), int(FLAGS.optimize), str(FLAGS.base_directory), str(FLAGS.policy_file), str(FLAGS.output_directory)) definitions = None try: definitions = naming.Naming(FLAGS.definitions_directory) except naming.NoDefinitionsError: err_msg = 'bad definitions directory: %s', FLAGS.definitions_directory logging.fatal(err_msg) sys.exit(1) # thead-safe list for storing files to write manager = multiprocessing.Manager() write_files = manager.list() with_errors = False if FLAGS.policy_file: # render just one file logging.info('rendering one file') RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions, FLAGS.exp_info, write_files) else: # render all files in parallel logging.info('finding policies...') pols = [] pols.extend(DescendRecursively(FLAGS.base_directory, FLAGS.output_directory, definitions)) pool = multiprocessing.Pool(processes=FLAGS.max_renderers) results = [] for x in pols: results.append(pool.apply_async(RenderFile, args=(x.get('in_file'), x.get('out_dir'), definitions, FLAGS.exp_info, write_files))) pool.close() pool.join() for result in results: try: result.get() except (ACLParserError, ACLGeneratorError) as e: with_errors = True logging.warn('\n\nerror encountered in rendering process:\n%s\n\n', e) # actually write files to disk WriteFiles(write_files) if with_errors: logging.warn('done, with errors.') sys.exit(1) else: logging.info('done.')