def run(sources, threads, prefetch, batch, steps, epochs, verbose, seed): """ sources: list of tf-record files threads: number of concurrent read pipelines prefetch: reader prefetch count batch: batch size steps: number of steps per epoch epochs: number of epochs verbose: verbose output """ reset_tensorflow(seed) datasets = [ make_tfr_dataset(source, threads=threads, read_rgb=False, read_mask=False, read_weight=False) for source in sources ] items = list() with tf.device('/cpu:0'): for epoch in range(epochs): for dataset in datasets: input_fn = make_tfr_input_fn(dataset, threads=threads, offset=max( 0, epoch * steps * batch), limit=max(0, steps * batch), shuffle=max(0, steps * batch), prefetch=prefetch, batch=batch, repeat=-1) with tf.Session() as session: try: it = input_fn() batch_counter = 0 while batch_counter < steps: features, label = session.run(it) names = features['name'] variants = features['variant'] for i in range(names.shape[0]): key = '%s-%s' % (str(names[i], 'utf-8'), str(variants[i], 'utf-8')) items.append(key) batch_counter += 1 except tf.errors.DataLossError as e: logger.warn(e) pass except tf.errors.OutOfRangeError: pass logger.info("ORDER: %d items", len(items)) for i in range(len(items)): print("%05d. %s" % (i, items[i]))
def run(sources): """ sources: list of tf-record files """ key = None for source in sources: logger.info(source) dataset = make_tfr_dataset(source) reset_tensorflow() step = 0 with tf.Session() as session: with tf.device('/cpu:0'): it = dataset().make_one_shot_iterator().get_next() while key != ord('q'): item = session.run(it) step += 1 name = str(item['name'], 'utf-8') rgb = cv2.cvtColor(item['rgb'], cv2.COLOR_RGB2BGR) mask = item['mask'] weight = item['weight'] result = resizemax( np.concatenate( [ rgb, cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR), tobytes(cv2.cvtColor(weight, cv2.COLOR_GRAY2BGR) / 255) # tobytes(mix(tofloats(cv2.cvtColor(mask, cv2.COLOR_GRAY2BGR)), 0.5 + 0.5 * tofloats(rgb))), # tobytes(mix(cv2.cvtColor(weight, cv2.COLOR_GRAY2BGR).astype(np.float32) / 65535.0, 0.5 + 0.5 * tofloats(rgb))) ], axis=0 ), 1024 ) cv2.putText(result, '%d' % step, (result.shape[1]-50, result.shape[0]-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 0, 0)) cv2.putText(result, name, (5, result.shape[0]-5), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255)) cv2.imshow('cv2: preview', result) key = cv2.waitKey() while key != ord('q') and key != ord('c'): key = cv2.waitKey()
def run(device, model, sources, threads, prefetch, batch, steps, verbose, seed): """ Evaluation task implementation. """ logger.info("Running graph on %s", device) # reset tensorflow reset_tensorflow(seed) # read parameters paramsfile = Path(model) / "parameters.json" if paramsfile.exists(): params = json.loads(paramsfile.read_text()) else: raise Exception("Model parameters.json not found") classes = params.get('classes', [0.0, 1.0]) # read previous evals evalsfile = Path(model) / "evals.json" if evalsfile.exists(): evalslogs = json.loads(evalsfile.read_text()) else: evalslogs = list() # read topology file logger.info("Creating model topology V%d (%s)", params.get('topology_version', 1), params.get('topology')) topologyfile = Path(model) / "topology.yaml" if not topologyfile.exists(): raise Exception("Model topology.yaml not found") topology = resolve_topology(params.get('topology_version', 1))( topologyfile, params.get("initializer", 'none'), params.get("regularizer", 'none'), params.get("constraint", 'none'), params.get("activation", 'none'), params.get("local_response_normalization", 0), params.get("batch_normalization", False), params.get("dropout_rate", 0.0)) # create tensorflow estimator estimator = tf.estimator.Estimator( model_dir=model, model_fn=topology_cnn_model(topology, verbose), params=params, config=tf.estimator.RunConfig(tf_random_seed=seed, device_fn=lambda op: device)) logger.debug("Using checkpoint: %s", estimator.latest_checkpoint()) # evaluate datasets using model for source in sources: eval_start_time = time.time() logger.info("Starting evaluation: %s (steps=%d, batch=%d)", source, steps, batch) results = estimator.evaluate(input_fn=make_tfr_input_fn( make_tfr_dataset(source, threads=threads), threads=threads, limit=max(0, steps * batch), prefetch=prefetch, batch=batch)) for i in range(len(classes)): logger.info("Class %d: precision=%02.02f%%, recall=%02.02f%%", i, results['precision_c%d' % i] * 100, results['recall_c%d' % i] * 100) logger.info("Evaluation completed: loss=%f, jaccard=%f (%s)", results['loss'], results['jaccard'], source) # write eval logs evalslogs.append({ 'dataset': source, 'checkpoint': Path(estimator.latest_checkpoint()).name, 'duration': int(time.time() - eval_start_time), 'loss': float(results['loss']), 'jaccard': float(results['jaccard']), 'precision': [float(results['precision_c%d' % i]) for i in range(len(classes))], 'recall': [float(results['recall_c%d' % i]) for i in range(len(classes))], 'f1': [ f1_score(results['precision_c%d' % i], results['recall_c%d' % i]) for i in range(len(classes)) ], }) evalsfile.write_text(json.dumps(evalslogs, indent=' '))
def run(device, name, sources, compression, batch, test_ratio, eval_ratio, size, blur_radius, blur_scale, center_crop, scales, flips, rotations, crops, brightness, contrast, gaussian_noise, uniform_noise, seed): """ device: tensorflow device name: tf-record prefix sources: list of input image folders compression: tf-record compression batch: image pipeline size test_ratio: percentage of final test instances in the total set eval_ratio: percentage of evaluation instances in the training set size: target image size (width and height) blur_radius: gaussian blur radius for mask/weight images blur_scale: gaussian blur scale factor for mask/weight images center_crop: source images center crop percentage scales: list of scale factors flips: list of flip modes rotations: list of rotations crops: number of crops/variations to create for each configuration brightness: random brightness factor contrast: random contrast factor brightness: random brightness factor gaussian_noise: gaussian noise standard deviation uniform_noise: uniform noise magnitude """ if compression == 'none': options = tf.python_io.TFRecordOptions( tf.python_io.TFRecordCompressionType.NONE) ext = 'tfr' elif compression == 'zlib': options = tf.python_io.TFRecordOptions( tf.python_io.TFRecordCompressionType.ZLIB) ext = 'tfr.zlib' elif compression == 'gzip': options = tf.python_io.TFRecordOptions( tf.python_io.TFRecordCompressionType.GZIP) ext = 'tfr.gz' else: raise Exception("Unsupported compression format (%s)" % compression) batch = max(1, batch) test_ratio = max(0.0, min(1.0, test_ratio)) eval_ratio = max(0.0, min(1.0, eval_ratio)) size = int(max(1, size)) blur_radius = int(max(0, min(size / 2, blur_radius))) center_crop = max(0.0, min(1.0, center_crop)) blur_scale = max(0.0, min(1.0, blur_scale)) scales = [max(0.0, min(1.0, scale)) for scale in scales] rotations = [max(-180, min(180, rotation)) for rotation in rotations] crops = int(max(1, crops)) brightness = max(0.0, min(1.0, brightness)) contrast = max(0.0, min(1.0, contrast)) gaussian_noise = max(0.0, gaussian_noise) uniform_noise = max(0.0, min(1.0, uniform_noise)) # find samples items = list() logger.info('Finding input files...') for source in sources: path = Path(source) bakemeta = path / "bake.json" if not bakemeta.exists(): meta_items = list() listing = path / "items.json" for (filename, meta) in json.loads(listing.read_text()).items(): if not meta['active']: continue rgb = path / filename mask = rgb.parent / (rgb.stem + '.mask') weight = rgb.parent / (rgb.stem + '.weight') width, height = image_size(rgb) assert width >= size and height >= size assert image_channels(rgb) == 3 if not mask.exists(): imask = np.zeros([height, width], dtype=np.uint8) cv2.imwrite(str(mask) + '.png', imask) Path(str(mask) + '.png').replace(mask) assert (width, height) == image_size(mask) assert image_channels(mask) == 1 if not weight.exists(): iweight = 255 * np.ones([height, width], dtype=np.uint8) cv2.imwrite(str(weight) + '.png', iweight) Path(str(weight) + '.png').replace(weight) assert (width, height) == image_size(weight) assert image_channels(weight) == 1 meta_items.append({ 'name': rgb.stem, 'orientation': image_orientation(rgb), 'width': width, 'height': height, 'rgbpath': str(rgb), 'maskpath': str(mask), 'weightpath': str(weight), }) random.shuffle(meta_items) bakemeta.write_text(json.dumps(meta_items, indent=' ')) items += json.loads(bakemeta.read_text()) # select test set i = int(len(items) * (1.0 - test_ratio)) remaining_items = items[0:i] test_items = items[i:] # select train/eval sets i = int(len(remaining_items) * (1.0 - eval_ratio)) train_items = remaining_items[0:i] eval_items = remaining_items[i:] # save partitioning if len(train_items) > 0: with io.open('%s-%dx%d-train.json' % (name, size, size), "w") as f: json.dump(train_items, f, indent=' ') if len(eval_items) > 0: with io.open('%s-%dx%d-eval.json' % (name, size, size), "w") as f: json.dump(eval_items, f, indent=' ') if len(test_items) > 0: with io.open('%s-%dx%d-test.json' % (name, size, size), "w") as f: json.dump(test_items, f, indent=' ') # generate tf-records def bake_records(session, writer, inputs, graphs): feeds = dict() for i in range(len(inputs)): if i > 0 and i % batch == 0: logger.debug('%.02f %%', 100.0 * i / len(inputs)) write_records(writer, session.run(graphs, feeds)) k = i % batch for (key, value) in inputs[i].items(): feeds[graphs[k][key]] = value k = len(inputs) % batch if k == 0: k = batch logger.debug('%.02f %%', 100) write_records(writer, session.run(graphs[0:k], feeds)) # show generation statistics start_time = time.time() n = len(scales) * len(flips) * len(rotations) m = crops logger.info('Generating %d variants (%d train, %d eval, %d test)...', n * m * len(items), n * m * len(train_items), n * m * len(eval_items), n * m * len(test_items)) train_writer = tf.python_io.TFRecordWriter( '%s-%dx%d-train.%s' % (name, size, size, ext), options) if len(train_items) > 0 else None eval_writer = tf.python_io.TFRecordWriter( '%s-%dx%d-eval.%s' % (name, size, size, ext), options) if len(eval_items) > 0 else None test_writer = tf.python_io.TFRecordWriter( '%s-%dx%d-test.%s' % (name, size, size, ext), options) if len(test_items) > 0 else None try: i = 0 for scale in scales: for flip in flips: for rotation in rotations: if i > 0: elapsed_time = (time.time() - start_time) / 60 total_time = elapsed_time * (n / i) remaining_time = total_time - elapsed_time logger.info( '[%d / %d] scale=%f, flip=%s, rotation=%f (remaining=%d[min], total=%d[min])', i + 1, n, scale, flip, rotation, remaining_time, total_time) else: logger.info('[%d / %d] scale=%f, flip=%s, rotation=%f', i + 1, n, scale, flip, rotation) i += 1 reset_tensorflow(seed) with tf.Session() as session: with tf.device(device): # build tf graph logger.info('building tf graphs...') graphs = build_graphs(batch, size, blur_radius, blur_scale, center_crop, scale, flip, rotation, crops, brightness, contrast, gaussian_noise, uniform_noise) # with io.open('%s-%dx%d.graph' % (name, size, size), 'w') as f: # f.write(str(tf.get_default_graph().as_graph_def())) # run tf graph if len(train_items) > 0: logger.info('processing samples (train)...') bake_records(session, train_writer, train_items, graphs) if len(eval_items) > 0: logger.info('processing samples (eval)...') bake_records(session, eval_writer, eval_items, graphs) if len(test_items) > 0: logger.info('processing samples (test)...') bake_records(session, test_writer, test_items, graphs) seed = seed * 11113 + seed finally: if train_writer is not None: train_writer.close() if eval_writer is not None: eval_writer.close() if test_writer is not None: test_writer.close()
def run(device, model, topology, topology_version, eval_source, train_sources, threads, prefetch, batch, steps, epochs, ram_cache, initializer, regularizer, constraint, activation, batch_normalization, local_response_normalization, dropout_rate, label_weighting, loss, optimizer, learning_rate, learning_rate_decay, verbose, seed): """ Training task implementation. """ classes = [0.0, 1.0] delta_loss_threshold = 0.01 max_stable_loss_counter = 5 max_overfit_loss_counter = 2 start_time = time.time() logger.info("Running graph on %s", device) # reset tensorflow reset_tensorflow(seed) # create model directory Path(model).mkdir(parents=True, exist_ok=True) # copy topology topologyfile = Path(model) / "topology.yaml" if not topologyfile.exists(): logger.info("Topology copied from %s", topology) topologyfile.write_text(Path(topology).read_text()) # read/update/store parameters paramsfile = Path(model) / "parameters.json" params = dict({ "batch_size": batch, "steps": steps, "epochs": epochs, "seed": seed, "classes": classes, "topology": topology, "topology_version": topology_version, "optimizer": optimizer, "learning_rate": learning_rate, "learning_rate_decay": learning_rate_decay, "loss": loss, "activation": activation, "local_response_normalization": local_response_normalization, "batch_normalization": batch_normalization, "dropout_rate": dropout_rate, "initializer": initializer, "regularizer": regularizer, "constraint": constraint, "label_weighting": label_weighting, }) if paramsfile.exists(): oldparams = json.loads(paramsfile.read_text()) for (key, value) in params.items(): if key not in oldparams: logger.info("Parameter added: %s=%s", key, str(value)) elif oldparams[key] != value: logger.info("Parameter changed: %s=%s (initial=%s)", key, value, oldparams[key]) for (key, value) in oldparams.items(): if key not in params: logger.info("Parameter obsolete: %s=%s", key, value) else: for (key, value) in params.items(): logger.debug("Parameter added: %s=%s", key, str(value)) paramsfile.write_text(json.dumps(params, indent=' ')) # read previous sessions epochsfile = Path(model) / "epochs.json" if epochsfile.exists(): epochlogs = json.loads(epochsfile.read_text()) else: epochlogs = list() epoch_offset = len(epochlogs) # read topology file logger.info("Creating model topology V%d (%s)", topology_version, topology) topology = resolve_topology(topology_version)( topologyfile, params.get("initializer", 'none'), params.get("regularizer", 'none'), params.get("constraint", 'none'), params.get("activation", 'none'), params.get("local_response_normalization", 0), params.get("batch_normalization", False), params.get("dropout_rate", 0.0)) # create tensorflow estimator estimator = tf.estimator.Estimator( model_dir=model, model_fn=topology_cnn_model(topology, verbose), params=params, config=tf.estimator.RunConfig(log_step_count_steps=10, save_summary_steps=10, save_checkpoints_secs=900, keep_checkpoint_max=epochs, tf_random_seed=seed, device_fn=lambda op: device)) logger.debug("Starting checkpoint: %s", estimator.latest_checkpoint()) # prepare inputs train_datasets = [ make_tfr_dataset(train_source, threads=threads, cache=ram_cache) for train_source in train_sources ] eval_dataset = make_tfr_dataset(eval_source, threads=threads, cache=ram_cache) # evaluate model eval_start_time = time.time() logger.info("Initial evaluation (batch=%d)", batch) results = estimator.evaluate(input_fn=make_tfr_input_fn(eval_dataset, threads=threads, prefetch=prefetch, batch=batch), ) for i in range(len(classes)): logger.info("Class %d: precision=%02.02f%%, recall=%02.02f%%", i, results['precision_c%d' % i] * 100, results['recall_c%d' % i] * 100) logger.info("Evaluation completed: loss=%f, jaccard=%f (%d[min])", results['loss'], results['jaccard'], (time.time() - eval_start_time) / 60) # run epochs train_time = time.time() best_loss = results['loss'] best_epoch = 0 best_checkpoint = estimator.latest_checkpoint() stable_loss_counter = 0 overfit_loss_counter = 0 for epoch in range(epochs): # abort training if eval getting worse if stable_loss_counter >= max_stable_loss_counter or overfit_loss_counter >= max_overfit_loss_counter: logger.info("Early-stop at epoch %d / %d (best=%d, checkpoint=%s)", epoch, epochs, best_epoch + 1, best_checkpoint) break # start epoch epoch_start_time = time.time() if epoch > 0: elapsed_time = (time.time() - train_time) / 60 total_time = elapsed_time * (epochs / epoch) remaining_time = total_time - elapsed_time logger.info( "Epoch started: %d / %d (batch=%d, remaining=%d[min], total=%d[min])", epoch + 1, epochs, batch, remaining_time, total_time) else: logger.info("Epoch started: %d / %d (batch=%d)", epoch + 1, epochs, batch) # train model for train_dataset in train_datasets: train_start_time = time.time() logger.info("Training epoch started (%d steps)", steps) estimator.train(input_fn=make_tfr_input_fn( train_dataset, threads=threads, offset=max(0, epoch * steps * batch), limit=max(0, steps * batch), shuffle=max(0, steps * batch), prefetch=prefetch, batch=batch, repeat=-1, seed=seed), steps=(steps if steps >= 0 else None)) logger.info("Training epoch completed (%d[min])", (time.time() - train_start_time) / 60) seed += seed * 11113 + seed # evaluate model eval_start_time = time.time() logger.info("Evaluation epoch started") results = estimator.evaluate(input_fn=make_tfr_input_fn( eval_dataset, threads=threads, prefetch=prefetch, batch=batch, )) if results['loss'] < best_loss: delta_loss = results['loss'] - best_loss best_loss = results['loss'] best_epoch = epoch best_checkpoint = estimator.latest_checkpoint() if abs(delta_loss) >= delta_loss_threshold * abs(results['loss']): stable_loss_counter = 0 status = 'learn(%.01e)' % delta_loss else: stable_loss_counter += 1 status = 'stable(%d/%d)' % (stable_loss_counter, max_stable_loss_counter) overfit_loss_counter = 0 elif results['loss'] > (best_loss * 1.1): stable_loss_counter += 1 overfit_loss_counter += 1 status = 'overfit(%d/%d)' % (overfit_loss_counter, max_overfit_loss_counter) else: stable_loss_counter += 1 status = 'stable(%d/%d)' % (stable_loss_counter, max_stable_loss_counter) for i in range(len(classes)): logger.info("Class %d: precision=%02.02f%%, recall=%02.02f%%", i, results['precision_c%d' % i] * 100, results['recall_c%d' % i] * 100) logger.info( "Evaluation epoch completed: status=%s, loss=%.03e, jaccard=%.03f (%d[min])", status, results['loss'], results['jaccard'], (time.time() - eval_start_time) / 60) # write epoch logs epochlogs.append({ 'epoch': epoch_offset + epoch + 1, 'checkpoint': Path(estimator.latest_checkpoint()).name, 'duration': int(time.time() - epoch_start_time), 'datasets': { 'train': train_sources, 'eval': eval_source, }, 'loss': float(results['loss']), 'learning': status.startswith('learn'), 'status': status, 'jaccard': float(results['jaccard']), 'precision': [float(results['precision_c%d' % i]) for i in range(len(classes))], 'recall': [float(results['recall_c%d' % i]) for i in range(len(classes))], 'f1': [ f1_score(results['precision_c%d' % i], results['recall_c%d' % i]) for i in range(len(classes)) ], }) epochsfile.write_text(json.dumps(epochlogs, indent=' ')) logger.info("Epoch completed (%d[min])", (time.time() - epoch_start_time) / 60) logger.info("Training completed: best epoch=%d (%d[min])", best_epoch + 1, (time.time() - train_time) / 60) logger.info("Total elapsed time: %d[min]", (time.time() - start_time) / 60)
def run(sources, threads, prefetch, batch, steps, epochs, verbose, seed): """ sources: list of tf-record files threads: number of concurrent read pipelines prefetch: reader prefetch count batch: batch size steps: number of steps per epoch epochs: number of epochs verbose: verbose output """ reset_tensorflow(seed) datasets = [ make_tfr_dataset(source, threads=threads, read_rgb=False, read_mask=False, read_weight=False) for source in sources ] global_stats = dict() global_batch_counter = 0 global_instance_counter = 0 with tf.device('/cpu:0'): for epoch in range(epochs): for dataset in datasets: input_fn = make_tfr_input_fn( dataset, threads=threads, offset=max(0, epoch*steps*batch), limit=max(0, steps*batch), shuffle=max(0, steps*batch), prefetch=prefetch, batch=batch, repeat=-1 ) stats = dict() batch_counter = 0 instance_counter = 0 with tf.Session() as session: try: it = input_fn() while batch_counter < steps: features, label = session.run(it) names = features['name'] variants = features['variant'] for i in range(names.shape[0]): key = '%s-%s' % (str(names[i], 'utf-8'), str(variants[i], 'utf-8')) if key not in stats: stats[key] = list() stats[key].append(instance_counter) instance_counter += 1 batch_counter += 1 except tf.errors.DataLossError as e: logger.warn(e) pass except tf.errors.OutOfRangeError: pass logger.info( "EPOCH[%d]: %d / %d batches, %d / %d instances", epoch, batch_counter, steps, instance_counter, steps * batch ) for key in sorted(stats.keys()): if verbose: logger.debug(" %20s: %4d %s", key, len(stats[key]), stats[key]) if key not in global_stats: global_stats[key] = 0 global_stats[key] += len(stats[key]) global_batch_counter += batch_counter global_instance_counter += instance_counter logger.info( "FINAL: %d / %d batches, %d / %d instances", global_batch_counter, epochs * len(datasets) * steps, global_instance_counter, epochs * len(datasets) * steps * batch ) min_p = 1.0 max_p = 0.0 sum_p = 0.0 for key in sorted(global_stats.keys()): p = global_stats[key] / global_instance_counter min_p = min(min_p, p) max_p = max(max_p, p) sum_p += p if verbose: logger.info(" %20s: %4d (%02.01f %%)", key, global_stats[key], 100 * p) logger.info( " %d instance, %d variants seen (C=[%d ; %d], P=[%.03e ; %.03e], E=%.03e, dE=%.03e)", global_instance_counter, len(global_stats), min_p * global_instance_counter, max_p * global_instance_counter, min_p, max_p, 1.0 / len(global_stats), 1.0 / len(global_stats) - sum_p / len(global_stats) )
def run(sources, threads, verbose, seed): """ sources: list of tf-record files threads: number of concurrent read pipelines verbose: verbose output """ reset_tensorflow(seed) global_instances = 0 global_pixels = 0 global_positives = 0 global_weighted_positives = 0 global_negatives = 0 global_weighted_negatives = 0 with tf.device('/cpu:0'): for source in sources: input_fn = make_tfr_input_fn( make_tfr_dataset(source, threads=threads, read_rgb=False, read_mask=True, read_weight=True), threads=threads, batch=10 ) instances = 0 pixels = 0 positives = 0 weighted_positives = 0 negatives = 0 weighted_negatives = 0 with tf.Session() as session: try: it = input_fn() while True: features, label = session.run(it) z = np.sum(1.0 - label) wz = np.sum((1.0 - label) * features['weight']) nz = np.sum(label) wnz = np.sum(label * features['weight']) instances += label.shape[0] pixels += label.size positives += nz weighted_positives += wnz negatives += z weighted_negatives += wz except tf.errors.DataLossError as e: logger.warn(e) pass except tf.errors.OutOfRangeError: pass global_instances += instances global_pixels += pixels global_positives += positives global_weighted_positives += weighted_positives global_negatives += negatives global_weighted_negatives += weighted_negatives logger.info( "%s: %d instances, %d pixels, %d:%d (%.02f:%.02f%%) std:weighted positives, %d:%d (%.02f:%.02f%%) std:weighted negatives", source, instances, pixels, positives, weighted_positives, 100 * positives / max(1, positives + negatives), 100 * weighted_positives / max(1, weighted_positives + weighted_negatives), negatives, weighted_negatives, 100 * negatives / max(1, positives + negatives), 100 * weighted_negatives / max(1, weighted_positives + weighted_negatives) ) logger.info( "TOTAL: %d instances, %d pixels, %d:%d (%.02f:%.02f%%) std:weighted positives, %d:%d (%.02f:%.02f%%) std:weighted negatives", global_instances, global_pixels, global_positives, global_weighted_positives, 100 * global_positives / max(1, global_positives + global_negatives), 100 * global_weighted_positives / max(1, global_weighted_positives + global_weighted_negatives), global_negatives, global_weighted_negatives, 100 * global_negatives / max(1, global_positives + global_negatives), 100 * global_weighted_negatives / max(1, global_weighted_positives + global_weighted_negatives) )
def run(device, model, sources, threads, prefetch, batch, epochs, verbose, seed): """ Prediction task implementation. """ logger.info("Running graph on %s", device) # reset tensorflow reset_tensorflow(seed) # read parameters paramsfile = Path(model) / "parameters.json" if paramsfile.exists(): params = json.loads(paramsfile.read_text()) else: raise Exception("Model parameters.json not found") # read topology file logger.info("Creating model topology V%d (%s)", params.get('topology_version', 1), params.get('topology')) topologyfile = Path(model) / "topology.yaml" if not topologyfile.exists(): raise Exception("Model topology.yaml not found") topology = resolve_topology(params.get('topology_version', 1))( topologyfile, params.get("initializer", 'none'), params.get("regularizer", 'none'), params.get("constraint", 'none'), params.get("activation", 'none'), params.get("local_response_normalization", 0), params.get("batch_normalization", False), params.get("dropout_rate", 0.0)) # create tensorflow estimator estimator = tf.estimator.Estimator( model_dir=model, model_fn=topology_cnn_model(topology, verbose), params=params, config=tf.estimator.RunConfig(tf_random_seed=seed, device_fn=lambda op: device)) logger.debug("Using checkpoint: %s", estimator.latest_checkpoint()) # predict datasets using model for source in sources: logger.info("Starting prediction: %s (batch=%d)", source, batch) if source.endswith('.tfr') or source.endswith( '.tfr.zlib') or source.endswith('.tfr.gz'): results = estimator.predict(input_fn=make_tfr_input_fn( make_tfr_dataset(source, threads=threads), threads=threads, prefetch=1, batch=batch)) else: results = estimator.predict(input_fn=make_predict_input_fn( make_predict_dataset(source, threads=threads), topology.config['input_size'], threads=threads, prefetch=1, batch=batch, repeat=epochs - 1)) if not show_results(params.get('topology'), results, topology.config['input_size'], verbose): logger.info("Aborted prediction: %s", source) break logger.info("Finished prediction: %s", source)