def main(nw_src, gt_src, node_label_src, num_find, nclus, verbose=True, unique_seeds=False):
start = time.time()
gts = common.load_comms(gt_src)
if len(gts) == 0:
print "Error! No training communities of size >3 found. Cannot run Bespoke."
return None
if len(gts) < nclus:
print "Error! Too few (<#patterns,",nclus,") training communities of size >3 found. Cannot run Bespoke."
return None
node_labels = common.load_labels(node_label_src)
nw = common.load_SimpleNW_graph(nw_src)
if len(nw.nodes()) != len(node_labels.keys()):
print "Error! Number of labeled nodes does not match number of nodes in the graph. #NW nodes=", len(nw.nodes()),"#labeled nodes=", len(node_labels.keys())
return None
if verbose:
print "Training..."
sys.stdout.flush()
ret_list = train.train(nw, gts, node_labels, nclus)
if verbose:
print "Training complete...\nBeginning extraction..."
sys.stdout.flush()
KM_obj, size_dist_per_group, seed_info_per_group = ret_list
end_train = time.time()
comms = get_comms(nw, num_find, seed_info_per_group, size_dist_per_group, KM_obj, node_labels, unique_seeds)
end = time.time()
tot_time, train_time = round(end-start,2),round(end_train-start,2)
if verbose:
print "Extraction complete."
sys.stdout.flush()
return comms, KM_obj, tot_time, train_time
def serve():
default_model_dir = '/media/mendel/detection-server/models'
default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
default_labels = 'coco_labels.txt'
parser = argparse.ArgumentParser()
parser.add_argument('--model', help='.tflite model path',
default=os.path.join(default_model_dir,default_model))
parser.add_argument('--labels', help='label file path',
default=os.path.join(default_model_dir, default_labels))
parser.add_argument('--camera_idx', type=int, help='Index of which video source to use. ', default=1)
parser.add_argument('--threshold', type=float, help='Detector threshold. ', default=0.7)
parser.add_argument('--display', dest='display', action='store_true', help='Display object data. ')
parser.set_defaults(display=False)
args = parser.parse_args()
print('Loading {} with {} labels.'.format(args.model, args.labels))
interpreter = common.make_interpreter(os.path.join(default_model_dir, args.model))
interpreter.allocate_tensors()
labels = common.load_labels(os.path.join(default_model_dir, args.labels))
# Get native camera resolution.
cap = cv2.VideoCapture(args.camera_idx)
camera_res = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
cap.release()
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
# Start a thread to detect objects in camera frames.
future = executor.submit(start_detector, args.camera_idx, interpreter,
args.threshold, labels, camera_res, args.display)
# Start other threads for the gprc server.
server = grpc.server(executor)
detection_server_pb2_grpc.add_DetectionServerServicer_to_server(
DetectionServerServicer(camera_res), server)
server.add_insecure_port('[::]:50051')
server.start()
# Show the value returned by the executor.submit call.
# This will wait forever unless a runtime error is encountered.
future.result()
server.stop(None)
def main(argv):
args = argparser().parse_args(argv[1:])
tokenizer = tokenization.FullTokenizer(
vocab_file=args.vocab_file,
do_lower_case=args.do_lower_case
)
label_list = load_labels(args.labels)
label_map = { l: i for i, l in enumerate(label_list) }
examples = []
for x, y in tsv_generator(args.input_file, tokenizer, label_map, args):
examples.append(Example(x, y))
if args.max_examples and len(examples) >= args.max_examples:
break
write_examples(examples, args.output_file)
return 0
def main(argv):
print_versions()
args = argument_parser('train').parse_args(argv[1:])
args.train_data = args.train_data.split(',')
if args.checkpoint_steps is not None:
os.makedirs(args.checkpoint_dir, exist_ok=True)
strategy = MirroredStrategy()
num_devices = strategy.num_replicas_in_sync
# Batch datasets with global batch size (local * GPUs)
global_batch_size = args.batch_size * num_devices
tokenizer = get_tokenizer(args)
label_list = load_labels(args.labels)
label_map = { l: i for i, l in enumerate(label_list) }
inv_label_map = { v: k for k, v in label_map.items() }
if args.task_name not in (["NER","RE"]):
raise ValueError("Task not found: {}".format(args.task_name))
if args.train_data[0].endswith('.tsv'):
if len(args.train_data) > 1:
raise NotImplementedError('Multiple TSV inputs')
train_data = TsvSequence(args.train_data[0], tokenizer, label_map,
global_batch_size, args)
input_format = 'tsv'
elif args.train_data[0].endswith('.tfrecord'):
train_data = train_tfrecord_input(args.train_data, args.max_seq_length,
global_batch_size)
input_format = 'tfrecord'
else:
raise ValueError('--train_data must be .tsv or .tfrecord')
if args.dev_data is None:
dev_x, dev_y = None, None
validation_data = None
else:
dev_x, dev_y = load_dataset(args.dev_data, tokenizer,
args.max_seq_length,
label_map, args)
validation_data = (dev_x, dev_y)
print('Number of devices: {}'.format(num_devices), file=sys.stderr,
flush=True)
if num_devices > 1 and input_format != 'tfrecord':
warning('TFRecord input recommended for multi-device training')
num_train_examples = num_examples(args.train_data)
num_labels = len(label_list)
print('num_train_examples: {}'.format(num_train_examples),
file=sys.stderr, flush=True)
with strategy.scope():
model = restore_or_create_model(num_train_examples, num_labels,
global_batch_size, args)
model.summary(print_fn=print)
callbacks = []
if args.checkpoint_steps is not None:
callbacks.append(ModelCheckpoint(
filepath=os.path.join(args.checkpoint_dir, CHECKPOINT_NAME),
save_freq=args.checkpoint_steps
))
callbacks.append(DeleteOldCheckpoints(
args.checkpoint_dir, CHECKPOINT_NAME, args.max_checkpoints
))
if input_format == 'tsv':
other_args = {
'workers': 10, # TODO
}
else:
assert input_format == 'tfrecord', 'internal error'
steps_per_epoch = int(np.ceil(num_train_examples/global_batch_size))
other_args = {
'steps_per_epoch': steps_per_epoch
}
model.fit(
train_data,
epochs=args.num_train_epochs,
callbacks=callbacks,
validation_data=validation_data,
validation_batch_size=global_batch_size,
**other_args
)
if validation_data is not None:
probs = model.predict(dev_x, batch_size=global_batch_size)
preds = np.argmax(probs, axis=-1)
correct, total = sum(g==p for g, p in zip(dev_y, preds)), len(dev_y)
print('Final dev accuracy: {:.1%} ({}/{})'.format(
correct/total, correct, total))
if args.model_dir is not None:
print('Saving model in {}'.format(args.model_dir))
save_model_etc(model, tokenizer, label_list, args)
return 0
def main():
default_model_dir = '/media/mendel/detection-server/models'
default_model = 'mobilenet_ssd_v2_coco_quant_postprocess_edgetpu.tflite'
default_labels = 'coco_labels.txt'
default_capture_labels = ['person', 'dog', 'cat']
default_capture_dir = '/media/mendel/detection-server/images'
parser = argparse.ArgumentParser()
parser.add_argument('--model',
help='.tflite model path',
default=os.path.join(default_model_dir, default_model))
parser.add_argument('--labels',
help='Label file path.',
default=os.path.join(default_model_dir,
default_labels))
parser.add_argument('--images',
help='Captured image path.',
default=default_capture_dir)
parser.add_argument('--capture',
nargs='+',
type=str,
help='Labels to capture.',
default=default_capture_labels)
parser.add_argument('--num_samples',
type=int,
help='Number of samples to capture.',
default=100)
parser.add_argument('--camera_idx',
type=int,
help='Index of which video source to use.',
default=1)
parser.add_argument('--threshold',
type=float,
help='Detector threshold.',
default=0.1)
parser.add_argument('--frame_rate',
type=float,
help='Capture frame rate.',
default=2.0)
parser.add_argument('--display',
dest='display',
action='store_true',
help='Display object data.')
parser.set_defaults(display=False)
parse_args = parser.parse_args()
print('Loading {} with {} labels.'.format(parse_args.model,
parse_args.labels))
interpreter = common.make_interpreter(
os.path.join(default_model_dir, parse_args.model))
interpreter.allocate_tensors()
labels = common.load_labels(
os.path.join(default_model_dir, parse_args.labels))
# Get native camera resolution.
cap = cv2.VideoCapture(parse_args.camera_idx)
camera_res = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(
cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
cap.release()
# Create named directories for captured images.
for name in parse_args.capture:
dir = os.path.join(parse_args.images, name)
try:
os.mkdir(dir)
print('Created {}'.format(dir))
except FileExistsError:
pass
capture(parse_args, interpreter, labels, camera_res)