def post_train_detector_job(dataset_name,
run_name,
epochs,
import_datasets=""):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
dc = DatasetConfig(dataset_name)
if rc.exists and dc.exists:
cmd = [
python_path, "training_script.py",
"--name={}".format(dataset_name),
"--experiment={}".format(run_name),
"--input_shape={}".format(rc.get('detector_resolution')),
"--train_data_dir=fjlfbwjefrlbwelrfb_man_we_need_a_better_detector_codebase",
"--batch_size={}".format(rc.get('detection_training_batch_size')),
"--image_shape={}".format(dc.get('video_resolution')),
"--epochs={}".format(epochs)
]
if import_datasets:
import_datasets = quote(import_datasets)
cmd.append("--import_datasets={}".format(import_datasets))
job_id = jm.run(cmd, "train_detector")
if job_id:
return (job_id, 202)
else:
return (NoContent, 503)
else:
return (NoContent, 404)
def get_run_config(dataset_name, run_name):
dataset_name = quote(dataset_name)
rc = RunConfig(dataset_name, run_name)
if rc.exists:
return rc.get_data()
else:
return (NoContent, 404)
def post_run_config(dataset_name, run_name, run_config):
if ' ' in run_name:
return ("Spaces are not allowed in run names!", 500)
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
if rc.set_data(run_config):
rc.save()
return (NoContent, 200)
else:
return ("Could not interpret run configuration. Is some required parameter missing?", 500)
def main():
parser = ArgumentParser(description="Run Gate APIv4 demo application")
parser.add_argument("-k", "--key", required=True, help="Gate APIv4 Key")
parser.add_argument("-s",
"--secret",
required=True,
help="Gate APIv4 Secret")
parser.add_argument("-u",
"--url",
required=False,
help="API base URL used to test")
parser.add_argument("tests", nargs='+', help="tests to run")
options = parser.parse_args()
host_used = options.url
if not host_used:
host_used = "https://api.gateio.ws/api/v4"
if not host_used.startswith("http"):
host_used = "https://" + host_used
host_used = host_used.rstrip("/")
if not host_used.endswith("/api/v4"):
host_used += '/api/v4'
run_config = RunConfig(options.key, options.secret, host_used)
for t in options.tests:
logger.info("run %s API demo", t)
if t == 'spot':
spot_demo(run_config)
elif t == 'margin':
margin_demo(run_config)
elif t == 'futures':
futures_demo(run_config)
else:
logger.warning("ignore unknown test %s", t)
def post_detect_objects_job(dataset_name, run_name):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
if rc.exists:
cmd = [python_path, "detect_csv.py",
"--dataset={}".format(dataset_name),
"--run={}".format(run_name),
"--res={}".format(rc.get("detector_resolution")),
"--conf={}".format(rc.get("confidence_threshold")),
"--bs={}".format(rc.get("detection_batch_size"))]
job_id = jm.run(cmd, "detect_objects")
if job_id:
return (job_id, 202)
else:
return (NoContent, 503)
else:
return (NoContent, 404)
def post_visualize_detections_job(dataset_name, run_name, confidence_threshold, coords):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
dc = DatasetConfig(dataset_name)
if rc.exists and dc.exists:
cmd = [python_path, "visualize_detections.py",
"--cmd=findvids",
"--dataset={}".format(dataset_name),
"--run={}".format(run_name),
"--res={}".format(rc.get("detector_resolution")),
"--conf={}".format(confidence_threshold),
"--fps={}".format(dc.get('video_fps')),
"--coords={}".format(coords)]
job_id = jm.run(cmd, "visualize_detections")
if job_id:
return (job_id, 202)
else:
return (NoContent, 503)
else:
return (NoContent, 404)
def post_detections_to_world_coordinates_job(dataset_name, run_name,
make_videos):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
dc = DatasetConfig(dataset_name)
if rc.exists and dc.exists:
cmd = [
python_path, "detections_world.py", "--cmd=findvids",
"--dataset={}".format(dataset_name), "--run={}".format(run_name),
"--make_videos={}".format(make_videos),
"--ssdres={}".format(rc.get("detector_resolution")),
"--vidres={}".format(dc.get('video_resolution')),
"--kltres={}".format(dc.get('point_track_resolution'))
]
job_id = jm.run(cmd, "detections_to_world")
if job_id:
return (job_id, 202)
else:
return (NoContent, 503)
else:
return (NoContent, 404)
def post_tracking_world_coordinates_job(dataset_name, run_name, confidence_threshold, make_videos):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
dc = DatasetConfig(dataset_name)
if rc.exists and dc.exists:
cmd = [python_path, "tracking_world.py",
"--cmd=findvids",
"--dataset={}".format(dataset_name),
"--run={}".format(run_name),
"--conf={}".format(confidence_threshold),
"--make_videos={}".format(make_videos)]
job_id = jm.run(cmd, "tracking_world_coordinates")
if job_id:
return (job_id, 202)
else:
return (NoContent, 503)
else:
return (NoContent, 404)
def post_optimize_tracking_world_coordinates_job(
csv_ground_truth_file, dataset_name, run_name, date, detection_id,
class_name_conversion, visualize, patience, per_iteration):
dataset_name = quote(dataset_name)
run_name = quote(run_name)
rc = RunConfig(dataset_name, run_name)
dc = DatasetConfig(dataset_name)
if rc.exists and dc.exists:
this_run_path = runs_path / "{dn}_{rn}".format(dn=dataset_name,
rn=run_name)
csv_path = this_run_path / "world_trajectory_gt.csv"
try:
gt = csv_ground_truth_file.decode('utf-8')
except:
return ("Could not parse .csv file as UTF-8", 400)
else:
with csv_path.open('w') as f:
f.write(gt)
cmd = [
python_path, "tracking_world_optimization.py",
"--dataset={}".format(dataset_name),
"--run={}".format(run_name), "--date={}".format(date),
"--gt_csv={}".format(csv_path),
"--det_id={}".format(detection_id),
"--gt_class_name_conversion={}".format(class_name_conversion),
"--visualize={}".format(visualize),
"--patience={}".format(patience),
"--per_iteration={}".format(per_iteration)
]
job_id = jm.run(cmd, "optimize_tracking_world_coordinates")
if job_id:
return (job_id, 202)
else:
return (NoContent, 404)
else:
s = dataset_name + '_' + run_name
return (s, 404)
else:
models = glob('output/models/*.model')
models.sort(key=lambda x: os.path.getmtime(x), reverse=True)
if os.name == 'nt':
model_name = models[0].split('\\')[1].split('.')[0]
else:
model_name = models[0].replace('output/models/', '').split('.')[0]
data_file = args.prediction
prediction.predict(model_name, data_file, use_gpu=use_gpu, show_ui=args.show_ui)
exit()
#Retrieve config file and create interval config dictionary.
config_file = "configurations/" + args.config + ".config"
write_out('Using config: %s' % config_file)
configs = RunConfig(config_file)
if configs.run_params["hide_ui"]:
write_out("Live plot deactivated, see output folder for plot.")
max_seq_length, use_evolutionary, n_proteins = configs.run_params["max_sequence_length"], configs.run_params["use_evolutionary"], configs.run_params["n_proteins"]
# start web server
if not configs.run_params["hide_ui"]:
start_dashboard_server()
process_raw_data(use_gpu, n_proteins=n_proteins, max_sequence_length=max_seq_length, force_pre_processing_overwrite=False)
datafolder = "data/preprocessed/"
def loop(args):
# create config and model collection objects, and retrieve the run config
configs = {}
models = {}
configs.update({'run': RunConfig(args.config_file)})
# set GPU-related environmental options and config settings
os.environ['CUDA_VISIBLE_DEVICES'] = str(
args.gpu) if args.gpu is not None else ''
setproctitle('RGN ' + configs['run'].names['run'] + ' on ' +
os.getenv('CUDA_VISIBLE_DEVICES', 'CPU'))
# derived files and directories
base_dir = args.base_directory
print('base_dir=', base_dir)
run_dir = os.path.join(base_dir, RUNS_DIRNAME, configs['run'].names['run'],
configs['run'].names['dataset'])
print('run_dir=', run_dir)
data_dir = os.path.join(base_dir, DATAS_DIRNAME,
configs['run'].names['dataset'])
print('data_dir=', data_dir)
checkpoints_dir = os.path.join(run_dir, CHECKPOINTS_DIRNAME, '')
print('checkpoints_dir=', checkpoints_dir)
logs_dir = os.path.join(run_dir, LOGS_DIRNAME, '')
print('logs_dir=', logs_dir)
stdout_err_file = os.path.join(base_dir, LOGS_DIRNAME,
configs['run'].names['run'] + '.log')
alphabet_file = os.path.join(
data_dir, ALPHABETS_DIRNAME, configs['run'].names['alphabet'] +
'.csv') if configs['run'].names['alphabet'] is not None else None
# this is all for evaluation models (including training, so training_batch_size is for evaluation)
full_training_glob = os.path.join(data_dir, FULL_TRAINING_DIRNAME,
configs['run'].io['full_training_glob'])
sample_training_glob = os.path.join(
data_dir, FULL_TRAINING_DIRNAME,
configs['run'].io['sample_training_glob'])
training_batch_size = configs['run'].evaluation['num_training_samples']
training_invocations = configs['run'].evaluation[
'num_training_invocations']
validation_glob = os.path.join(data_dir, SAMPLE_VALIDATION_DIRNAME,
configs['run'].io['sample_validation_glob'])
validation_batch_size = configs['run'].evaluation['num_validation_samples']
validation_invocations = configs['run'].evaluation[
'num_validation_invocations']
testing_glob = os.path.join(data_dir, FULL_TESTING_DIRNAME,
configs['run'].io['full_testing_glob'])
testing_batch_size = configs['run'].evaluation['num_testing_samples']
testing_invocations = configs['run'].evaluation['num_testing_invocations']
if not args.prediction_only:
eval_num_epochs = None
else:
eval_num_epochs = 1
training_batch_size = validation_batch_size = testing_batch_size = 1
training_invocations = validation_invocations = testing_invocations = 1
# redirect stdout/err to file
sys.stderr.flush()
if not os.path.exists(os.path.dirname(stdout_err_file)):
os.makedirs(os.path.dirname(stdout_err_file))
stdout_err_file_handle = open(stdout_err_file, 'w')
os.dup2(stdout_err_file_handle.fileno(), sys.stderr.fileno())
sys.stdout = stdout_err_file_handle
# select device placement taking into consideration the interaction between training and evaluation models
if configs['run'].computing['training_device'] == 'GPU' and configs[
'run'].computing['evaluation_device'] == 'GPU':
fod_training = {'/cpu:0': ['point_to_coordinate']}
fod_evaluation = {'/cpu:0': ['point_to_coordinate']}
dd_training = ''
dd_evaluation = ''
elif configs['run'].computing['training_device'] == 'GPU' and configs[
'run'].computing['evaluation_device'] == 'CPU':
fod_training = {'/cpu:0': ['point_to_coordinate', 'loss_history']}
fod_evaluation = {}
dd_training = ''
dd_evaluation = '/cpu:0'
else:
fod_training = {}
fod_evaluation = {}
dd_training = '/cpu:0'
dd_evaluation = '/cpu:0'
# create models configuration templates
configs.update({
'training':
RGNConfig(
args.config_file, {
'name':
'training',
'dataFilesGlob':
full_training_glob,
'checkpointsDirectory':
checkpoints_dir,
'logsDirectory':
logs_dir,
'fileQueueCapacity':
configs['run'].queueing['training_file_queue_capacity'],
'batchQueueCapacity':
configs['run'].queueing['training_batch_queue_capacity'],
'minAfterDequeue':
configs['run'].queueing['training_min_after_dequeue'],
'shuffle':
configs['run'].queueing['training_shuffle'],
'tertiaryNormalization':
configs['run'].loss['training_tertiary_normalization'],
'batchDependentNormalization':
configs['run'].loss['training_batch_dependent_normalization'],
'alphabetFile':
alphabet_file,
'functionsOnDevices':
fod_training,
'defaultDevice':
dd_training,
'fillGPU':
args.fill_gpu
})
})
configs.update({
'evaluation':
RGNConfig(
args.config_file, {
'fileQueueCapacity':
configs['run'].queueing['evaluation_file_queue_capacity'],
'batchQueueCapacity':
configs['run'].queueing['evaluation_batch_queue_capacity'],
'minAfterDequeue':
configs['run'].queueing['evaluation_min_after_dequeue'],
'shuffle':
configs['run'].queueing['evaluation_shuffle'],
'tertiaryNormalization':
configs['run'].loss['evaluation_tertiary_normalization'],
'batchDependentNormalization':
configs['run'].
loss['evaluation_batch_dependent_normalization'],
'alphabetFile':
alphabet_file,
'functionsOnDevices':
fod_evaluation,
'defaultDevice':
dd_evaluation,
'numEpochs':
eval_num_epochs,
'bucketBoundaries':
None
})
})
# Override included evaluation models with list from command-line if specified (assumes none are included and then includes ones that are specified)
if args.evaluation_model:
for prefix in ['', 'un']:
for group in ['training', 'validation', 'testing']:
configs['run'].evaluation.update(
{'include_' + prefix + 'weighted_' + group: False})
for entry in args.evaluation_model:
configs['run'].evaluation.update({'include_' + entry: True})
# Override other command-lind arguments
if args.gpu_fraction:
configs['training'].computing.update(
{'gpu_fraction': args.gpu_fraction})
if args.milestone:
configs['run'].optimization.update(
{'validation_milestone': dict(args.milestone)})
# Ensure that correct validation reference is chosen if not predicting, and turn off evaluation loss if predicting
if not args.prediction_only:
if ((not configs['run'].evaluation['include_weighted_validation']) and configs['run'].optimization['validation_reference'] == 'weighted') or \
((not configs['run'].evaluation['include_unweighted_validation']) and configs['run'].optimization['validation_reference'] == 'unweighted'):
raise RuntimeError(
'Chosen validation reference is not included in run.')
else:
configs['evaluation'].loss['include'] = False
# rescaling needed to adjust for how frequently loss_history is updated
if configs['training'].curriculum['behavior'] == 'loss_change':
configs['training'].curriculum['change_num_iterations'] //= configs[
'run'].io['evaluation_frequency'] # result must be >=1
configs['evaluation'].curriculum['change_num_iterations'] //= configs[
'run'].io['evaluation_frequency'] # ditto
# create training model
models = {}
models.update({'training': RGNModel('training', configs['training'])})
print('*** training configuration ***')
pprint(configs['training'].__dict__)
# create weighted training evaluation model (conditional)
if configs['run'].evaluation['include_weighted_training']:
configs.update({'eval_wt_train': deepcopy(configs['evaluation'])})
configs['eval_wt_train'].io['name'] = 'evaluation_wt_training'
configs['eval_wt_train'].io['data_files_glob'] = sample_training_glob
configs['eval_wt_train'].optimization[
'batch_size'] = training_batch_size
configs['eval_wt_train'].queueing[
'num_evaluation_invocations'] = training_invocations
models.update({
'eval_wt_train':
RGNModel('evaluation', configs['eval_wt_train'])
})
print('\n\n\n*** weighted training evaluation configuration ***')
pprint(configs['eval_wt_train'].__dict__)
# create weighted validation evaluation model (conditional)
if configs['run'].evaluation['include_weighted_validation']:
configs.update({'eval_wt_val': deepcopy(configs['evaluation'])})
configs['eval_wt_val'].io['name'] = 'evaluation_wt_validation'
configs['eval_wt_val'].io['data_files_glob'] = validation_glob
configs['eval_wt_val'].optimization[
'batch_size'] = validation_batch_size
configs['eval_wt_val'].queueing[
'num_evaluation_invocations'] = validation_invocations
if configs['run'].optimization['validation_reference'] == 'weighted':
configs['eval_wt_val'].curriculum['update_loss_history'] = True
models.update(
{'eval_wt_val': RGNModel('evaluation', configs['eval_wt_val'])})
print('\n\n\n*** weighted validation evaluation configuration ***')
pprint(configs['eval_wt_val'].__dict__)
# create weighted testing evaluation model (conditional)
if configs['run'].evaluation['include_weighted_testing']:
configs.update({'eval_wt_test': deepcopy(configs['evaluation'])})
configs['eval_wt_test'].io['name'] = 'evaluation_wt_testing'
configs['eval_wt_test'].io['data_files_glob'] = testing_glob
configs['eval_wt_test'].optimization['batch_size'] = testing_batch_size
configs['eval_wt_test'].queueing[
'num_evaluation_invocations'] = testing_invocations
models.update(
{'eval_wt_test': RGNModel('evaluation', configs['eval_wt_test'])})
print('\n\n\n*** weighted testing evaluation configuration ***')
pprint(configs['eval_wt_test'].__dict__)
# create equivalents for unweighted loss if there's a curriculum.
if configs['training'].curriculum['mode'] is not None:
# create unweighted training evaluation model (conditional)
if configs['run'].evaluation['include_unweighted_training']:
configs.update(
{'eval_unwt_train': deepcopy(configs['evaluation'])})
configs['eval_unwt_train'].io['name'] = 'evaluation_unwt_training'
configs['eval_unwt_train'].io[
'data_files_glob'] = sample_training_glob
configs['eval_unwt_train'].optimization[
'batch_size'] = training_batch_size
configs['eval_unwt_train'].queueing[
'num_evaluation_invocations'] = training_invocations
configs['eval_unwt_train'].curriculum['mode'] = None
configs['eval_unwt_train'].curriculum['behavior'] = None
models.update({
'eval_unwt_train':
RGNModel('evaluation', configs['eval_unwt_train'])
})
# create unweighted validation evaluation model (conditional)
if configs['run'].evaluation['include_unweighted_validation']:
configs.update({'eval_unwt_val': deepcopy(configs['evaluation'])})
configs['eval_unwt_val'].io['name'] = 'evaluation_unwt_validation'
configs['eval_unwt_val'].io['data_files_glob'] = validation_glob
configs['eval_unwt_val'].optimization[
'batch_size'] = validation_batch_size
configs['eval_unwt_val'].queueing[
'num_evaluation_invocations'] = validation_invocations
configs['eval_unwt_val'].curriculum['mode'] = None
configs['eval_unwt_val'].curriculum['behavior'] = None
if configs['run'].optimization[
'validation_reference'] == 'unweighted':
configs['eval_unwt_val'].curriculum[
'update_loss_history'] = True
models.update({
'eval_unwt_val':
RGNModel('evaluation', configs['eval_unwt_val'])
})
# create unweighted testing evaluation model (conditional)
if configs['run'].evaluation['include_unweighted_testing']:
configs.update({'eval_unwt_test': deepcopy(configs['evaluation'])})
configs['eval_unwt_test'].io['name'] = 'evaluation_unwt_testing'
configs['eval_unwt_test'].io['data_files_glob'] = testing_glob
configs['eval_unwt_test'].optimization[
'batch_size'] = testing_batch_size
configs['eval_unwt_test'].queueing[
'num_evaluation_invocations'] = testing_invocations
configs['eval_unwt_test'].curriculum['mode'] = None
configs['eval_unwt_test'].curriculum['behavior'] = None
models.update({
'eval_unwt_test':
RGNModel('evaluation', configs['eval_unwt_test'])
})
# start head model and related prep
stdout_err_file_handle.flush()
session = models['training'].start(models.values())
global_step = models['training'].current_step(session)
current_log_step = (global_step //
configs['run'].io['prediction_frequency']) + 1
log_dir = os.path.join(run_dir, str(current_log_step))
restart = False
# predict or train depending on set mode behavior
if args.prediction_only:
try:
while not models['training'].is_done():
predict_and_log(log_dir, configs, models, session)
except tf.errors.OutOfRangeError:
pass
except:
print('Unexpected error: ', sys.exc_info()[0])
raise
finally:
if models['training']._is_started:
models['training'].finish(session, save=False)
stdout_err_file_handle.close()
else:
# clean up post last checkpoint residue if any
if global_step != 0:
# remove future directories
last_log_step = sorted([
int(os.path.basename(os.path.normpath(dir)))
for dir in glob(os.path.join(run_dir, '*[0-9]'))
])[-1]
for step in range(current_log_step + 1, last_log_step + 1):
rmtree(os.path.join(run_dir, str(step)))
# remove future log entries in current log files
log_file = os.path.join(log_dir, 'error.log')
if os.path.exists(log_file):
with open(log_file, 'rw+') as f:
while True:
new_line = f.readline().split()
if len(new_line) > 1:
step = int(new_line[1])
if step == global_step:
f.truncate()
break
else: # reached end without seeing global_step, means checkpoint is ahead of last recorded log entry
break
# training loop
try:
while not models['training'].is_done():
# Train for one step
global_step, ids = models['training'].train(session)
# Set and create logging directory and files if needed
log_dir = os.path.join(
run_dir,
str((global_step //
configs['run'].io['prediction_frequency']) + 1))
log_file = os.path.join(log_dir, 'error.log')
if not os.path.exists(log_dir): os.makedirs(log_dir)
# Evaluate error, get diagnostics, and raise exceptions if necessary
if global_step % configs['run'].io['evaluation_frequency'] == 0:
diagnostics = evaluate_and_log(log_file, configs, models,
session)
# restart if a milestone is missed
val_ref_set_prefix = 'un' if configs['run'].optimization[
'validation_reference'] == 'unweighted' else ''
min_loss_achieved = diagnostics[
val_ref_set_prefix +
'wt_val_loss']['min_tertiary_loss_achieved_all']
for step, loss in configs['run'].optimization[
'validation_milestone'].iteritems():
if global_step >= step and min_loss_achieved > loss:
raise MilestoneError('Milestone at step ' + str(global_step) + \
' missed because minimum loss achieved so far is ' + str(min_loss_achieved))
# restart if gradients are zero
if (diagnostics['min_grad'] == 0 and diagnostics['max_grad'] == 0) or \
(configs['run'].evaluation['include_diagnostics'] and (np.isnan(diagnostics['min_grad']) or np.isnan(diagnostics['max_grad']))):
raise DeadGradientError('Gradient is dead.')
# Predict structures. Currently assumes that weighted training and validation models are available, and fails if they're not.
if global_step % configs['run'].io['prediction_frequency'] == 0:
predict_and_log(log_dir, configs, models, session)
# Checkpoint
if global_step % configs['run'].io['checkpoint_frequency'] == 0:
models['training'].save(session)
except tf.errors.OutOfRangeError:
print('Epoch limit reached.')
except (tf.errors.InvalidArgumentError, DeadGradientError
): # InvalidArgumentError is usually triggered by a nan
models['training'].finish(session, save=False)
if args.restart_on_dead_gradient:
print(
'Nan or dead gradient encountered; model will be resumed from last checkpoint if one exists, or restarted from scratch otherwise.'
)
if not os.path.isdir(checkpoints_dir):
for sub_dir in next(os.walk(run_dir))[1]:
rmtree(os.path.join(
run_dir, sub_dir)) # erase all old directories
restart = True
else:
print(
'Nan or dead gradient encountered; model will be terminated.'
)
except MilestoneError:
models['training'].finish(session, save=False)
if args.restart_on_missed_milestone:
print(
'Milestone missed; model will be restarted from scratch with an incremented seed.'
)
for sub_dir in next(os.walk(run_dir))[1]:
rmtree(os.path.join(run_dir,
sub_dir)) # erase all old directories
# modify configuration file with new seed
old_seed = configs['training'].initialization['graph_seed']
new_seed = old_seed + args.seed_increment
for line in fileinput.input(args.config_file, inplace=True):
print line.replace('randSeed ' + str(old_seed),
'randSeed ' + str(new_seed)),
restart = True
else:
print('Milestone missed; model will be terminated.')
except:
print('Unexpected error: ', sys.exc_info()[0])
raise
finally:
# Wrap up (ask threads to stop, save final checkpoint, etc.)
if models['training']._is_started:
models['training'].finish(session,
save=args.checkpoint_on_finish)
stdout_err_file_handle.close()
return restart
def main(dataset, run, n_clips, clip_length):
dc = DatasetConfig(dataset)
rc = RunConfig(dataset, run)
mask = Masker(dataset)
classes = get_classnames(dataset)
num_classes = len(classes) + 1
calib = Calibration(dataset)
dataset_path = "{dsp}{ds}/".format(dsp=datasets_path, ds=dataset)
run_path = "{rp}{ds}_{r}/".format(rp=runs_path, ds=dataset, r=run)
# Grab a bunch of videos
vids_query = "{dsp}videos/*.mkv".format(dsp=dataset_path)
all_vids = glob(vids_query)
all_vids = [right_remove(x.split('/')[-1], '.mkv') for x in all_vids]
all_vids.sort()
vids = []
if n_clips > len(all_vids):
n_clips = len(all_vids)
if n_clips == len(all_vids):
vids = all_vids
else:
while len(vids) < n_clips:
vid = choice(all_vids)
if not vid in vids:
vids.append(vid)
print_flush(vids)
# Find out what has been run on all of these videos, what to include
include_klt = True
include_pixeldets = True
include_worlddets = True
include_worldtracks = True
klts = []
pixeldets = []
worlddets = []
worldtracks = []
# Point tracks need to be converted for faster access
vidres = dc.get('video_resolution')
kltres = dc.get('point_track_resolution')
class KLTConfig(object):
klt_x_factor = 0
klt_y_factor = 0
klt_config = KLTConfig()
klt_config.klt_x_factor = vidres[0] / kltres[0]
klt_config.klt_y_factor = vidres[1] / kltres[1]
ssdres = rc.get('detector_resolution')
x_scale = vidres[0] / ssdres[0]
y_scale = vidres[1] / ssdres[1]
colors = class_colors(num_classes)
for vid in vids:
f = get_klt_path(dataset_path, vid)
if not isfile(f):
include_klt = False
else:
klt = load(f)
klt, klt_frames = convert_klt(klt, klt_config)
pts = (klt, klt_frames, class_colors(n_cols_klts))
klts.append(pts)
f = get_pixeldet_path(run_path, vid)
if not isfile(f):
include_pixeldets = False
else:
dets = pd.read_csv(f)
pixeldets.append((dets, colors, x_scale, y_scale))
f = get_worlddet_path(run_path, vid)
if not isfile(f):
include_worlddets = False
else:
dets = pd.read_csv(f)
worlddets.append((dets, colors, calib))
f = get_worldtracks_path(run_path, vid)
if not isfile(f):
include_worldtracks = False
else:
tracks = load(f)
worldtracks.append((tracks, class_colors(n_cols_tracks), calib))
print_flush("Point tracks: {}".format(include_klt))
print_flush("Pixel coordinate detections: {}".format(include_pixeldets))
print_flush("World coordinate detections: {}".format(include_worlddets))
print_flush("World coordinate tracks: {}".format(include_worldtracks))
# Decide where to start and stop in the videos
clip_length = clip_length * dc.get(
'video_fps') # convert from seconds to frames
print_flush("Clip length in frames: {}".format(clip_length))
clips = []
for vid in vids:
start, stop = make_clip(vid, clip_length, dataset_path)
clips.append((start, stop))
incs = [
include_klt, include_pixeldets, include_worlddets, include_worldtracks
]
funs = [klt_frame, pixeldet_frame, worlddet_frame, worldtracks_frame]
dats = [klts, pixeldets, worlddets, worldtracks]
nams = [
"Point tracks", "Detections in pixel coordinates",
"Detections in world coordinates", "Tracks in world coordinates"
]
print_flush(clips)
with iio.get_writer("{trp}summary.mp4".format(trp=run_path),
fps=dc.get('video_fps')) as outvid:
for i_vid, vid in enumerate(vids):
print_flush(vid)
old_prog = 0
with iio.get_reader("{dsp}videos/{v}.mkv".format(dsp=dataset_path,
v=vid)) as invid:
start, stop = clips[i_vid]
for i_frame in range(start, stop):
frame = invid.get_data(i_frame)
pieces = []
for inc, fun, dat, nam in zip(incs, funs, dats, nams):
if inc:
piece = fun(dat[i_vid],
mask.mask(frame.copy(), alpha=0.5),
i_frame)
draw_text(piece, vid, i_frame, nam)
pieces.append(piece)
outvid.append_data(join(pieces))
prog = float(i_frame - start) / (stop - start)
if prog - old_prog > 0.1:
print_flush("{}%".format(round(prog * 100)))
old_prog = prog
print_flush("Done!")
a = []
a.append(df.iloc[i]['FirstWord'])
a.append(df.iloc[i]['SecondWord'])
texts.append(a)
for i in range(len(df)):
labels.append(df.iloc[i]['Relation'])
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
c = RunConfig()
# Loading tokenizer, configuration, and model
tokenizer = BertTokenizer.from_pretrained(c.pretrained_transformer)
filepath = "./datasets/CNItalianDataFiltered.csv"
load(filepath)
finaldict = featurize_texts(texts,
tokenizer,
labels,
max_length=128,
add_special_tokens=True,
is_text_pair=True,
has_toktype_ids=True)
import torch
import os
from config import RunConfig, ParallelAdapterBertConfig, BottleneckAdapterBertConfig
import torch
import sys
logger = logging.getLogger(__name__)
# Setup logging
logging.basicConfig(
format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
level=logging.INFO,
)
c = RunConfig()
bertonja = BertModel.from_pretrained(c.pretrained_transformer)
train_data = torch.load(c.train_set)
eval_data = torch.load(c.val_set)
if not os.path.exists(c.output_dir):
os.makedirs(c.output_dir)
# Set seed
trainer.set_seed(c.seed)
# Loading tokenizer, configuration, and model
tokenizer = BertTokenizer.from_pretrained(c.pretrained_transformer)
match = [f for f in os.listdir(c.output_dir) if f.startswith("best")]
def predict(args):
# create config and model collection objects, and retrieve the run config
configs = {}
models = {}
configs.update({"run": RunConfig(args.config_file)})
# set GPU-related environmental options and config settings
os.environ["CUDA_VISIBLE_DEVICES"] = str(
args.gpu) if args.gpu is not None else ""
setproctitle("RGN " + configs["run"].names["run"] + " on " +
os.getenv("CUDA_VISIBLE_DEVICES", "CPU"))
# derived files and directories
checkpoints_dir = args.checkpoint
logs_dir = os.path.join(LOGS_DIRNAME, "")
# this is all for evaluation models (including training, so training_batch_size is for evaluation)
full_training_glob = args.input_file
training_invocations = configs["run"].evaluation[
"num_training_invocations"]
testing_glob = args.input_file
testing_batch_size = configs["run"].evaluation["num_testing_samples"]
testing_invocations = configs["run"].evaluation["num_testing_invocations"]
eval_num_epochs = 1
training_batch_size = validation_batch_size = testing_batch_size = 1
training_invocations = validation_invocations = testing_invocations = 1
# select device placement taking into consideration the interaction between training and evaluation models
if (configs["run"].computing["training_device"] == "GPU"
and configs["run"].computing["evaluation_device"] == "GPU"):
fod_training = {"/cpu:0": ["point_to_coordinate"]}
fod_evaluation = {"/cpu:0": ["point_to_coordinate"]}
dd_training = ""
dd_evaluation = ""
elif (configs["run"].computing["training_device"] == "GPU"
and configs["run"].computing["evaluation_device"] == "CPU"):
fod_training = {"/cpu:0": ["point_to_coordinate", "loss_history"]}
fod_evaluation = {}
dd_training = ""
dd_evaluation = "/cpu:0"
else:
fod_training = {}
fod_evaluation = {}
dd_training = "/cpu:0"
dd_evaluation = "/cpu:0"
# create models configuration templates
configs.update({
"training":
RGNConfig(
args.config_file,
{
"name":
"training",
"dataFilesGlob":
full_training_glob,
"checkpointsDirectory":
checkpoints_dir,
"logsDirectory":
logs_dir,
"fileQueueCapacity":
configs["run"].queueing["training_file_queue_capacity"],
"batchQueueCapacity":
configs["run"].queueing["training_batch_queue_capacity"],
"minAfterDequeue":
configs["run"].queueing["training_min_after_dequeue"],
"shuffle":
configs["run"].queueing["training_shuffle"],
"tertiaryNormalization":
configs["run"].loss["training_tertiary_normalization"],
"batchDependentNormalization":
configs["run"].loss["training_batch_dependent_normalization"],
# "alphabetFile": None,
"functionsOnDevices":
fod_training,
"defaultDevice":
dd_training,
"fillGPU":
args.fill_gpu,
},
)
})
configs.update({
"evaluation":
RGNConfig(
args.config_file,
{
"fileQueueCapacity":
configs["run"].queueing["evaluation_file_queue_capacity"],
"batchQueueCapacity":
configs["run"].queueing["evaluation_batch_queue_capacity"],
"minAfterDequeue":
configs["run"].queueing["evaluation_min_after_dequeue"],
"shuffle":
configs["run"].queueing["evaluation_shuffle"],
"tertiaryNormalization":
configs["run"].loss["evaluation_tertiary_normalization"],
"batchDependentNormalization":
configs["run"].
loss["evaluation_batch_dependent_normalization"],
# "alphabetFile": alphabet_file,
"functionsOnDevices":
fod_evaluation,
"defaultDevice":
dd_evaluation,
"numEpochs":
eval_num_epochs,
"bucketBoundaries":
None,
},
)
})
# Override included evaluation models with list from command-line if specified (assumes none are included and then includes ones that are specified)
for prefix in ["", "un"]:
for group in ["training", "validation", "testing"]:
configs["run"].evaluation.update(
{"include_" + prefix + "weighted_" + group: False})
for entry in args.evaluation_model:
configs["run"].evaluation.update({"include_" + entry: True})
# Override other command-lind arguments
if args.gpu_fraction:
configs["training"].computing.update(
{"gpu_fraction": args.gpu_fraction})
configs["evaluation"].loss["include"] = False
# create training model
models = {}
models.update({"training": RGNModel("training", configs["training"])})
# print('*** training configuration ***')
# pprint(configs['training'].__dict__)
# create weighted testing evaluation model (conditional)
if configs["run"].evaluation["include_weighted_testing"]:
configs.update({"eval_wt_test": deepcopy(configs["evaluation"])})
configs["eval_wt_test"].io["name"] = "evaluation_wt_testing"
configs["eval_wt_test"].io["data_files_glob"] = testing_glob
configs["eval_wt_test"].optimization["batch_size"] = testing_batch_size
configs["eval_wt_test"].queueing[
"num_evaluation_invocations"] = testing_invocations
models.update(
{"eval_wt_test": RGNModel("evaluation", configs["eval_wt_test"])})
# print('\n\n\n*** weighted testing evaluation configuration ***')
# pprint(configs['eval_wt_test'].__dict__)
# start head model and related prep
session = models["training"].start(models.values())
global_step = models["training"].current_step(session)
current_log_step = (global_step //
configs["run"].io["prediction_frequency"]) + 1
# predict or train depending on set mode behavior
result = {}
try:
while not models["training"].is_done():
pred = predict_tertiary(configs, models, session)
if pred is not None:
idx, tertiary = pred
result[idx] = tertiary
except tf.errors.OutOfRangeError:
pass
except:
print("Unexpected error: ", sys.exc_info()[0])
raise
finally:
if models["training"]._is_started:
models["training"].finish(session, save=False)
return result