def verify_socket_directory(self, user, hosts):
"""Verify the domain socket directory is present and owned by this user.
Args:
user (str): user to verify has ownership of the directory
hosts (list): list of hosts on which to verify the directory exists
Raises:
CommandFailure: if the socket directory does not exist or is not
owned by the user and could not be created
"""
if isinstance(self.yaml, YamlParameters):
directory = self.get_user_file()
self.log.info("Verifying %s socket directory: %s", self.command,
directory)
status, nodes = check_file_exists(hosts, directory, user)
if not status:
self.log.info(
"%s: creating socket directory %s for user %s on %s",
self.command, directory, user, nodes)
try:
create_directory(nodes, directory, sudo=True)
change_file_owner(nodes, directory, user, user, sudo=True)
except DaosTestError as error:
raise CommandFailure(
"{}: error setting up missing socket directory {} for "
"user {} on {}:\n{}".format(self.command, directory,
user, nodes, error))
def copy_certificates(self, source, hosts):
"""Copy certificates files from the source to the destination hosts.
Args:
source (str): source of the certificate files.
hosts (list): list of the destination hosts.
"""
names = set()
yaml = self.yaml
while yaml is not None and hasattr(yaml, "other_params"):
if hasattr(yaml, "get_certificate_data"):
self.log.debug("Copying certificates for %s:", self._command)
data = yaml.get_certificate_data(
yaml.get_attribute_names(LogParameter))
for name in data:
create_directory(hosts,
name,
verbose=False,
raise_exception=False)
for file_name in data[name]:
src_file = os.path.join(source, file_name)
dst_file = os.path.join(name, file_name)
self.log.debug(" %s -> %s", src_file, dst_file)
result = distribute_files(hosts,
src_file,
dst_file,
mkdir=False,
verbose=False,
raise_exception=False,
sudo=True,
owner=self.certificate_owner)
if result.exit_status != 0:
self.log.info(
" WARNING: %s copy failed on %s:\n%s",
dst_file, hosts, result)
names.add(name)
yaml = yaml.other_params
# debug to list copy of cert files
if names:
self.log.debug("Copied certificates for %s (in %s):",
self._command, ", ".join(names))
for line in get_file_listing(hosts,
names).stdout_text.splitlines():
self.log.debug(" %s", line)
def main(input_files, output_directory, mode, minimum_sequence_length, refs,
bait_size, bait_offset, bait_identity, bait_coverage, minimum_region,
minimum_exact_match, cluster_probes, cluster_identity,
cluster_coverage, exclude_regions, exclude_pident, exclude_coverage,
threads, report, report_identities, report_coverages, fixed_xaxis,
fixed_yaxis, tsv_output):
# create output directory if it does not exist
exists = gu.create_directory(output_directory)
# get absolute paths for all input files
genomes = gu.absolute_paths(input_files)
# read file with references basenames
with open(refs, 'r') as infile:
ref_set = [g[0] for g in list(csv.reader(infile, delimiter='\t'))]
ref_set = [os.path.join(input_files, g) for g in ref_set]
# reorder genome list
genomes = ref_set + [g for g in genomes if g not in ref_set]
# attribute shorter and unique identifiers to each input
short_samples = gu.common_suffixes(genomes)
inv_short = {v: k for k, v in short_samples.items()}
ref_set = [short_samples[g] for g in ref_set]
# determine number of sequences and total length per input file
nr_contigs = {
short_samples[f]: gu.count_contigs(f, minimum_sequence_length)
for f in genomes
}
# shred refs to get initial set of baits
# not generating kmers that cover the end of the sequences!
baits_file = os.path.join(output_directory, 'baits.fasta')
for g in ref_set:
nr_baits = gu.generate_baits(inv_short[g], baits_file, bait_size,
bait_offset, minimum_sequence_length)
print('\nCreated initial set of {0} probes based on {1} '
'inputs.'.format(nr_baits, len(ref_set)))
# identify unique baits
unique_baits = os.path.join(output_directory, 'unique_baits.fasta')
total, unique_seqids = gu.determine_distinct(baits_file, unique_baits)
print('Removed {0} repeated probes.\n'.format(total))
# start mapping baits against remaining genomes
# mapping against ref_set to cover missing regions
bait_creation_dir = os.path.join(output_directory,
'incremental_bait_creation')
exists = gu.create_directory(bait_creation_dir)
total = 0
coverage_info = {}
discarded_baits_files = []
missing_files = []
processed = 0
for g in genomes:
generated = incremental_bait_generator(g,
unique_baits,
bait_creation_dir,
bait_size,
bait_coverage,
bait_identity,
bait_offset,
minimum_region,
nr_contigs,
short_samples,
minimum_exact_match,
generate=True,
depth=False)
coverage_info[short_samples[g]] = generated[0]
discarded_baits_files.append(generated[2])
missing_files.append(generated[3])
total += generated[1]
processed += 1
print('\r',
'Generated baits for {0}/{1} '
'inputs.'.format(processed, len(genomes)),
end='')
print('\nAdded {0} probes to cover {1} assemblies.\nTotal '
'of {2} probes.'.format(total, len(genomes), nr_baits + total))
if cluster_probes is True:
clustering_dir = os.path.join(output_directory, 'clustering')
exists = gu.create_directory(clustering_dir)
unique_baits, removed = exclude_similar_probes(unique_baits,
clustering_dir,
cluster_identity,
cluster_coverage,
bait_size, threads)
total -= len(removed)
# need to copy the Fasta file without the excluded baits to the main output directory!
exclude_stats = 'None'
if exclude_regions is not None:
exclude_dir = os.path.join(output_directory, 'exclude')
exists = gu.create_directory(exclude_dir)
unique_baits, removed = exclude_contaminant(unique_baits,
exclude_regions,
exclude_pident,
exclude_coverage,
bait_size, exclude_dir)
# determine number of exclude regions and total bps
exclude_stats = [
len(rec) for rec in SeqIO.parse(exclude_regions, 'fasta')
]
exclude_stats = len(exclude_stats)
total -= len(removed)
# create TSV output with bait identifier and bait sequence columns
if tsv_output is True:
tsv_output_file = os.path.join(output_directory, 'unique_baits.tsv')
tsv_baits = write_tsv_output(unique_baits, tsv_output_file)
print('Wrote {0} baits to {1}'.format(tsv_baits, tsv_output_file))
if report is True:
# create report directory
report_dir = os.path.join(output_directory, 'report_data')
exists = gu.create_directory(report_dir)
# determine contig order from longest to shortest
ordered_contigs = gu.order_contigs(genomes, short_samples)
# create dict with config values
configs = {
'Number of inputs': len(genomes),
'Minimum sequence length': minimum_sequence_length,
'Number of references': len(ref_set),
'Bait size': bait_size,
'Bait offset': bait_offset,
'Bait identity': bait_identity,
'Bait coverage': bait_coverage,
'Minimum exact match': minimum_exact_match
}
configs['Cluster probes'] = str(cluster_probes)
if cluster_probes is True:
configs['Cluster identity'] = cluster_identity
configs['Cluster coverage'] = cluster_coverage
configs['Sequences to exclude'] = '{0}'.format(exclude_stats)
if exclude_stats != 'None':
configs['Exclusion identity'] = exclude_pident
configs['Exclusion coverage'] = exclude_coverage
# get baits positions
baits_pos = gu.get_baits_pos(unique_baits, short_samples)
# create a report for each bait identity and coverage pair
for i, v in enumerate(report_identities):
configs['Report bait identity'] = v
configs['Report bait coverage'] = report_coverages[i]
# determine breadth of coverage for all assemblies
# and depth of coverage for each base
final_coverage_dir = os.path.join(output_directory,
'final_coverage_{0}'.format(i))
exists = gu.create_directory(final_coverage_dir)
final_info = {}
discarded_baits_files = []
#missing_files = []
processed = 0
for g in genomes:
generated = incremental_bait_generator(g,
unique_baits,
final_coverage_dir,
bait_size,
report_coverages[i],
v,
bait_offset,
minimum_region,
nr_contigs,
short_samples,
0,
generate=False,
depth=True)
final_info[short_samples[g]] = generated[0]
discarded_baits_files.append(generated[2])
#missing_files.append(generated[3])
processed += 1
print('\r',
'Evaluated coverage for {0}/{1} '
'inputs with bait_identity={2} and '
'bait_coverage={3}.'.format(processed, len(genomes), v,
report_coverages[i]),
end='')
# save depth values
depth_files_dir = os.path.join(
output_directory, 'depth_files_idnt{0}_cov{1}'.format(
str(v).replace('.', ''),
str(report_coverages[i]).replace('.', '')))
exists = gu.create_directory(depth_files_dir)
depth_files = [
mu.write_depth(k, v[3], depth_files_dir)
for k, v in final_info.items()
]
test_fig = create_report(coverage_info, final_info, report_dir,
short_samples, ordered_contigs,
fixed_xaxis, fixed_yaxis, ref_set,
nr_contigs, configs, baits_pos,
nr_baits + total, nr_baits, total,
missing_files)
print('\nCoverage report for bait_identity={0} and '
'bait_coverage={1} available in {2}'.format(
v, report_coverages[i], report_dir))
args = parser.parse_args()
logger = get_logger("heuristic_covariance_w_tpreg_flow")
if args.today is None:
args.today = datetime.date.today().strftime("%Y-%m-%d")
logger.info(f"Using experiment date: {args.today}")
# Experiment Name
exp_name = f"cov_tpreg_ninit_{args.n_init}_ntrain_{args.n_train}_{args.today}"
logger.info(f"Experiment name: {exp_name}")
# create directories for plots, models & evaluation
plots_dir = f"{args.base_plots_dir}/{exp_name}"
logger.info(f"For plotting using dir: {plots_dir}")
plots_dir = create_directory(plots_dir, delete_if_exists=True)
save_dir = f"{args.base_models_dir}/{exp_name}"
logger.info(f"For model saving using dir: {save_dir}")
save_dir = create_directory(save_dir, delete_if_exists=True)
export_dir = f"{args.base_evaluation_dir}/{exp_name}"
logger.info(f"For exporting final evaluation using dir: {export_dir}")
export_dir = create_directory(export_dir, delete_if_exists=True)
# Connect to server & start experiment
ccexp = crayon_create_experiment(exp_name, CrayonClient())
# seed
logger.info(f"Using seed: {args.numpy_seed}")
np.random.seed(args.numpy_seed)
def main(_):
random.seed(0)
np.random.seed(123456789)
tf.set_random_seed(123456789)
global config, actions, checkpoint_dir, output_dir, train_set, test_set, x_test, y_test, dec_in_test
config = training_config.train_Config(FLAGS.dataset, FLAGS.datatype, FLAGS.action)
if FLAGS.longterm == True:
config.filename = 'walking'
config.output_window_size = 100
# Define checkpoint & output directory
checkpoint_dir, output_dir = create_directory(config)
# Train model
if FLAGS.training:
train_set, test_set, x_test, y_test, dec_in_test, config = read_data(config, True)
actions = list(x_test.keys())
train()
# Predict on test set with trained model
try: x_test
except NameError: x_test = None
if config.test_output_window > config.output_window_size or x_test is None:
train_set, test_set, x_test, y_test, dec_in_test, config = read_data(config, False)
actions = list(x_test.keys())
if FLAGS.longterm is True:
x_test = {}
y_test = {}
dec_in_test = {}
test_set = test_set[list(test_set.keys())[0]]
x_test[FLAGS.action] = np.reshape(test_set[:config.input_window_size-1,:], [1, -1, config.input_size])
y_test[FLAGS.action] = np.reshape(test_set[config.input_window_size:, :], [1, -1, config.input_size])
dec_in_test[FLAGS.action] = np.reshape(test_set[config.input_window_size-1:-1, :], [1, -1, config.input_size])
config.test_output_window = y_test[FLAGS.action].shape[1]
config.batch_size = 1
actions = [FLAGS.action]
test_actions = [FLAGS.action]
else:
test_actions = actions
y_predict = predict()
if not (os.path.exists(output_dir)):
os.makedirs(output_dir)
print("Outputs saved to: " + output_dir)
for action in test_actions:
if config.datatype == 'lie':
y_predict[action] = restore(y_predict[action])
y_test[action] = restore(y_test[action])
mean_error, _ = data_utils.mean_euler_error(config, action, y_predict[action], y_test[action])
sio.savemat(output_dir + 'error_' + action + '.mat', dict([('error', mean_error)]))
for i in range(y_predict[action].shape[0]):
y_p = y_predict[action][i]
y_t = y_test[action][i]
sio.savemat(output_dir + 'prediction_lie_' + action + '_' + str(i) + '.mat', dict([('prediction', y_p)]))
sio.savemat(output_dir + 'gt_lie_' + action + '_' + str(i) + '.mat', dict([('gt', y_t)]))
# Forward Kinematics to obtain 3D xyz locations
y_p[:,0:6] = y_t[:,0:6]
y_p_xyz = data_utils.fk(y_p, config)
y_t_xyz = data_utils.fk(y_t, config)
sio.savemat(output_dir + 'prediction_xyz_' + action + '_' + str(i) + '.mat', dict([('prediction', y_p_xyz)]))
sio.savemat(output_dir + 'gt_xyz_' + action + '_' + str(i) + '.mat', dict([('gt', y_t_xyz)]))
filename = action + '_' + str(i)
if FLAGS.visualize:
# Visualize prediction
predict_plot = plot_animation(y_p_xyz, y_t_xyz, config, filename)
predict_plot.plot()
else:
for i in range(y_predict[action].shape[0]):
y_p = y_predict[action][i]
y_t = y_test[action][i]
y_p_xyz = np.reshape(y_p, [y_p.shape[0], -1, 3])
y_t_xyz = np.reshape(y_t, [y_t.shape[0], -1, 3])
sio.savemat(output_dir + 'prediction_xyz_' + action + '_' + str(i) + '.mat', dict([('prediction', y_p)]))
sio.savemat(output_dir + 'gt_xyz_' + action + '_' + str(i) + '.mat', dict([('gt', y_t)]))
# Inverse Kinematics to obtain lie parameters
y_p_lie = data_utils.inverse_kinematics(y_p, config)
y_t_lie = data_utils.inverse_kinematics(y_t, config)
sio.savemat(output_dir + 'prediction_lie_' + action + str(i) + '.mat', dict([('prediction', y_p_lie)]))
sio.savemat(output_dir + 'gt_lie_' + action + str(i) + '.mat', dict([('gt', y_t_lie)]))
filename = action + '_' + str(i)
if FLAGS.visualize:
# Visualize prediction
predict_plot = plot_animation(y_p_xyz, y_t_xyz, config, filename)
predict_plot.plot()
def main(_):
tf.set_random_seed(112858)
global config, actions, checkpoint_dir, output_dir, train_set, test_set, x_test, y_test, dec_in_test
config = training_config.train_Config(FLAGS.dataset, FLAGS.datatype,
FLAGS.action)
if FLAGS.longterm == True:
config.output_window_size = 100
if FLAGS.action not in ['walking', 'eating', 'smoking']:
raise Exception(
"Invalid action! For long-term prediction, action can only be 'walking', 'smoking' or 'eating'."
)
# Define checkpoint & output directory
checkpoint_dir, output_dir = create_directory(config)
# Train model
if FLAGS.training:
train_set, test_set, x_test, y_test, dec_in_test, config = read_data(
config, True)
actions = list(x_test.keys())
train()
# Predict on test set with trained model
try:
x_test
except NameError:
x_test = None
if config.test_output_window > config.output_window_size or x_test is None:
train_set, test_set, x_test, y_test, dec_in_test, config = read_data(
config, False)
actions = list(x_test.keys())
if FLAGS.longterm is True:
x_test = {}
y_test = {}
dec_in_test = {}
test_set = test_set[list(test_set.keys())[0]]
x_test[FLAGS.action] = np.reshape(
test_set[:config.input_window_size - 1, :],
[1, -1, config.input_size])
y_test[FLAGS.action] = np.reshape(
test_set[config.input_window_size:, :], [1, -1, config.input_size])
dec_in_test[FLAGS.action] = np.reshape(
test_set[config.input_window_size - 1:-1, :],
[1, -1, config.input_size])
config.test_output_window = y_test[FLAGS.action].shape[1]
config.batch_size = 1
actions = [FLAGS.action]
test_actions = [FLAGS.action]
else:
test_actions = actions
y_predict = predict()
if not (os.path.exists(output_dir)):
os.makedirs(output_dir)
print("Outputs saved to: " + output_dir)
for action in test_actions:
if config.datatype == 'lie':
mean_error, _ = data_utils.mean_euler_error(
config, action, y_predict[action], y_test[action])
sio.savemat(output_dir + 'error_' + action + '.mat',
dict([('error', mean_error)]))
for i in range(y_predict[action].shape[0]):
if config.dataset == 'Human':
y_p = data_utils.unNormalizeData(y_predict[action][i],
config.data_mean,
config.data_std,
config.dim_to_ignore)
y_t = data_utils.unNormalizeData(y_test[action][i],
config.data_mean,
config.data_std,
config.dim_to_ignore)
expmap_all = data_utils.revert_coordinate_space(
np.vstack((y_t, y_p)), np.eye(3), np.zeros(3))
y_p = expmap_all[config.test_output_window:]
y_t = expmap_all[:config.test_output_window]
else:
y_p = y_predict[action][i]
y_t = y_test[action][i]
sio.savemat(
output_dir + 'prediction_lie_' + action + '_' + str(i) +
'.mat', dict([('prediction', y_p)]))
sio.savemat(
output_dir + 'gt_lie_' + action + '_' + str(i) + '.mat',
dict([('gt', y_t)]))
# Forward Kinematics to obtain 3D xyz locations
# y_p[:,0:6] = y_t[:,0:6]
y_p_xyz = data_utils.fk(y_p, config)
y_t_xyz = data_utils.fk(y_t, config)
sio.savemat(
output_dir + 'prediction_xyz_' + action + '_' + str(i) +
'.mat', dict([('prediction', y_p_xyz)]))
sio.savemat(
output_dir + 'gt_xyz_' + action + '_' + str(i) + '.mat',
dict([('gt', y_t_xyz)]))
filename = action + '_' + str(i)
if FLAGS.visualize:
# Visualize prediction
predict_plot = plot_animation(y_p_xyz, y_t_xyz, config,
filename)
predict_plot.plot()
else:
for i in range(y_predict[action].shape[0]):
y_p = y_predict[action][i]
y_t = y_test[action][i]
y_p_xyz = np.reshape(y_p, [y_p.shape[0], -1, 3])
y_t_xyz = np.reshape(y_t, [y_t.shape[0], -1, 3])
sio.savemat(
output_dir + 'prediction_xyz_' + action + '_' + str(i) +
'.mat', dict([('prediction', y_p)]))
sio.savemat(
output_dir + 'gt_xyz_' + action + '_' + str(i) + '.mat',
dict([('gt', y_t)]))
# Inverse Kinematics to obtain lie parameters
y_p_lie = data_utils.inverse_kinematics(y_p, config)
y_t_lie = data_utils.inverse_kinematics(y_t, config)
sio.savemat(
output_dir + 'prediction_lie_' + action + str(i) + '.mat',
dict([('prediction', y_p_lie)]))
sio.savemat(output_dir + 'gt_lie_' + action + str(i) + '.mat',
dict([('gt', y_t_lie)]))
filename = action + '_' + str(i)
if FLAGS.visualize:
# Visualize prediction
predict_plot = plot_animation(y_p_xyz, y_t_xyz, config,
filename)
predict_plot.plot()
