def make_dirs(self,
audio_path,
frames_path,
remake_audio_dir=False,
remake_frames_dir=False):
utils.make_dirs(audio_path, remake_audio_dir)
utils.make_dirs(frames_path, remake_frames_dir)
def __init__(self, config, model, experiments):
self.config = config
self.model = model
self.experiments = experiments
self.trainloader, _ = data_loader(config)
# checkpoint
self.checkpoint_dir = make_dirs(
os.path.join(self.config.result_path, self.config.checkpoint_path))
self.ckpt = tf.train.Checkpoint(enc=self.model.enc,
dec=self.model.dec,
optim=self.model.optim,
epoch=self.model.global_epoch)
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt,
directory=self.checkpoint_dir,
checkpoint_name='ckpt',
max_to_keep=2)
# tensorboard
self.tensorboard_dir = make_dirs(
os.path.join(self.config.result_path,
self.config.tensorboard_path))
self.summary_writer = tf.summary.create_file_writer(
self.tensorboard_dir)
def generate_files_from_template(self) -> None:
src_dir_path = self.template.root_dir
file_pathes = utils.get_files(src_dir_path,
excepts='/templates/manual')
for src_file_path in file_pathes:
src_file = Path(src_file_path)
if src_file.is_file():
relative_file_path = str(src_file.relative_to(src_dir_path))
dest_file_path = path.join(self.config.output_pj_path,
relative_file_path)
dest_file_dir_path = path.dirname(dest_file_path)
# if the file's dir not exist, make it
utils.make_dirs([dest_file_dir_path])
if 'tpl' in path.basename(src_file_path) and path.basename(
src_file_path)[0] != '.':
relative_src_file_path = str(
src_file.relative_to(self.template.root_dir))
self.template.generate(relative_src_file_path,
dest_file_dir_path)
else:
shutil.copy2(src_file_path, dest_file_path)
def start_browser_fuzz(browser, fuzz_type, limit=-1):
"""Start Fuzzing
Limit: -1 (Infinite fuzz)
"""
print "[INFO] Running Browser Fuzzer"
print "\n[DETAILS]"
print "Browser: " + browser
print "Fuzzer Type: " + fuzz_type
print "Fuzz Iteration: " + str(limit)
utils.make_dirs()
utils.adb_connection_int(settings.BROWSERS)
iteration = 1
fuzz_server_url = "http://" + settings.FUZZ_IP + ":" + str(
settings.SERVER_PORT)
if fuzz_type == "domato":
while True:
url = fuzz_server_url + "/fuzz_html/" + str(iteration)
browser_fuzz(browser, fuzz_type, iteration, url)
if iteration == limit:
break
iteration += 1
elif fuzz_type == "pregenerated":
for html in get_htmls():
html_file_name = os.path.basename(html)
url = fuzz_server_url + "/html/" + html_file_name
browser_fuzz(browser, fuzz_type, iteration, url)
if iteration == limit:
break
iteration += 1
print "[INFO] Browser Fuzzing Completed!"
print "\n[Status]"
print "Browser: " + browser
print "Fuzzer Type: " + fuzz_type
print "Fuzz Iteration: " + str(limit)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input_path", type=str, required=True)
parser.add_argument("-o", "--output_path", type=str, required=True)
args = parser.parse_args()
dirs = ["trainval/", "test/"]
dirs = [args.output_path + d for d in dirs]
utils.make_dirs(dirs)
conf = configuration.load_configuration()
images = utils.collect_images(os.path.dirname(args.input_path))
# reproducible datasets
random.seed(1337)
random.shuffle(images)
rois = roi_metadata.read_metadata(args.input_path)
features_train = set(
os.path.basename(i)
for i in images[0:int(len(images) * conf.split_ratio)])
generate_dataset(rois, os.path.dirname(args.input_path), features_train,
dirs[0])
features_test = set(
os.path.basename(i)
for i in images[int(len(images) * conf.split_ratio):])
generate_dataset(rois, os.path.dirname(args.input_path), features_test,
dirs[1])
def prepare(self):
"""remove temporary files, create the directory structure"""
utils.rmtree(self.work_path)
utils.make_dirs(os.path.join(self.work_iso, 'seedbank/etc/runonce.d'))
utils.make_dirs(self.work_initrd)
utils.run('bsdtar -C "%s" -xf "%s"' % (self.work_iso, self.iso_file))
utils.run('chmod -R u+w "%s"' % self.work_iso)
def before_request():
""" Ensure input/output directories exist """
config = app.config['soccer']
make_dirs(
config['input_dir'],
config['output_dir']
)
def _touch(self, directory, path):
if os.path.exists(path):
os.utime(path, None)
else:
utils.make_dirs(directory)
f = open(path, 'w')
f.close()
def copy_dir_contents(self, src, dst):
"""find and copy all files from src to dst"""
utils.make_dirs(dst)
files = [os.path.join(root, file_name) for root, _, files in
os.walk(src) if files for file_name in files]
for src in files:
utils.file_copy(src, dst)
def __init__(self, config, model, experiments):
self.config = config
self.model = model
self.experiments = experiments
self.trainloader, _ = data_loader(config)
# checkpoint
self.checkpoint_dir = make_dirs(
os.path.join(self.config.result_path, self.config.checkpoint_path))
self.ckpt = tf.train.Checkpoint(gen=self.model.gen,
g_optim=self.model.g_optim,
dis=self.model.dis,
d_optim=self.model.d_optim,
epoch=self.model.global_epoch)
self.ckpt_manager = tf.train.CheckpointManager(
self.ckpt,
directory=self.checkpoint_dir,
checkpoint_name='ckpt',
max_to_keep=1)
# tensorboard
self.tensorboard_dir = make_dirs(
os.path.join(self.config.result_path,
self.config.tensorboard_path))
self.summary_writer = tf.summary.create_file_writer(
self.tensorboard_dir)
tf.summary.trace_on(graph=True, profiler=True)
def transform_multiplex_networks(layer_info_file, edge_list_file, outdir, drop_weight=False):
make_dirs(outdir)
networks = pd.read_csv(layer_info_file, sep='\s+')
edge_df = pd.read_csv(edge_list_file,
names=['layerID', 'source_node', 'target_node', 'weight'], sep='\s+')
columns_to_output = ['source_node', 'target_node', 'weight']
if drop_weight:
columns_to_output = columns_to_output[:-1]
graph_files = []
for i, (net_id, net_name) in networks.iterrows():
out_fname = os.path.join(outdir, net_name + ".edgelist")
edge_df.loc[edge_df['layerID'] == net_id, columns_to_output].to_csv(
out_fname,
header=False,
index=False,
sep=' '
)
graph_files.append(out_fname)
return graph_files
def read_flickr_lastfm_data(test_ratio, k_nearest, input_dir="data/flickr_vs_lastfm/"):
tg_data_dirname = os.path.join(input_dir, 'tg_datasets', 'tr_' + str(test_ratio))
if os.path.isdir(tg_data_dirname):
return select_top_k_anchors(load_tg_dataset(tg_data_dirname), k_nearest)
graph_files = glob.glob(os.path.join(input_dir, 'edgelist_data', "*.edgelist"))
graph_data = []
attributes = []
for gf in graph_files:
g = nx.read_edgelist(gf)
g.graph['name'] = os.path.basename(gf).replace('.edgelist', '')
g.graph['centrality_file'] = os.path.join(os.path.dirname(gf), g.graph['name'] + '.centrality')
if os.path.isfile(g.graph['centrality_file']):
g.graph['centrality'] = unpickle_data(g.graph['centrality_file'])
else:
g.graph['centrality'] = None
attributes.append(np.load(gf.replace(".edgelist", ".attr.npy")))
graph_data.append(g)
tg_data = from_nx_to_tg_graphs(graph_data, attributes=attributes, test_ratio=test_ratio)
make_dirs(tg_data_dirname)
save_tg_dataset(tg_data, tg_data_dirname)
return select_top_k_anchors(tg_data, k_nearest)
def read_synthetic_network_dataset(input_dir, test_ratio, k_nearest):
tg_data_dirname = os.path.join(input_dir, 'tg_datasets', 'tr_' + str(test_ratio))
if os.path.isdir(tg_data_dirname):
return select_top_k_anchors(load_tg_dataset(tg_data_dirname), k_nearest)
graph_files = glob.glob(os.path.join(input_dir, 'edgelist_data', "*.edgelist"))
graph_data = []
for gf in sorted(graph_files):
g = nx.read_edgelist(gf)
g.graph['name'] = os.path.basename(gf).replace('.edgelist', '')
g.graph['centrality_file'] = os.path.join(os.path.dirname(gf), g.graph['name'] + '.centrality')
if os.path.isfile(g.graph['centrality_file']):
g.graph['centrality'] = unpickle_data(g.graph['centrality_file'])
else:
g.graph['centrality'] = None
graph_data.append(g)
tg_data = from_nx_to_tg_graphs(graph_data, test_ratio=test_ratio)
make_dirs(tg_data_dirname)
save_tg_dataset(tg_data, tg_data_dirname)
return select_top_k_anchors(tg_data, k_nearest)
def generate_vocabs(base_folder, all_vocab_folder, data):
all_diffs_vocab_filename = os.path.join(all_vocab_folder,
DIFFS_VOCAB_FILENAME)
all_msgs_vocab_filename = os.path.join(all_vocab_folder,
MSGS_VOCAB_FILENAME)
for folder, diffs_filename, msgs_finame, diffs_at_least, msgs_at_least in data:
diffs_word_freq = word_freq(
os.path.join(base_folder, folder, diffs_filename),
os.path.join(base_folder, folder, DIFFS_VOCAB_FILENAME),
)
new_diffs_vocab = generate_new_vocab(diffs_word_freq,
all_diffs_vocab_filename,
diffs_at_least)
msgs_word_freq = word_freq(
os.path.join(base_folder, folder, msgs_finame),
os.path.join(base_folder, folder, MSGS_VOCAB_FILENAME),
)
new_msgs_vocab = generate_new_vocab(msgs_word_freq,
all_msgs_vocab_filename,
msgs_at_least)
output_folder = os.path.join(base_folder, "vocabs", folder)
if not os.path.isdir(output_folder):
make_dirs(output_folder)
save_file(os.path.join(output_folder, DIFFS_VOCAB_FILENAME),
new_diffs_vocab)
save_file(os.path.join(output_folder, MSGS_VOCAB_FILENAME),
new_msgs_vocab)
def __init__(self, args):
super().__init__()
self.lr = args.learning_rate
self.z_dim = 100
self.eval_interval = 100
self.eval_size = 16
self.data_dir = args.data_dir
self.device = "cpu" if args.no_cuda else "cuda"
# evaluate generator based pn fixed noise during training
self.fixed_z = to_tensor(
np.random.normal(0, 1, size=(self.eval_size, self.z_dim)),
False).to(self.device)
self.label_smooth = args.label_smooth
self.G_loss = []
self.D_loss = []
self.path = args.path
self.batch_size = args.batch_size
self.checkpoint_path = os.path.join(self.path, "checkpoint")
self.images_path = os.path.join(self.path, "images")
self.train_images_path = os.path.join(self.images_path, "train_images")
self.val_images_path = os.path.join(self.images_path, "val_images")
make_dirs(self.checkpoint_path, self.train_images_path,
self.val_images_path)
def generate_faces():
# Test Path #
make_dirs(config.inference_path)
# Prepare Generator #
G = Generator().to(device)
G.load_state_dict(
torch.load(
os.path.join(
config.weights_path,
'Face_Generator_Epoch_{}.pkl'.format(config.num_epochs))))
G.eval()
# Start Generating Faces #
count = 1
while (True):
# Prepare Fixed Noise and Generator #
noise = torch.randn(config.batch_size, config.noise_dim).to(device)
generated = G(noise)
for i in range(config.batch_size):
save_image(
denorm(generated[i].data),
os.path.join(config.inference_path,
"Generated_CelebA_Faces_{}.png".format(count)),
)
count += 1
if count > config.limit:
print("Generating fake CelebA faces is finished.")
break
def transform_cosnet_files_to_standard_format(g1_files, g2_files, mapping_file, output_dir):
make_dirs(output_dir)
map_df = pd.read_csv(mapping_file, sep="\s+", names=['g1', 'g2'])
g1_nodes = pd.read_csv(g1_files['nodes'], sep='\t', names=['id', 'username'])
g2_nodes = pd.read_csv(g2_files['nodes'], sep='\t', names=['id', 'username'])
g1_nx = nx.read_edgelist(g1_files['edges'], nodetype=int)
g2_nx = nx.read_edgelist(g2_files['edges'], nodetype=int)
g1 = generate_subgraph(g1_nx, list(g1_nodes.loc[g1_nodes['username'].isin(map_df['g1']), 'id'].values), g1_nodes)
g2 = generate_subgraph(g2_nx, list(g2_nodes.loc[g2_nodes['username'].isin(map_df['g2']), 'id'].values), g2_nodes)
print(g1.number_of_nodes())
print(g2.number_of_nodes())
map_df = map_df.loc[map_df['g1'].isin(list(g1.nodes())) & map_df['g2'].isin(list(g2.nodes()))]
if 'attributes' in g1_files:
attr_mat = create_attribute_files(g1_files['attributes'], list(g1.nodes()))
np.save(os.path.join(output_dir, g1_files['name'] + ".attr"), attr_mat)
if 'attributes' in g2_files:
attr_mat = create_attribute_files(g2_files['attributes'], list(g2.nodes()))
np.save(os.path.join(output_dir, g2_files['name'] + ".attr"), attr_mat)
map_dict = dict(zip(map_df['g1'], map_df['g2']))
nx.relabel_nodes(g1, map_dict, copy=False)
nx.write_edgelist(g1, os.path.join(output_dir, g1_files['name'] + ".edgelist"))
nx.write_edgelist(g2, os.path.join(output_dir, g2_files['name'] + ".edgelist"))
def _move(self, dst):
"""search and move all files from a given directory"""
utils.make_dirs(dst)
files = (os.path.join(root, file_name) for root, _, files in
os.walk(self.temp) if files for file_name in files)
for src in files:
utils.file_copy(src, dst)
def inference():
# Inference Path #
make_dirs(config.inference_path)
# Prepare Data Loader #
test_loader_selfie, test_loader_anime = get_selfie2anime_loader('test', config.batch_size)
# Prepare Generator #
G_A2B = Generator(image_size=config.crop_size, num_blocks=config.num_blocks).to(device)
G_A2B.load_state_dict(torch.load(os.path.join(config.weights_path, 'U-GAT-IT_G_A2B_Epoch_{}.pkl'.format(config.num_epochs))))
# Inference #
print("U-GAT-IT | Generating Selfie2Anime images started...")
with torch.no_grad():
for i, (selfie, anime) in enumerate(zip(test_loader_selfie, test_loader_anime)):
# Prepare Data #
real_A = selfie.to(device)
# Generate Fake Images #
fake_B = G_A2B(real_A)[0]
# Save Images (Selfie -> Anime) #
result = torch.cat((real_A, fake_B), dim=0)
save_image(denorm(result.data),
os.path.join(config.inference_path, 'U-GAT-IT_Selfie2Anime_Results_%03d.png' % (i + 1))
)
# Make a GIF file #
make_gifs_test("U-GAT-IT", "Selfie2Anime", config.inference_path)
def prepare(self):
"""remove temporary files, create the directory structure"""
utils.rmtree(self.work_path)
utils.make_dirs(os.path.join(self.work_iso, 'seedbank/etc/runonce.d'))
utils.make_dirs(self.work_initrd)
utils.run('bsdtar -C "%s" -xf "%s"' % (self.work_iso, self.iso_file))
utils.run('chmod -R u+w "%s"' % self.work_iso)
def __init__(self, graph: Graph, output_path: str,
archs: Dict[str, str]) -> None:
self.archs = archs
self.graph = graph
self.root_path = path.join(output_path, 'tvm_runtime', 'lib')
for arch in archs:
utils.make_dirs(path.join(self.root_path, arch))
def write(self, contents):
"""write the pxe boot file"""
file_name = os.path.join(cfg['paths']['tftpboot'], 'pxelinux.cfg',
self.address)
directory = os.path.dirname(file_name)
utils.make_dirs(directory)
utils.file_delete('%s.disabled' % file_name)
utils.file_write(file_name, contents)
def write(self, contents):
"""write the pxe boot file"""
file_name = os.path.join(cfg['paths']['tftpboot'], 'pxelinux.cfg',
self.address)
directory = os.path.dirname(file_name)
utils.make_dirs(directory)
utils.file_delete('%s.disabled' % file_name)
utils.file_write(file_name, contents)
def __init__(self, log, exit_callback=None):
self._log = log
self._server_proxy = None
self._data_file = None
self._data_file_path = None
self._data_file_state = 1
self._data_file_hdr_row = 'Year,Month,Day,Hour,Minute,Second,Modem_on,FG_on,SC_on,CASES_on,HF_On,Htr_On,Garmin_GPS_on,Overcurrent_status_on,T_batt_1,T_batt_2,T_batt_3,T_FG_electronics,T_FG_sensor,T_router,V_batt_1,V_batt_2,V_batt_3,I_input,P_input,lat,long,sys_time_error_secs,UTC_sync_age_secs,Uptime_secs,CPU_load_1_min,CPU_load_5_min,CPU_load_15_min\n'
utils.make_dirs(super_config.hskp_temp_dir, self._log)
def run(self, task_context):
for root in [
task_context['root-prefix-path'],
task_context['root-build-path']
]:
for platform in task_context['platforms']:
path = join(root, platform)
print('Creating directory %s' % path)
make_dirs(path)
def copy_dir_contents(self, src, dst):
"""find and copy all files from src to dst"""
utils.make_dirs(dst)
files = [
os.path.join(root, file_name) for root, _, files in os.walk(src)
if files for file_name in files
]
for src in files:
utils.file_copy(src, dst)
def _download(self, src, dst_path):
"""download a file"""
src_file = os.path.basename(src)
dst = os.path.join(dst_path, src_file)
if os.path.isfile(dst):
logging.info('"%s" already exists, download skipped', dst)
else:
utils.make_dirs(dst_path)
utils.download(src, dst)
def run(self, task_context):
for dir in [
task_context['root-src-path'], task_context['root-build-path'],
task_context['root-prefix-path'],
task_context['root-output-path'],
task_context['root-android-output-path'],
task_context['root-ios-output-path']
]:
make_dirs(dir)
def _download(self, src, dst_path):
"""download a file"""
src_file = os.path.basename(src)
dst = os.path.join(dst_path, src_file)
if os.path.isfile(dst):
logging.info('"%s" already exists, download skipped', dst)
else:
utils.make_dirs(dst_path)
utils.download(src, dst)
def main():
remove_dir("java_template")
make_dirs("java_template")
generate_template("original/java/train.4186.diff",
"java_template/train.4186.diff.new", "train")
generate_template("original/java/test.436.diff",
"java_template/test.436.diff.new", "test")
generate_template("original/java/valid.453.diff",
"java_template/valid.453.diff.new", "valid")
def _pxe_default(self):
"""manage the pxelinux.cfg default file"""
src = os.path.join(self.cfg['paths']['templates'], 'pxe_default')
directory = os.path.join(self.cfg['paths']['tftpboot'], 'pxelinux.cfg')
dst = os.path.join(directory, 'default')
if os.path.isfile(dst):
return
logging.info('created default pxelinux.cfg file "%s"', dst)
utils.make_dirs(directory)
utils.file_copy(src, dst)
def _pxe_default(self):
"""manage the pxelinux.cfg default file"""
src = os.path.join(self.cfg['paths']['templates'], 'pxe_default')
directory = os.path.join(self.cfg['paths']['tftpboot'], 'pxelinux.cfg')
dst = os.path.join(directory, 'default')
if os.path.isfile(dst):
return
logging.info('created default pxelinux.cfg file "%s"', dst)
utils.make_dirs(directory)
utils.file_copy(src, dst)
def _extract(self, prefix, files, src, dst, target):
"""extract files to the seedbank temp directory and move those"""
archive = os.path.join(dst, os.path.basename(src))
files = (os.path.join(prefix, file_name) for file_name in files)
temp_manage = os.path.join(self.temp, 'manage')
if os.path.isdir(temp_manage):
utils.rmtree(temp_manage)
utils.make_dirs(temp_manage)
utils.untar_files(archive, files, temp_manage)
self.copy_dir_contents(temp_manage, target)
utils.rmtree(temp_manage)
def _extract(self, prefix, files, src, dst, target):
"""extract files to the seedbank temp directory and move those"""
archive = os.path.join(dst, os.path.basename(src))
files = (os.path.join(prefix, file_name) for file_name in files)
temp_manage = os.path.join(self.temp, 'manage')
if os.path.isdir(temp_manage):
utils.rmtree(temp_manage)
utils.make_dirs(temp_manage)
utils.untar_files(archive, files, temp_manage)
self.copy_dir_contents(temp_manage, target)
utils.rmtree(temp_manage)
def _debian_firmware(self, name):
"""integrate Debian non free firmware"""
temp_initrd = os.path.join(self.temp, 'initrd')
initrd = os.path.join(self.cfg['paths']['tftpboot'], 'seedbank', name,
'initrd.gz')
utils.make_dirs(temp_initrd)
utils.initrd_extract(temp_initrd, initrd)
dst = os.path.join(self.temp, 'initrd/lib/firmware')
self._add_firmware(name, dst)
utils.initrd_create(temp_initrd, initrd)
utils.rmtree(temp_initrd)
def inference():
# Inference Path #
make_dirs(config.inference_path)
# Prepare Data Loader #
val_loader = get_edges2shoes_loader(purpose='val',
batch_size=config.val_batch_size)
# Prepare Generator #
G_A2B = Generator().to(device)
G_B2A = Generator().to(device)
G_A2B.load_state_dict(
torch.load(
os.path.join(
config.weights_path,
'DiscoGAN_Generator_A2B_Epoch_{}.pkl'.format(
config.num_epochs))))
G_B2A.load_state_dict(
torch.load(
os.path.join(
config.weights_path,
'DiscoGAN_Generator_B2A_Epoch_{}.pkl'.format(
config.num_epochs))))
# Test #
print("DiscoGAN | Generating Edges2Shoes images started...")
for i, (real_A, real_B) in enumerate(val_loader):
# Prepare Data #
real_A = real_A.to(device)
real_B = real_B.to(device)
# Generate Fake Images #
fake_B = G_A2B(real_A)
fake_A = G_B2A(real_B)
# Generated Reconstructed Images #
fake_ABA = G_B2A(fake_B)
fake_BAB = G_A2B(fake_A)
# Save Images #
result = torch.cat(
(real_A, fake_A, fake_BAB, real_B, fake_B, fake_ABA), dim=0)
save_image(denorm(result.data),
os.path.join(
config.inference_path,
'DiscoGAN_Edges2Shoes_Results_%03d.png' % (i + 1)),
nrow=8,
normalize=True)
# Make a GIF file #
make_gifs_test("DiscoGAN", config.inference_path)
def inference():
# Inference Path #
make_dirs(config.inference_path)
# Prepare Data Loader #
val_loader = get_edges2handbags_loader('val', config.val_batch_size)
# Prepare Generator #
G = Generator(z_dim=config.z_dim).to(device)
G.load_state_dict(
torch.load(
os.path.join(
config.weights_path,
'BicycleGAN_Generator_Epoch_{}.pkl'.format(
config.num_epochs))))
G.eval()
# Fixed Noise #
fixed_noise = torch.randn(config.test_size, config.num_images,
config.z_dim).to(device)
# Test #
print("BiCycleGAN | Generating Edges2Handbags Images started...")
for iters, (sketch, ground_truth) in enumerate(val_loader):
# Prepare Data #
N = sketch.size(0)
sketch = sketch.to(device)
results = torch.FloatTensor(N * (1 + config.num_images), 3,
config.crop_size, config.crop_size)
# Generate Fake Images #
for i in range(N):
results[i * (1 + config.num_images)] = sketch[i].data
for j in range(config.num_images):
image = sketch[i].unsqueeze(dim=0)
noise_to_generator = fixed_noise[i, j, :].unsqueeze(dim=0)
out = G(image, noise_to_generator)
results[i * (1 + config.num_images) + j + 1] = out
# Save Images #
save_image(
denorm(results.data),
os.path.join(
config.inference_path,
'BicycleGAN_Edges2Handbags_Results_%03d.png' %
(iters + 1)),
nrow=(1 + config.num_images),
)
make_gifs_test("BicycleGAN", config.inference_path)
def setup(self):
utils.make_dirs()
random.seed(args.manual_seed)
torch.manual_seed(args.manual_seed)
torch.cuda.manual_seed_all(args.manual_seed)
# import a custom data loader
Datawrapper = getattr(__import__(args.datamodule, fromlist=[None]),
args.datawrapper)
self.train_data = Datawrapper(args.train_path)
# self.epoch_len = self.train_data.epoch_len()
self.test_data = Datawrapper(args.test_path)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("-i", "--input_path", type=str, required=True)
parser.add_argument("-o", "--output_path", type=str, required=True)
args = parser.parse_args()
utils.make_dirs([args.output_path + "/annotations/"])
rois = dataset.load_all_rois(args.input_path, dataset.no_false_positives)
generate_rois(os.path.dirname(args.input_path), rois,
args.output_path + "/annotations/")
def iso(self, name):
"""download ISOs"""
dst = os.path.join(self.cfg['paths']['isos'], name + '.iso')
if os.path.isfile(dst):
logging.info('nothing to do, "%s" already exists', dst)
return
distribution = name.split('-', 1)[0]
if distribution == 'ubuntu':
url = self.iso_ubuntu(name)
elif distribution == 'debian':
url = self.iso_debian(name)
utils.make_dirs(self.cfg['paths']['isos'])
utils.download(url, dst)
def handle_asset(asset, handle_formats, dir, flip, objMesh):
for id, obj in asset.objects.items():
try:
otype = obj.type
except Exception as e:
error("[Error] %s" % (e))
continue
if otype not in handle_formats:
continue
d = obj.read()
save_path = os.path.join(dir, obj.type, d.name)
utils.make_dirs(save_path)
if otype == "Mesh":
try:
mesh_data = None
if not objMesh:
mesh_data = BabylonMesh(d).export()
utils.write_to_file(save_path + ".babylon", mesh_data, mode="w")
mesh_data = OBJMesh(d).export()
utils.write_to_file(save_path + ".obj", mesh_data, mode="w")
except (NotImplementedError, RuntimeError) as e:
error("WARNING: Could not extract %r (%s)" % (d, e))
mesh_data = pickle.dumps(d._obj)
utils.write_to_file(save_path + ".Mesh.pickle", mesh_data, mode="wb")
elif otype == "TextAsset":
if isinstance(d.script, bytes):
utils.write_to_file(save_path + ".bin", d.script, mode="wb")
else:
utils.write_to_file(save_path + ".txt", d.script)
elif otype == "Texture2D":
filename = d.name + ".png"
try:
image = d.image
if image is None:
info("WARNING: %s is an empty image" % (filename))
utils.write_to_file(save_path + ".empty", "")
else:
info("Decoding %r" % (d))
img = image
if flip:
img = ImageOps.flip(image)
img.save(save_path + ".png")
except Exception as e:
error("Failed to extract texture %s (%s)" % (d.name, e))
def pimp(self, seeds, overlay, manifests):
"""pimp the seed file template"""
commands = self.cfg['commands']
values = self.cfg['seed']
if self.target == 'iso':
cmd_overlay = commands['iso_overlay']
cmd_early = commands['iso_early_command']
cmd_late = commands['iso_late_command']
values['late_command'] += commands['iso_mount_command']
elif self.target == 'pxe':
cmd_overlay = commands['pxe_overlay']
cmd_puppet_manifest = commands['pxe_puppet_manifest']
cmd_early = commands['pxe_early_command']
cmd_late = commands['pxe_late_command']
if overlay:
values['late_command'] += cmd_overlay
if self.target == 'pxe' and manifests:
values['late_command'] += commands['pxe_puppet_manifests']
for manifest in manifests:
values['manifest'] = manifest
if self.target == 'pxe':
puppet_command = commands_merge(cmd_puppet_manifest, values)
values['late_command'] += [puppet_command]
elif self.target == 'iso':
src = os.path.join(self.cfg['paths']['templates'],
self.cfg['templates']['puppet_manifest'])
path = os.path.join(self.cfg['paths']['temp'], 'seedbank',
values['fqdn'], 'iso/iso/seedbank/etc/runonce.d')
utils.make_dirs(path)
dst = os.path.join(path, 'puppet_manifest_%s.enabled' %
manifest)
utils.write_template(values, src, dst)
values['early_command'] += cmd_early
values['early_command'] = commands_merge(values['early_command'],
values)
values['late_command'] += cmd_late
values['late_command'] = commands_merge(values['late_command'], values)
seed_file = self._merge_seeds(seeds, values)
logging.debug(seed_file)
logging.info('%(fqdn)s - generated preseed file', values)
return seed_file
def collection_qa_analyzer(col_name, env='active', fields=None):
# parse configuration file
cfg_fpath = os.path.join(basepath, "db_and_fpath.cfg")
parser = ConfigParser()
parser.read(cfg_fpath)
# get detailed info from configuration
mongo_uri = parser.get(env, 'uri')
db_name = parser.get(env, 'db_name')
io_fpath = parser.get(env, 'io_fpath')
# get current time in string
current_time = get_current_time(flag='short')
# create directories if not existing, and return the collection's path
col_fpath = make_dirs(io_fpath, current_time, db_name, col_name)
# get all file paths that are required
basic_stat_fpath = os.path.join(col_fpath, 'basic_stat.json')
domain_fpath = os.path.join(col_fpath, 'domain.json')
existing_checking_fpath = os.path.join(col_fpath, 'existing.json')
field_fpath = os.path.join(col_fpath, 'fields.json')
type_checking_fpath = os.path.join(col_fpath, 'type_checking.json')
nullable_checking_fpath = os.path.join(col_fpath, 'nullable.json')
empty_checking_fpath = os.path.join(col_fpath, 'empty.json')
coverage_fpath = os.path.join(col_fpath, 'coverage.json')
merger_fpath = os.path.join(col_fpath, 'merger.txt')
# MongoEye
mongoEye = MongoEye(mongo_uri, db_name, col_name)
# get detailed STAT
mongoEye.save_existing_checking(existing_checking_fpath)
mongoEye.save_fields(existing_checking_fpath, field_fpath)
mongoEye.save_type_checking(type_checking_fpath, field_fpath)
mongoEye.save_nullable_checking(field_fpath, nullable_checking_fpath)
mongoEye.save_empty_checking(type_checking_fpath, empty_checking_fpath)
mongoEye.save_coverage(existing_checking_fpath,
nullable_checking_fpath,
empty_checking_fpath,
coverage_fpath)
mongoEye.merger(field_fpath, type_checking_fpath,
existing_checking_fpath, nullable_checking_fpath,
empty_checking_fpath, coverage_fpath, merger_fpath)
# get basic STAT
mongoEye.save_domain_counters(domain_fpath)
mongoEye.get_basic_stat(merger_fpath, basic_stat_fpath, domain_fpath)
# test count unique domains
#print mongoEye.count_domains()
#===============================================================================
# 获得某一个字段值的分布(distribution)
#===============================================================================
if fields:
field_dist_collector(mongoEye, fields, col_fpath)
def _debian_firmware(self, target):
"""download and integrate the debian non free firmware"""
distribution, release, _ = target.split('-')
path = 'firmware-' + distribution + '-' + release
dst = os.path.join(self.cfg['paths']['archives'], path)
temp_initrd = os.path.join(self.temp, 'initrd')
temp_firmware = os.path.join(self.temp, 'firmware')
firmware = os.path.join(dst, 'firmware.tar.gz')
initrd = os.path.join(self.cfg['paths']['tftpboot'], 'seedbank', target,
'initrd.gz')
url = self.cfg['urls']['debian_firmware'].replace('${release}', release)
self._download(url, dst)
utils.untar(firmware, temp_firmware)
self._extract_debs(temp_firmware)
utils.make_dirs(temp_initrd)
utils.initrd_extract(temp_initrd, initrd)
src = os.path.join(temp_firmware, 'temp', 'lib/firmware')
dst = os.path.join(self.temp, 'initrd/lib/firmware')
utils.file_move(src, dst)
self._disable_usb(temp_initrd)
utils.initrd_create(temp_initrd, initrd)
max_runtime="2-00:00" # Two days. Each script needs ~10-15 minutes, 30 is recommended for buffer
memory="32000" # 16000 might also work
submission_command="sbatch" # Your cluster submission command, eg sbatch, qsub
submission_args="-p russpold --qos russpold" # Does not need spaces to left and right
# Get the base and present working directory
base = get_base()
here = get_pwd()
data = os.path.abspath("%s/data" %(base))
results = os.path.abspath("%s/results" %(base))
output_folder = "%s/permutations" %results
# Make the output directory
make_dirs(output_folder,reason="for permutation results.")
# Images by Concepts data frame
labels_tsv = "%s/concepts_binary_df.tsv" %results
images = pandas.read_csv(labels_tsv,sep="\t",index_col=0)
image_lookup = "%s/image_nii_lookup.pkl" %results
# We will need these folders to exist for job and output files
log_folders = ["%s/.out" %here,"%s/.job" %here]
make_dirs(log_folders)
# Image metadata with number of subjects included
contrast_file = "%s/filtered_contrast_images.tsv" %results
for image1_holdout in images.index.tolist():
print "Parsing %s" %(image1_holdout)
problem = request.form['level']
if '_' not in problem:
return ''
category, name = problem.split('_', 1)
if category in problems and name in problems[category]['problems']:
return load_code(problem, source)
else:
return ''
if __name__ == '__main__':
print ' - Loading init.sql'
init_sql_file = open(INIT_SQL, 'rb')
init_sql_data = init_sql_file.read()
init_sql_file.close()
make_dirs(OUTPUT_PATH)
with app.app_context():
print ' - Executing init.sql'
get_db().executescript(init_sql_data)
get_db().commit()
print ' - Execution success: init.sql'
load_problem() # load problem
app.jinja_env.globals.update(issolved=issolved)
app.jinja_env.globals.update(load_instruction=load_instruction)
app.jinja_env.globals.update(load_example=load_example)
app.debug = True
app.run(host='0.0.0.0', port=3333)
window_size = 4
num_feats=3
base_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(base_dir, 'data')
ann_dir = os.path.join(base_dir, 'annotation/coloncancer')
plain_dir = os.path.join(base_dir, 'original')
train_dir = os.path.join(data_dir, 'train')
dev_dir = os.path.join(data_dir, 'dev')
test_dir = os.path.join(data_dir, 'test')
make_dirs([train_dir, dev_dir, test_dir])
preprocess_data(os.path.join(ann_dir, "Train"), os.path.join(plain_dir, "train"),
train_dir, window_size, num_feats)
preprocess_data(os.path.join(ann_dir, "Dev"), os.path.join(plain_dir, "dev"),
dev_dir, window_size, num_feats)
ann_dir_2 = os.path.join(base_dir, 'thymedata-1.2.0-coloncancer-test-event-time/coloncancer')
preprocess_test_data_phase2(os.path.join(plain_dir, "test"), os.path.join(ann_dir_2, "Test"), test_dir, window_size, num_feats)
build_vocab(
glob.glob(os.path.join(data_dir, '*/*.toks')),
os.path.join(data_dir, 'vocab-cased.txt'),
import pandas
import shutil
import sys
if sys.version_info < (3, 0):
from exceptions import ValueError
# Get the base and present working directory
base = get_base()
here = get_pwd()
data_directory = os.path.abspath("%s/data" %(base))
results_directory = os.path.abspath("%s/results" %(base))
folders = [data_directory,results_directory]
make_dirs(folders)
# Get all collections
collections = api.get_collections()
# Filter images to those that have a DOI
collections = collections[collections.DOI.isnull()==False]
# Useless, but might as well save it
collections.to_csv("%s/collections_with_dois.tsv" %(results_directory),encoding="utf-8",sep="\t")
# Get image meta data for collections
images = api.get_images(collection_pks=collections.collection_id.tolist())
# load list of included image IDs (curated by Poldracklab) and exclude others
