def rsync_file(from_path, to_path):
make_dirs(os.path.dirname(to_path))
subprocess_cmd = [
"rsync",
"--progress",
"--chmod=D555",
"--chmod=F444",
"--times",
"--copy-links",
from_path,
to_path,
]
log.info(" ".join(subprocess_cmd))
# The following is a way to use the logging module with subprocess.
# See
# https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging.
process = Popen(subprocess_cmd, stdout=PIPE, stderr=STDOUT)
with process.stdout:
for line in iter(process.stdout.readline, b""):
log.info(line)
if os.path.getsize(to_path) != os.path.getsize(from_path):
log.error("copy failed for %s to %s", from_path, to_path)
def BuildVEL(args):
print("Log CMake version")
cmake_ver_cmd = 'cmake --version'
RunShellCmd(cmake_ver_cmd)
GTEST_DIR = RepoRelative("external/googletest")
if not os.path.exists(GTEST_DIR):
print("Clone Testing Framework Source Code")
clone_gtest_cmd = f'git clone https://github.com/google/googletest.git {GTEST_DIR}'
RunShellCmd(clone_gtest_cmd)
print("Get Specified Testing Source")
gtest_checkout_cmd = 'git checkout tags/release-1.10.0'
RunShellCmd(gtest_checkout_cmd, GTEST_DIR)
utils.make_dirs(VEL_BUILD_DIR)
print("Run CMake for Extension Layer")
cmake_cmd = f'cmake -DUPDATE_DEPS=ON -DCMAKE_BUILD_TYPE={args.configuration.capitalize()} {args.cmake} ..'
if IsWindows(): cmake_cmd = cmake_cmd + f' -A {args.arch}'
RunShellCmd(cmake_cmd, VEL_BUILD_DIR)
print("Build Extension Layer and Tests")
build_cmd = f'cmake --build . --config {args.configuration}'
if not IsWindows(): build_cmd = build_cmd + f' -- -j{os.cpu_count()}'
RunShellCmd(build_cmd, VEL_BUILD_DIR)
def save_pred_label(self, predict_label, iter):
vid_name = None
if iter == 1:
vid_name = '보이그룹-안무하'
elif iter == 2:
vid_name = '걸그룹-안무하'
elif iter == 3:
vid_name = '솔로-안무중'
elif iter == 4:
vid_name = '걸그룹-안무중'
elif iter == 5:
vid_name = '보이그룹-안무중'
og_vid_dir = join(self.original_vid_dir, vid_name)
demo_dir = join(self.demo_dir, vid_name)
label_dir = join(demo_dir, vid_name + '.csv')
make_dirs(demo_dir)
# save csv file
predict_label = np.array(predict_label)
with open(label_dir, "a") as output:
writer = csv.writer(output, lineterminator='\n')
for i in range(predict_label.shape[0]):
writer.writerow(predict_label[i])
# generate demo video
vid_preprocess(og_vid_dir, demo_dir, predict_label[:, 1], self.in_fps,
self.out_fps, self.width, self.height)
def BuildVVL(args):
print("Log CMake version")
cmake_ver_cmd = 'cmake --version'
RunShellCmd(cmake_ver_cmd)
GTEST_DIR = RepoRelative("external/googletest")
if not os.path.exists(GTEST_DIR):
print("Clone Testing Framework Source Code")
clone_gtest_cmd = f'git clone https://github.com/google/googletest.git {GTEST_DIR}'
RunShellCmd(clone_gtest_cmd)
print("Get Specified Testing Source")
gtest_checkout_cmd = 'git checkout tags/release-1.8.1'
RunShellCmd(gtest_checkout_cmd, GTEST_DIR)
utils.make_dirs(VVL_BUILD_DIR)
print("Run CMake for Validation Layers")
cmake_cmd = f'cmake -DUPDATE_DEPS=ON -DCMAKE_BUILD_TYPE={args.configuration.capitalize()} {args.cmake} ..'
if IsWindows(): cmake_cmd = cmake_cmd + f' -A {args.arch}'
RunShellCmd(cmake_cmd, VVL_BUILD_DIR)
print("Build Validation Layers and Tests")
build_cmd = f'cmake --build . --config {args.configuration}'
if not IsWindows(): build_cmd = build_cmd + f' -- -j{os.cpu_count()}'
RunShellCmd(build_cmd, VVL_BUILD_DIR)
print('Run vk_validation_stats.py')
utils.make_dirs(
os.path.join(VVL_BUILD_DIR, 'layers', args.configuration.capitalize()))
RunShellCmd(
f'python3 ../scripts/vk_validation_stats.py ../{EXTERNAL_DIR_NAME}/Vulkan-Headers/registry/validusage.json -text layers/{args.configuration.capitalize()}/vuid_coverage_database.txt',
VVL_BUILD_DIR)
def BuildMockICD(args):
VT_DIR = RepoRelative("%s/Vulkan-Tools" % EXTERNAL_DIR_NAME)
if not os.path.exists(VT_DIR):
print("Clone Vulkan-Tools Repository")
clone_tools_cmd = 'git clone https://github.com/KhronosGroup/Vulkan-Tools.git'
RunShellCmd(clone_tools_cmd, EXTERNAL_DIR)
ICD_BUILD_DIR = RepoRelative("%s/Vulkan-Tools/%s" %
(EXTERNAL_DIR_NAME, BUILD_DIR_NAME))
print("Running update_deps.py for ICD")
RunShellCmd(
f'python3 scripts/update_deps.py --dir {EXTERNAL_DIR_NAME} --config {args.configuration} --arch {args.arch}',
VT_DIR)
print("Run CMake for ICD")
utils.make_dirs(ICD_BUILD_DIR)
cmake_cmd = \
f'cmake -DCMAKE_BUILD_TYPE={args.configuration.capitalize()} -DBUILD_CUBE=NO -DBUILD_VULKANINFO=NO -DINSTALL_ICD=OFF -DVULKAN_HEADERS_INSTALL_DIR={EXTERNAL_DIR}/Vulkan-Headers/{BUILD_DIR_NAME}/install {args.cmake} ..'
RunShellCmd(cmake_cmd, ICD_BUILD_DIR)
print("Build Mock ICD")
build_cmd = f'cmake --build . --config {args.configuration}'
if not IsWindows(): build_cmd = build_cmd + f' -- -j{os.cpu_count()}'
RunShellCmd(build_cmd, ICD_BUILD_DIR)
def BuildMockICD(args):
if not os.path.exists(RepoRelative("%s/Vulkan-Tools" % EXTERNAL_DIR_NAME)):
print("Clone Vulkan-Tools Repository")
clone_tools_cmd = 'git clone https://github.com/KhronosGroup/Vulkan-Tools.git'
RunShellCmd(clone_tools_cmd, EXTERNAL_DIR)
print("Run CMake for ICD")
ICD_BUILD_DIR = RepoRelative("%s/Vulkan-Tools/%s" % (EXTERNAL_DIR_NAME,BUILD_DIR_NAME))
utils.make_dirs(ICD_BUILD_DIR)
cmake_cmd = \
f'cmake -DCMAKE_BUILD_TYPE={args.configuration.capitalize()} -DBUILD_CUBE=NO -DBUILD_VULKANINFO=NO -DINSTALL_ICD=OFF -DVULKAN_HEADERS_INSTALL_DIR={EXTERNAL_DIR}/Vulkan-Headers/{BUILD_DIR_NAME}/install {args.cmake} ..'
RunShellCmd(cmake_cmd, ICD_BUILD_DIR)
VVL_REG_DIR = "%s/Vulkan-Headers/registry" % EXTERNAL_DIR
VT_SCRIPTS_DIR = "%s/Vulkan-Tools/scripts" % EXTERNAL_DIR
print ("Geneating ICD Source Code")
VT_ICD_DIR = "%s/Vulkan-Tools/icd/generated" % EXTERNAL_DIR
LVL_GEN_SCRIPT = RepoRelative("scripts/lvl_genvk.py")
typemap_cmd = 'python3 %s -registry %s/vk.xml vk_typemap_helper.h' % (LVL_GEN_SCRIPT, VVL_REG_DIR)
RunShellCmd(typemap_cmd, VT_ICD_DIR)
KVT_GEN_SCRIPT = "%s/Vulkan-Tools/scripts/kvt_genvk.py" % EXTERNAL_DIR
icd_cpp_cmd = 'python3 %s -registry %s/vk.xml mock_icd.cpp' % (KVT_GEN_SCRIPT, VVL_REG_DIR)
RunShellCmd(icd_cpp_cmd, VT_ICD_DIR)
icd_h_cmd = 'python3 %s -registry %s/vk.xml mock_icd.h' % (KVT_GEN_SCRIPT, VVL_REG_DIR)
RunShellCmd(icd_h_cmd, VT_ICD_DIR)
print("Build Mock ICD")
build_cmd = f'cmake --build . --config {args.configuration}'
if not IsWindows(): build_cmd = build_cmd + f' -- -j{os.cpu_count()}'
RunShellCmd(build_cmd, ICD_BUILD_DIR)
def download_folder(self, bucket_name, folder_prefix, out_dir):
bucket = boto3.resource('s3').Bucket(bucket_name)
for file_object in bucket.objects.filter(Prefix=folder_prefix):
object_name = file_object.key
print(object_name)
file_name = os.path.join(out_dir, object_name)
make_dirs(os.path.dirname(file_name))
self.download_file(file_name, bucket_name, object_name)
def save_summary(self, result: TrainingResult):
result_dir = self.resultPathProvider.provide()
make_dirs(result_dir)
model_id = result.model_id
result.model.save(os.path.join(result_dir, f"{model_id}.pb"))
self.save_history(os.path.join(result_dir, f"{model_id}.json"),
result.metrics)
def __init__(self, args, dataloader_train, dataloader_dev, char_embedding_config, word_embedding_config,
sentence_encoding_config, pair_encoding_config, self_matching_config, pointer_config):
# for validate
expected_version = "1.1"
with open(args.dev_json) as dataset_file:
dataset_json = json.load(dataset_file)
if dataset_json['version'] != expected_version:
print('Evaluation expects v-' + expected_version +
', but got dataset with v-' + dataset_json['version'],
file=sys.stderr)
self.dev_dataset = dataset_json['data']
self.dataloader_train = dataloader_train
self.dataloader_dev = dataloader_dev
self.model = RNet.Model(args, char_embedding_config, word_embedding_config, sentence_encoding_config,
pair_encoding_config, self_matching_config, pointer_config)
self.parameters_trainable = list(
filter(lambda p: p.requires_grad, self.model.parameters()))
self.optimizer = optim.Adadelta(self.parameters_trainable, rho=0.95)
self.best_f1 = 0
self.step = 0
self.start_epoch = args.start_epoch
self.name = args.name
self.start_time = datetime.datetime.now().strftime('%b-%d_%H-%M')
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
self.start_epoch = checkpoint['epoch']
self.best_f1 = checkpoint['best_f1']
self.name = checkpoint['name']
self.step = checkpoint['step']
self.model.load_state_dict(checkpoint['state_dict'])
self.optimizer.load_state_dict(checkpoint['optimizer'])
self.start_time = checkpoint['start_time']
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
raise ValueError("=> no checkpoint found at '{}'".format(args.resume))
else:
self.name += "_" + self.start_time
# use which device
if torch.cuda.is_available():
self.model = self.model.cuda(args.device_id)
else:
self.model = self.model.cpu()
self.loss_fn = torch.nn.CrossEntropyLoss()
configure("log/%s" % (self.name), flush_secs=5)
self.checkpoint_path = os.path.join(args.checkpoint_path, self.name)
make_dirs(self.checkpoint_path)
def rsync_file(file_instance, to_storage, tantalus_api):
""" Rsync a single file from one storage to another
"""
# Get the file resource associated with the file instance
file_resource = tantalus_api.get("file_resource",
id=file_instance["file_resource"])
local_filepath = get_new_filepath(to_storage, file_resource)
remote_filepath = file_instance["filepath"]
if file_instance["file_resource"]["is_folder"]:
local_filepath = local_filepath + "/"
remote_filepath = remote_filepath + "/"
if os.path.isfile(local_filepath):
if check_file_same_local(file_instance["file_resource"],
local_filepath):
return
error_message = "target file {filepath} already exists on {storage} with different size".format(
filepath=local_filepath, storage=to_storage["name"])
raise FileAlreadyExists(error_message)
if file_instance["storage"]["server_ip"] == to_storage["server_ip"]:
remote_location = remote_filepath
else:
remote_location = (file_instance["storage"]["username"] + "@" +
file_instance["storage"]["server_ip"] + ":" +
remote_filepath)
make_dirs(os.path.dirname(local_filepath))
subprocess_cmd = [
"rsync",
"--progress",
# '--info=progress2',
"--chmod=D555",
"--chmod=F444",
"--times",
"--copy-links",
remote_location,
local_filepath,
]
if file_instance["file_resource"]["is_folder"]:
subprocess_cmd.insert(1, "-r")
sys.stdout.flush()
sys.stderr.flush()
subprocess.check_call(subprocess_cmd, stdout=sys.stdout, stderr=sys.stderr)
if not check_file_same_local(file_instance["file_resource"],
local_filepath):
error_message = "transfer to {filepath} on {storage} failed".format(
filepath=local_filepath, storage=to_storage["name"])
raise Exception(error_message)
def main(_):
utils.make_dirs(FLAGS.outdir)
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
filename='{}/eval.log'.format(FLAGS.outdir),
filemode='w',
level=logging.INFO)
evaluate(FLAGS.outdir)
def get_chunk_paths_and_hashes(self, filepath):
"""
Compute the hashes of chunks of a file.
:param filepath: path of file
:return: dictionary, with key is path of chunk
and value is string of hash of this chunk.
"""
shutil.rmtree(self._tmp_dir)
make_dirs(self._tmp_dir)
return split_file_and_get_hash(filepath, self._tmp_dir)
def convert_to_tfrecord(x, outdir, fname):
utils.make_dirs(outdir)
fn = os.path.join(outdir, fname + '.tfrecord')
writer = tf.python_io.TFRecordWriter(fn)
print('Writing', fn)
if isinstance(x[0], (list, tuple)):
_convert_pair_to_tfrecord(x, writer)
else:
_convert_nopair_to_tfrecord(x, writer)
def __init__(self):
self.train_frames = args.train_frames
print("===> Building model")
# Model Setting
# prtrained model setting
self.extractor = resnet101(num_classes=400,
shortcut_type='B',
cardinality=32,
sample_size=args.crop_size,
sample_duration=args.train_frames)
# load pretrained model
# eval 모드를 무조건 해야됨!
weight = get_pretrain_weight(pretrain_path, self.extractor)
self.extractor.load_state_dict(weight)
self.extractor.eval()
self.G = BaseNet(16384 * 4 * 4 * 4, 10)
# optimizer
self.optimizerG = optim.Adam(self.G.parameters(),
lr=args.lr,
betas=(0.9, 0.999),
eps=1e-8)
# loss
self.BCE_loss = nn.BCELoss()
self.L1_loss = nn.L1Loss()
self.L2_loss = nn.MSELoss()
# data
self.train_data = train_data_loader
# self.test_data = test_data_loader
# training
self.device = device
self.epochs = args.epochs
self.avg_G_loss_arr = []
self.checkpoint = args.checkpoint
# cuda
if torch.cuda.is_available():
self.extractor.cuda()
self.G.cuda()
# model save
self.save_mname = args.save_model_name
# checkpoint root
make_dirs(log_path)
self.log_dir = log_path + f'/{self.save_mname}'
# self.load_check_dir =
self.save_check_dir = make_dirs(self.log_dir) + '/' + 'checkpoint.pkl'
# CSV logging system
self.CSVlogger = LogCSV(log_dir=self.log_dir +
f"/{self.save_mname}_log.csv",
header=['epoch', 'avg_G_Loss', 'accuracy'])
def main(
storage_name,
temp_output_dir,
flowcell_id,
bcl_dir,
tag_name=None,
update=False,
no_bcl2fastq=False,
threshold=20,
):
storage = tantalus_api.get("storage", name=storage_name)
storage_client = tantalus_api.get_storage_client(storage_name)
make_dirs(temp_output_dir)
datasets = list(
tantalus_api.list(
"sequence_dataset",
sequence_lanes__flowcell_id=flowcell_id,
dataset_type="FQ",
))
if len(datasets) > 0:
logging.warning("found dataset {}".format(','.join(
[str(d["id"]) for d in datasets])))
if not no_bcl2fastq:
run_bcl2fastq(
flowcell_id,
bcl_dir,
temp_output_dir,
)
# Import fastqs
load_brc_fastqs(
flowcell_id,
temp_output_dir,
storage_name,
storage,
tantalus_api,
storage_client,
tag_name=tag_name,
update=update,
threshold=threshold,
)
# add 4 lanes generated by bcl2fastq on colossus in order to be picked up for analysis
add_lanes(flowcell_id)
def upload_new_version(self, new_file_object_paths):
"""
Upload new version of backup if backup folder is modified.
:param: new_file_object_paths: list of paths of new file_objects
:return: string, the path of new metadata
"""
for file_object_path in new_file_object_paths:
object_name = os.path.relpath(file_object_path, self._PREFIX_PATH)
object_name = replace_backslashes_with_forward_slashes(object_name)
self._user.upload_file(file_object_path, self._bucket, object_name)
new_metadata_dir = os.path.join(self._metadata_dir,
"v{}".format(self._version))
make_dirs(new_metadata_dir)
self.upload_new_metadata(new_metadata_dir)
return new_metadata_dir
def train(self):
# network on training mode
self.G.train()
for epoch in range(self.epochs):
epoch_G_loss = 0.0
for iter, batch in enumerate(self.train_data, 1):
# you must setting data to cpu to gpu using .to(self.device)
# tensor shape : [batch_size, vid_num, frame_num, H, W, C]
input, output = batch[0].to(self.device), batch[1].to(
self.device)
# check the input tensor imgs
tensor_imgsave(join(args.root_path, 'check_tensor'), input,
self.train_frames)
# input video가 8개인데 이걸 1개 1개 분리
# pretrained model에 하나하나 집어넣어줌
# permute로 [N, V, F, H, W, C] to [N, V, C , H, W, F]
features = self.extractor(input[:, 0, :, :, :, :].permute(
0, 4, 1, 2, 3))
for i in range(1, args.num_of_vid):
features = torch.cat(
(features,
self.extractor(input[:, 0, :, :, :, :].permute(
0, 4, 1, 2, 3))),
dim=1)
print(features.shape)
self.optimizerG.zero_grad()
fake_out = self.G(input)
G_loss = self.BCE_loss(fake_out, output)
G_loss.backward()
self.optimizerG.step()
epoch_G_loss += G_loss.data
print("===> Epoch[{}]({}/{}): G_Loss: {:.4f} ".format(
epoch + 1, iter, len(self.train_data), G_loss.item()))
avg_G_loss = epoch_G_loss / len(self.train_data)
self.avg_G_loss_arr.append(avg_G_loss.item())
# learning rate decay
if (epoch + 1) % (self.epochs / 2) == 0:
for param_group in self.optimizerG.param_groups:
param_group['lr'] /= 2.0
print('G: Learning rate decay: lr={}'.format(
self.optimizerG.param_groups[0]['lr']))
# test PSNR and check the training state
if (epoch + 1) % self.checkpoint == 0:
# save training img
train_log_dir = make_dirs(self.log_dir + '/training_img')
# tensor_imgsave(train_log_dir, fake_out, target, epoch=epoch)
# testing set5
# make loss graph
# make_graph(np.array(range(epoch + 1)), np.array(self.avg_G_loss_arr), self.model)
# save check point
check_name = self.save_check_dir + '_epoch_' + str(epoch + 1)
torch.save(self.model, check_name)
def imgsave(log_dir, img, frame_num, denorm):
"""
Todo numpy array img save
:param log_dir: log directory for image saving
:param img: save images
:param frame_num: number of image for saving
:param denorm: denormalization method in input images
"""
now = time.gmtime(time.time())
t = '{0}_{1}'.format(now.tm_min, now.tm_sec)
make_dirs(log_dir)
# [V, D, H, W, C]
for i in range(frame_num):
img0 = img[0, i, :, :, :]
img0 = denorm(img0)
img1 = img[1, i, :, :, :]
img1 = denorm(img1)
img2 = img[2, i, :, :, :]
img2 = denorm(img2)
img3 = img[3, i, :, :, :]
img3 = denorm(img3)
img_tmp1 = np.concatenate((img0, img1, img2, img3), axis=0)
img4 = img[4, i, :, :, :]
img4 = denorm(img4)
img5 = img[5, i, :, :, :]
img5 = denorm(img5)
img6 = img[6, i, :, :, :]
img6 = denorm(img6)
img7 = img[7, i, :, :, :]
img7 = denorm(img7)
img_tmp2 = np.concatenate((img4, img5, img6, img7), axis=0)
# imgsave를 위해선 uint8로 casting 해줘야함
img_out = np.concatenate((img_tmp1, img_tmp2), axis=1).astype('uint8')
plt.imsave(log_dir + '/time_' + t + '_frame_' + str(i),
img_out,
format='png')
def main(_):
current_time = datetime.now().strftime("%Y%m%d-%H%M")
outdir = os.path.join(
FLAGS.outdir, FLAGS.arch + '-' + current_time + '-' + str(os.getpid()))
trained_model = os.path.join(outdir, 'trained_model')
summary_dir = os.path.join(outdir, 'summary')
utils.make_dirs(trained_model)
utils.print_arguments(args=FLAGS.__flags,
log_fn=os.path.join(outdir, 'args.log'))
logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s',
filename='{}/train.log'.format(outdir),
filemode='w',
level=logging.INFO)
train(trained_model, summary_dir)
def backup_dir_prompt(self):
try:
prompt = "Enter a directory to save your backup: "
# NOTE: for test
backup_dir = "../test"
#backup_dir = str(input(prompt).strip())
backup_dir = os.path.abspath(backup_dir)
try:
print("your backup_dir: ", backup_dir)
make_dirs(backup_dir)
except:
print("Invalid directory!")
return
return backup_dir
except:
print("Timeout!")
return
def main():
url_info = get_cmake_url()
cmake_exe_path = get_cmake_exe_path(url_info)
if not os.path.exists(cmake_exe_path):
url = urlparse(url_info['url'])
print(f'cmake minimum version does not exist locally; downloading from {url_info["url"]}')
cmake_archive = os.path.basename(url.path)
with utils.URLRequest(url) as res:
with open(cmake_archive, 'wb') as fd: fd.write(res.read())
utils.expand_archive(cmake_archive)
cmake_build_dir = 'build-cmake-test'
utils.make_dirs(cmake_build_dir, clean=True)
currDir = os.getcwd()
cmake_args = get_cmake_args(cmake_exe_path)
subprocess.check_call(cmake_args, cwd=cmake_build_dir)
def __init__(self, stat_cache_dir, backup_folder):
self._stat_cache_dir = stat_cache_dir # str, abs path
self._backup_folder = backup_folder # str, abs path
# latest file backup
self._latest_stat_cache_dir = os.path.join(stat_cache_dir, "latest")
make_dirs(self._latest_stat_cache_dir)
# newest file backup; compared to latest
self._new_stat_cache_dir = os.path.join(stat_cache_dir, "new")
make_dirs(self._new_stat_cache_dir)
# root stat file
self._root_stat = os.path.join(
self._latest_stat_cache_dir,
os.path.basename(self._backup_folder) + ".dir")
self.initialized = True
def save_pred_label(self, predict_label, iter):
vid_name = None
if iter == 1:
vid_name = '오마이걸-불꽃놀이'
og_vid_dir = join(self.original_vid_dir, vid_name)
demo_dir = join(self.demo_dir, vid_name)
label_dir = join(demo_dir, vid_name + '.csv')
make_dirs(demo_dir)
# save csv file
with open(label_dir, "a") as output:
writer = csv.writer(output, lineterminator='\n')
for i in range(predict_label.shape[0]):
writer.writerow(predict_label[i])
# generate demo video
make_stage_mix(og_vid_dir, demo_dir, predict_label[:, 1], self.in_fps,
self.out_fps, self.width, self.height)
def download_from_blob(self, file_instance, to_storage, tantalus_api):
""" Transfer a file from blob to a server.
This should be called on the from server.
"""
cloud_filepath = file_instance["filepath"]
cloud_container, cloud_blobname = cloud_filepath.split("/", 1)
assert cloud_container == file_instance["storage"]["storage_container"]
# Get the file resource associated with the file instance
file_resource = tantalus_api.get("file_resource",
id=file_instance["file_resource"])
local_filepath = get_new_filepath(to_storage, file_resource)
make_dirs(os.path.dirname(local_filepath))
if not self.block_blob_service.exists(cloud_container, cloud_blobname):
error_message = "source file {filepath} does not exist on {storage} for file instance with pk: {pk}".format(
filepath=cloud_filepath,
storage=file_instance["storage"]["name"],
pk=file_instance["id"],
)
raise FileDoesNotExist(error_message)
if os.path.isfile(local_filepath):
if self._check_file_same_blob(file_resource, cloud_container,
cloud_blobname):
return
error_message = "target file {filepath} already exists on {storage}".format(
filepath=local_filepath, storage=to_storage["name"])
raise FileAlreadyExists(error_message)
self.block_blob_service.get_blob_to_path(
cloud_container,
cloud_blobname,
local_filepath,
progress_callback=TransferProgress().print_progress,
max_connections=1,
)
os.chmod(local_filepath, 0o444)
def checkpoint_set(self):
"""
Todo setting checkpoint argument
:param self.save_mname: saved model name in each checkpoint
:param self.log_dir: location of checkpoint saving folder
:param self.save_check_dir: filename of checkpoint
:param self.CSVlogger: CSVlogger setting for check a loss
"""
# model save
self.save_mname = args.save_model_name
# checkpoint root
make_dirs(log_path)
self.log_dir = log_path + f'/{self.save_mname}'
self.save_check_dir = make_dirs(self.log_dir)
self.graph_dir = self.log_dir
# CSV logging system
self.CSVlogger = LogCSV(log_dir=self.log_dir + f"/{self.save_mname}_log.csv",
header=['epoch', 'avg_G_Loss', 'accuracy'])
def test(self, epoch, avg_G_loss, avg_D_loss):
avg_accuracy = 0
# test for set 5
with torch.no_grad():
for i, batch in enumerate(self.test_data, 1):
input, target = batch[0].to(self.device), batch[1].to(
self.device)
predict = self.G(input)
accuracy = calculate_accuracy(predict, target)
avg_accuracy += accuracy
# save generated and target img
img_dir = join(self.log_dir, 'epoch_' + f'{epoch}')
make_dirs(img_dir)
avg_accuracy = accuracy / len(self.test_data)
print("===> Avg. PSNR: {:.4f} dB".format(avg_accuracy))
self.CSVlogger([epoch + 1, avg_G_loss, avg_D_loss, avg_accuracy])
def _retrieve_backup_data_from_file_objects_and_metadata(
self, encrypted_file_objects_dir, metadata_dir, backup_data_dir,
original_metadata_dir):
"""
Decrypt data keys by control key, then decrypt file objects by data
key and join these file objects
:param encrypted_file_objects_dir: string, path of directory containing
encrypted file objects downloaded from S3
:param metadata_dir: string, path of metadata_dir
:param backup_data_dir: string, path of folder containing retrieved backup
data
:param original_metadata_dir: metadata dir in the first call
(not recursive)
:return
"""
for metadata_path in os.listdir(metadata_dir):
metadata_path = os.path.join(metadata_dir, metadata_path)
if os.path.isfile(metadata_path):
backup_file_path_rel = os.path.relpath(metadata_path,
original_metadata_dir)
backup_file_path_rel = backup_file_path_rel[:
backup_file_path_rel
.find(".metadata")]
metadata = Metadata.read(metadata_path)
data_keys = self.decrypt_data_keys(
metadata.encrypted_data_keys)
backup_file_path = os.path.join(backup_data_dir,
backup_file_path_rel)
make_dirs(os.path.dirname(backup_file_path))
with open(backup_file_path, "wb") as f:
for file_id, data_key in \
zip(metadata.file_ids, data_keys):
encrypted_file_object_path = os.path.join(
encrypted_file_objects_dir, str(file_id))
chunk = decryptFile(data_key,
encrypted_file_object_path)
f.write(chunk)
else:
self._retrieve_backup_data_from_file_objects_and_metadata(
encrypted_file_objects_dir, metadata_path, backup_data_dir,
original_metadata_dir)
def BuildVVL(args, build_tests=False):
print("Log CMake version")
cmake_ver_cmd = 'cmake --version'
RunShellCmd(cmake_ver_cmd)
utils.make_dirs(VVL_BUILD_DIR)
print("Run CMake for Validation Layers")
cmake_cmd = f'cmake -DUPDATE_DEPS=ON -DCMAKE_BUILD_TYPE={args.configuration.capitalize()} {args.cmake} ..'
if IsWindows(): cmake_cmd = cmake_cmd + f' -A {args.arch}'
if build_tests: cmake_cmd = cmake_cmd + ' -DBUILD_TESTS=ON'
RunShellCmd(cmake_cmd, VVL_BUILD_DIR)
print("Build Validation Layers and Tests")
build_cmd = f'cmake --build . --config {args.configuration}'
if not IsWindows(): build_cmd = build_cmd + f' -- -j{os.cpu_count()}'
RunShellCmd(build_cmd, VVL_BUILD_DIR)
print('Run vk_validation_stats.py')
utils.make_dirs(os.path.join(VVL_BUILD_DIR, 'layers', args.configuration.capitalize()))
RunShellCmd(f'python3 ../scripts/vk_validation_stats.py ../{EXTERNAL_DIR_NAME}/Vulkan-Headers/registry/validusage.json -text layers/{args.configuration.capitalize()}/vuid_coverage_database.txt', VVL_BUILD_DIR)
def _create_new_metadata_of_modified_file(self, filepath, file_ids,
data_keys):
"""
Create new metadata of a modified file
:param filepath: path of modified file
:param file_ids: list integer, ids of file objects
which are chunks of this file
:param data_keys: list string, keys to encrypt/decrypt file objects
:return
"""
encrypted_data_keys = self.encrypt_data_keys(data_keys)
relative_path_from_backup_root = os.path.relpath(
filepath, self._backup_folder)
metadata = Metadata(relative_path_from_backup_root, file_ids,
encrypted_data_keys, self._version)
path = os.path.join(self._metadata_dir, "v{}".format(self._version),
relative_path_from_backup_root + ".metadata")
make_dirs(os.path.dirname(path))
metadata.save(path)
return path
def _copy_old_metadata_and_get_set_file_ids_if_unmodified(
self, list_unmodified_files):
"""
Copy metadata of old version to current version
:param list_unmodified_files: list of strings, each string is
a path of an unmodified file
:return: set of integers, containing file ids of unmodified files
"""
set_file_ids = set()
for filepath in list_unmodified_files:
relative_path_from_backup_root = os.path.relpath(
filepath, self._backup_folder)
new_version_path = os.path.join(
self._metadata_dir, "v{}".format(self._version),
relative_path_from_backup_root + ".metadata")
old_version_path = os.path.join(
self._metadata_dir, "v{}".format(self._version - 1),
relative_path_from_backup_root + ".metadata")
make_dirs(os.path.dirname(new_version_path))
if os.path.isfile(old_version_path):
# file already exists
shutil.copy(old_version_path, new_version_path)
file_ids = Metadata.read(new_version_path).file_ids
for file_id in file_ids:
set_file_ids.add(file_id)
else:
# deal with duplicated file: create new metadata for them
chunk_paths_and_hashes = \
self._object_db.get_chunk_paths_and_hashes(filepath)
file_ids = []
data_keys = []
for chunk_path, h in chunk_paths_and_hashes.items():
file_id, data_key = self._object_db.query(h)
file_ids.append(file_id)
data_keys.append(data_key)
self._create_new_metadata_of_modified_file(
filepath, file_ids, data_keys)
return set_file_ids
