def test_verify_cshd(caplog):
test_verify_root = os.path.join(TESTS_DIR, "test_verify_files", "tt")
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
caplog.set_level(logging.INFO, logger='Checksum_Helper')
# ------------ 1 wrong crc, 1 missing ----------
hfile_path = os.path.join(test_verify_root, "new_cshd_1+3+missing.cshd")
a = Args(hash_file_name=[hfile_path])
starting_cwd = os.getcwd()
caplog.clear()
# files_total, nr_matches, nr_missing, nr_crc_errors
assert _cl_verify_hfile(a) == (3, 1, 1, 1)
# cwd hasn't changed
assert starting_cwd == os.getcwd()
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new_cshd.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}new_cshd_3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.WARNING,
f'sub2{os.sep}new_cshd_missing.txt: MISSING'),
])
assert caplog.record_tuples[3:] == [
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}new_cshd_1+3+missing.cshd: 1 files with wrong CRCs!'
),
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}new_cshd_1+3+missing.cshd: 1 missing files!'
),
]
def test_build_most_current_cshd(hash_fn_filter, search_depth,
dont_filter_deleted, verified_cshd_name,
setup_dir_to_checksum, monkeypatch):
root_dir = setup_dir_to_checksum
shutil.copy2(
os.path.join(TESTS_DIR, "test_build_most_current_files",
"pre-existing.cshd"), root_dir)
a = Args(path=root_dir,
hash_filename_filter=hash_fn_filter,
discover_hash_files_depth=search_depth,
dont_filter_deleted=dont_filter_deleted,
hash_algorithm="sha512",
out_filename="most_current.cshd")
_cl_build_most_current(a)
verified_cshd_contents = read_file(
os.path.join(TESTS_DIR, "test_build_most_current_files",
verified_cshd_name))
generated_cshd_name = f"{root_dir}{os.sep}most_current.cshd"
generated_cshd_contents = read_file(generated_cshd_name)
print("VERIFIED:", verified_cshd_contents.strip())
print("GEN:", generated_cshd_contents)
assert (verified_cshd_contents == generated_cshd_contents)
def test_copyto(setup_tmpdir_param, monkeypatch, caplog) -> None:
tmpdir = setup_tmpdir_param
root_dir = os.path.join(tmpdir, "tt")
shutil.copytree(os.path.join(TESTS_DIR, "test_copyto_files", "tt"),
os.path.join(root_dir))
with open(os.path.join(TESTS_DIR, "test_copyto_files", "tt", "tt.sha512"),
'r',
encoding='utf-8-sig') as f:
orig = f.read().replace("\\", os.sep)
# set input to automatically answer with y so we write the file when asked
monkeypatch.setattr('builtins.input', lambda x: "y")
a = Args(source_path=os.path.join(root_dir, "tt.sha512"),
dest_path=f".{os.sep}sub2{os.sep}tt_moved.sha512")
hf = _cl_copy(a)
assert os.path.isfile(os.path.join(root_dir, "sub2", "tt_moved.sha512"))
with open(os.path.join(root_dir, "sub2", "tt_moved.sha512"),
'r',
encoding='utf-8-sig') as f:
moved = f.read()
expected = orig.replace("*n", f"*../n").replace(f"*sub2/", "*").replace(
"*sub1", f"*../sub1")
assert expected == moved
a = Args(source_path=os.path.join(root_dir, "sub2", "tt_moved.sha512"),
dest_path=f"..{os.sep}sub1{os.sep}sub2{os.sep}tt_moved2.sha512")
hf = _cl_copy(a)
assert os.path.isfile(
os.path.join(root_dir, "sub1", "sub2", "tt_moved2.sha512"))
with open(os.path.join(root_dir, "sub1", "sub2", "tt_moved2.sha512"),
'r',
encoding='utf-8-sig') as f:
moved = f.read()
expected = orig.replace("*n", f"*../../n").replace(
f"*sub2/", f"*../../sub2/").replace(f"*sub1/sub2/",
"*").replace(f"*sub1/", f"*../")
assert expected == moved
a = Args(source_path=os.path.join(root_dir, "sub1", "sub2",
"tt_moved2.sha512"),
dest_path=f"..{os.sep}.{os.sep}tt_moved3.sha512")
hf = _cl_copy(a)
# not reading in the written file only making sure it was written to the correct loc
assert os.path.isfile(os.path.join(root_dir, "sub1", "tt_moved3.sha512"))
def test_do_incremental_per_dir(whitelist, blacklist, expected_dir,
setup_tmpdir_param):
tmpdir = setup_tmpdir_param
root_dir = os.path.join(tmpdir, "tt")
shutil.copytree(
os.path.join(TESTS_DIR, "test_incremental_files", "per_dir"),
os.path.join(root_dir, ""))
a = Args(path=root_dir,
hash_filename_filter=None,
single_hash=True,
dont_include_unchanged=False,
discover_hash_files_depth=-1,
hash_algorithm="sha512",
per_directory=True,
whitelist=whitelist,
blacklist=blacklist,
skip_unchanged=False,
dont_collect_mtime=False)
_cl_incremental(a)
expected_res = [
("root.sha512", f"tt_{time.strftime('%Y-%m-%d')}.sha512"),
("sub1.sha512",
os.path.join("sub1", f"sub1_{time.strftime('%Y-%m-%d')}.sha512")),
("sub2.sha512",
os.path.join("sub2", f"sub2_{time.strftime('%Y-%m-%d')}.sha512")),
("sub3.sha512",
os.path.join("sub3", f"sub3_{time.strftime('%Y-%m-%d')}.sha512")),
("sub4.sha512",
os.path.join("sub4", f"sub4_{time.strftime('%Y-%m-%d')}.sha512")),
]
for expected_fn, result_fn in expected_res:
try:
verified_sha_contents = read_file(
os.path.join(TESTS_DIR, "test_incremental_files", expected_dir,
expected_fn))
except FileNotFoundError:
# make sure generated file is also missing
assert not os.path.exists(os.path.join(root_dir, result_fn))
continue
generated_sha_contents = read_file(os.path.join(root_dir, result_fn))
compare_lines_sorted(verified_sha_contents, generated_sha_contents)
def test_white_black_list(depth, hash_fn_filter, include_unchanged, whitelist,
blacklist, verified_sha_name, setup_tmpdir_param,
caplog, monkeypatch):
tmpdir = setup_tmpdir_param
root_dir = os.path.join(tmpdir, "wl_bl")
# When using copytree, you need to ensure that src exists and dst does not exist.
# Even if the top level directory contains nothing, copytree won't work because it
# expects nothing to be at dst and will create the top level directory itself.
shutil.copytree(os.path.join(TESTS_DIR, "test_incremental_files", "wl_bl"),
os.path.join(root_dir))
caplog.clear()
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
caplog.set_level(logging.WARNING, logger='Checksum_Helper')
monkeypatch.setattr('builtins.input', lambda x: "y")
a = Args(path=root_dir,
hash_filename_filter=hash_fn_filter,
single_hash=True,
dont_include_unchanged=not include_unchanged,
discover_hash_files_depth=depth,
hash_algorithm="sha512",
per_directory=False,
whitelist=whitelist,
blacklist=blacklist,
skip_unchanged=False,
dont_collect_mtime=False)
_cl_incremental(a)
if whitelist is not None and blacklist is not None:
assert caplog.record_tuples == [
('Checksum_Helper', logging.ERROR,
'Can only use either a whitelist or blacklist - not both!'),
]
else:
verified_sha_contents = read_file(
os.path.join(TESTS_DIR, "test_incremental_files",
verified_sha_name))
# find written sha (current date is appended)
generated_sha_name = f"wl_bl_{time.strftime('%Y-%m-%d')}.sha512"
generated_sha_contents = read_file(
os.path.join(root_dir, generated_sha_name))
compare_lines_sorted(verified_sha_contents, generated_sha_contents)
def test_build_most_current_single(hash_fn_filter, search_depth,
dont_filter_deleted, verified_sha_name,
setup_dir_to_checksum, monkeypatch):
root_dir = setup_dir_to_checksum
a = Args(path=root_dir,
hash_filename_filter=hash_fn_filter,
discover_hash_files_depth=search_depth,
dont_filter_deleted=dont_filter_deleted,
hash_algorithm="sha512",
out_filename="most_current.sha512")
_cl_build_most_current(a)
verified_sha_contents = read_file(
os.path.join(TESTS_DIR, "test_build_most_current_files",
verified_sha_name))
generated_sha_name = f"{root_dir}{os.sep}most_current.sha512"
generated_sha_contents = read_file(generated_sha_name)
assert (verified_sha_contents == generated_sha_contents)
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self).__init__()
self.dropout = nn.Dropout(p=dropout)
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
dimention = torch.arange(0, d_model)
pe[:, 0::2] = torch.sin(
position / torch.pow(10000, 2 *
(dimention // 2)[::2].float() / d_model))
pe[:, 1::2] = torch.cos(
position / torch.pow(10000, 2 *
(dimention // 2)[1::2].float() / d_model))
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
def forward(self, x):
x = x + self.pe[:, :x.size(1), :]
return self.dropout(x)
if __name__ == "__main__":
from utils import Args
args = Args('train_data_path', 'save_path')
args.vocab_size = 200
model = Vae(args)
model.generate(decode='greedy')
import torch import torch.nn as nn import numpy as np from utils import Args, EarlyStopping_unlearning from losses.confusion_loss import confusion_loss from losses.dice_loss import dice_loss from sklearn.utils import shuffle import torch.optim as optim from train_utils_segmentation import train_unlearn, val_unlearn, train_encoder_unlearn, val_encoder_unlearn import sys ######################################################################################################################## # Create an args class args = Args() args.channels_first = True args.epochs = 300 args.batch_size = 3 args.diff_model_flag = False args.alpha = 50 args.patience = 25 args.epoch_stage_1 = 100 args.epoch_reached = 1 args.beta = 10 cuda = torch.cuda.is_available() LOAD_PATH_UNET = None LOAD_PATH_SEGMENTER = None LOAD_PATH_DOMAIN = None
import torch.nn as nn import numpy as np from sklearn.utils import shuffle from utils import Args, EarlyStopping_unlearning from losses.confusion_loss import confusion_loss from losses.dice_loss import dice_loss import torch.optim as optim from train_utils_segmentation import train_encoder_domain_unlearn_semi, val_encoder_domain_unlearn_semi, train_unlearn_semi, val_unlearn_semi import sys ######################################################################################################################## # Create an args class args = Args() args.channels_first = True args.epochs = 300 args.batch_size = 4 args.diff_model_flag = False args.alpha = 50 args.patience = 100 cuda = torch.cuda.is_available() LOAD_PATH_UNET = None LOAD_PATH_SEGMENTER = None LOAD_PATH_DOMAIN = None PRETRAIN_UNET = 'pretrain_unet' PATH_UNET = 'unet_pth'
import sys
import numpy as np
import gflags
import models
from utils import Accumulator, Args, make_batch
# PyTorch
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.nn.functional as F
import torch.optim as optim
FLAGS = gflags.FLAGS
args = Args()
gflags.DEFINE_enum(
"style", "dynamic",
["static", "static2", "dynamic", "dynamic2", "fakedynamic", "fakestatic"],
"Specify dynamic or static RNN loops.")
gflags.DEFINE_boolean("smart_batching", True,
"Bucket batches for similar length.")
# Parse command line flags.
FLAGS(sys.argv)
# Set args.
args.training_data_path = 'trees/dev.txt'
args.eval_data_path = 'trees/dev.txt'
args.embedding_data_path = 'glove.6B.50d.txt'
def test_verify_all(caplog):
test_verify_root = os.path.join(TESTS_DIR, "test_verify_files", "tt")
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
caplog.set_level(logging.INFO, logger='Checksum_Helper')
# ------------ 2 wrong crc, 2 missing, MixedAlgo ----------
root_dir = test_verify_root
a = Args(root_dir=[root_dir],
discover_hash_files_depth=1,
hash_filename_filter=())
caplog.clear()
# files_total, nr_matches, nr_missing, nr_crc_errors
assert _cl_verify_all(a) == (16, 10, 3, 3)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new 2.txt: MD5 OK'),
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.INFO, f'new_cshd.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}new_cshd_3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.WARNING,
f'sub2{os.sep}new_cshd_missing.txt: MISSING'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}new_cshd2.txt: SHA512 OK'),
])
assert caplog.record_tuples[16:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 3 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 3 missing files!"
),
]
# ------------ all matching, 1 missing, most_current single hash file ----------
root_dir = os.path.join(test_verify_root, "sub1", "sub2")
a = Args(root_dir=[root_dir],
discover_hash_files_depth=0,
hash_filename_filter=("*.cshd", ))
caplog.clear()
assert _cl_verify_all(a) == (4, 3, 1, 0)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING, f'new 8.txt: MISSING'),
])
assert caplog.record_tuples[4:] == [
('Checksum_Helper', logging.INFO,
f'{root_dir}{os.sep}sub2_most_current_{time.strftime("%Y-%m-%d")}.sha512: All files matching their hashes!'
),
('Checksum_Helper', logging.WARNING,
f'{root_dir}{os.sep}sub2_most_current_{time.strftime("%Y-%m-%d")}.sha512: 1 missing files!'
),
]
# ------------ 3 wrong crc, 4 missing ----------
root_dir = test_verify_root
a = Args(root_dir=[root_dir],
discover_hash_files_depth=-1,
hash_filename_filter=())
caplog.clear()
assert _cl_verify_all(a) == (20, 13, 4, 3)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(root_dir, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.INFO, f'new 2.txt: MD5 OK'),
('Checksum_Helper', logging.INFO, f'sub3{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub3{os.sep}sub1{os.sep}file1.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 8.txt: MISSING'),
('Checksum_Helper', logging.INFO, f'new_cshd.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}new_cshd_3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.WARNING,
f'sub2{os.sep}new_cshd_missing.txt: MISSING'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}new_cshd2.txt: SHA512 OK'),
])
assert caplog.record_tuples[21:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 3 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 4 missing files!"
),
]
# ------------ 2 wrong crc, 3 missing, single hash, md5+cshd filtered ----------
root_dir = test_verify_root
# hash_filename_filter literally only filters out the hashfile if a str of
# hash_filename_filter is in the name of the file without the extension
a = Args(root_dir=[root_dir],
discover_hash_files_depth=-1,
hash_filename_filter=("*.md5", "*.cshd"))
caplog.clear()
assert _cl_verify_all(a) == (15, 10, 3, 2)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(root_dir, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.INFO, f'sub3{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub3{os.sep}sub1{os.sep}file1.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 8.txt: MISSING'),
])
assert caplog.record_tuples[16:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.sha512: 2 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.sha512: 3 missing files!"
),
]
def test_move_files(src, dst, depth, hash_fn_filter, expected_files_dirname,
moved_paths, extra_cmp, setup_dir_to_checksum, caplog,
monkeypatch):
root_dir = setup_dir_to_checksum
# abspath test windows only
if os.name != 'nt' and os.path.isabs(dst):
return
caplog.set_level(logging.WARNING)
# clear logging records
caplog.clear()
a = Args(root_dir=root_dir,
hash_filename_filter=hash_fn_filter,
discover_hash_files_depth=depth,
source_path=src,
mv_path=dst)
_cl_move(a)
if src == os.path.join("sub4", "file1.txt") and dst == os.path.join(
"sub2", "sub1"):
# filter out hash file pardir warning
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR, "File %s already exists!" %
(os.path.join(root_dir, "sub2", "sub1", "file1.txt"), )),
]
elif src == os.path.join("sub4", "file1.txt") and dst == os.path.join(
"sub2", "sub1", "file1.txt"):
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR, "File %s already exists!" %
(os.path.join(root_dir, "sub2", "sub1", "file1.txt"), )),
]
elif src == "sub1" and dst == "sub2":
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR,
"Couldn't move file(s): Destination path '%s' already exists" %
(os.path.join(root_dir, "sub2", "sub1"), )),
]
elif src == "sub1" and dst == os.path.join("sub1", "sub5"):
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR,
"Couldn't move file(s): Cannot move a directory '%s' into itself '%s'."
% (os.path.join(root_dir,
"sub1"), os.path.join(root_dir, "sub1", "sub5"))),
]
elif src == os.path.join("sub3", "sub1") and dst == "sub4":
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR,
"Couldn't move file(s): Destination path '%s' already exists" %
(os.path.join(root_dir, "sub4", "sub1"))),
]
elif src == os.path.join("sub4", "file1.txt") and dst == os.path.join(
[d for d in "abcdefghijklmnopqrstuvwxyz" if d != TESTS_DIR[0].lower()
][0] + ":", os.sep, "sub2", "sub1"):
assert [
rt for rt in caplog.record_tuples
if not rt[2].startswith("Found reference beyond")
] == [
('Checksum_Helper', logging.ERROR,
"Can't move files to a different drive than the hash files "
"that hold their hashes!"),
]
else:
# only check that src doesnt exist when we shouldn't error
assert not os.path.exists(src)
# check that files are at the expected locations
assert all(
[os.path.exists(os.path.join(root_dir, p)) for p in moved_paths])
# check that file paths were moved inside of hash files
for hf_name in [hf for hf in HASH_FILES if not hf.startswith(src)]:
assert (sort_hf_contents(
read_file(os.path.join(root_dir, hf_name),
encoding="UTF-8-SIG")) == sort_hf_contents(
read_file(os.path.join(test_modf_dir_abs,
expected_files_dirname,
os.path.basename(hf_name)),
encoding="UTF-8-SIG")))
# additional compares when a hash file was moved
for hf_current, hf_expected in extra_cmp:
assert (sort_hf_contents(
read_file(os.path.join(root_dir, hf_current),
encoding="UTF-8-SIG")) == sort_hf_contents(
read_file(os.path.join(test_modf_dir_abs,
expected_files_dirname,
hf_expected),
encoding="UTF-8-SIG")))
def main(env_name):
# 获取所有参数
args = Args(env_name)
env = args.env
max_epochs = args.max_epochs
max_timesteps = args.max_timesteps
update_timestep = args.update_timestep
print_interval = args.print_interval
# 初始化memory
memory = Memory()
# 创建agent实例
agent = Agent(input_size=args.input_size,
output_size=args.output_size,
hidden_size=args.hidden_size,
lr=args.lr,
beta=args.beta,
gamma=args.gamma,
update_epoch=args.update_epoch,
epsilon=args.epsilon)
reward_plot = [0] #记录每print_interval个epoch的平均reward 画图用
timestep_count = 0 #记录步长 到update_timestep清零
interval_reward = 0 #记录每print_interval个epoch的平均reward 后清零
interval_timestep = 0 #记录每print_interval个epoch的平均步长 后清零
file_name = 'RL_Proj_2/{}.txt'.format(args.env_name)
# training loop
for epoch in range(1, max_epochs + 1):
state = env.reset() #与env交互随机获取一个state
# agent做出action
for timestep in range(max_timesteps):
timestep_count += 1
# old policy sampling 做出action 与环境交互
action = agent.old_policy.act(state, memory)
state, reward, done, _ = env.step(action)
memory.rewards.append(reward)
memory.is_done.append(done)
# 判断是否需要更新 policy
if timestep_count % update_timestep == 0:
agent.update(memory)
memory.clear_memory()
timestep_count = 0
interval_reward += reward
env.render()
if done:
break
interval_timestep += timestep
# 每print_interval打印一次数据
if epoch % print_interval == 0:
interval_timestep = np.divide(interval_timestep, print_interval)
interval_reward = np.divide(interval_reward, print_interval)
reward_plot.append(interval_reward)
# 储存数据
with open(file_name, 'a') as f:
f.write(
str(epoch) + ' ' + str(interval_timestep) + ' ' +
str(interval_reward) + '\n')
print('Epoch {} \t average timestep: {} \t reward: {}'.format(
epoch, interval_timestep, interval_reward))
interval_reward = 0
interval_timestep = 0
# 训练结束后 存储模型
torch.save(agent.policy.state_dict(),
'RL_Proj_2/{}.pth'.format(args.env_name))
#画图
plt.plot(reward_plot)
plt.xlabel('Epoch = tick times {}'.format(print_interval))
plt.ylabel('Reward')
plt.savefig('RL_Proj_2/{}.png'.format(args.env_name))
plt.show()
from datasets.numpy_dataset import numpy_dataset_three, numpy_dataset from torch.utils.data import DataLoader import torch import torch.nn as nn import numpy as np from sklearn.utils import shuffle from utils import Args, EarlyStopping_unlearning from losses.confusion_loss import confusion_loss import torch.optim as optim from train_utils import train_unlearn_distinct, val_unlearn_distinct, val_encoder_domain_unlearn_distinct, train_encoder_domain_unlearn_distinct import sys ######################################################################################################################## # Create an args class args = Args() args.channels_first = True args.epochs = 300 args.batch_size = 16 args.diff_model_flag = False args.alpha = 1 args.patience = 150 args.learning_rate = 1e-4 LOAD_PATH_ENCODER = None LOAD_PATH_REGRESSOR = None LOAD_PATH_DOMAIN = None PRE_TRAIN_ENCODER = 'pretrain_encoder' PATH_ENCODER = 'encoder_pth' CHK_PATH_ENCODER = 'encoder_chk_pth'
def test_verify_filter(caplog):
test_verify_root = os.path.join(TESTS_DIR, "test_verify_files", "tt")
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
caplog.set_level(logging.INFO, logger='Checksum_Helper')
# ------------ 3 wrong crc, no missing, MixedAlgo ----------
root_dir = test_verify_root
a = Args(root_dir=root_dir,
discover_hash_files_depth=1,
hash_filename_filter=(),
filter=[
f"sub1{os.sep}*",
"new ?.txt",
])
caplog.clear()
_cl_verify_filter(a)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new 2.txt: MD5 OK'),
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 FAILED'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}new_cshd_3.txt: SHA512 FAILED'),
])
assert caplog.record_tuples[10:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 3 files with wrong CRCs!"
),
('Checksum_Helper', logging.INFO,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: No missing files!"
),
]
# ------------ 1 crc err, 2 missing, MixedAlgo ----------
root_dir = test_verify_root
a = Args(root_dir=root_dir,
discover_hash_files_depth=-1,
hash_filename_filter=(),
filter=[
"*file?.txt",
f"s*{os.sep}sub1{os.sep}**",
])
caplog.clear()
_cl_verify_filter(a)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(root_dir, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.INFO, f'sub3{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub3{os.sep}sub1{os.sep}file1.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
])
assert caplog.record_tuples[7:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 1 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 2 missing files!"
),
]
# ------------ 2 wrong crc, 4 missing, HashFile, md5 filtered ----------
root_dir = test_verify_root
# hash_filename_filter literally only filters out the hashfile if a str of
# hash_filename_filter is in the name of the file without the extension
a = Args(root_dir=root_dir,
discover_hash_files_depth=-1,
hash_filename_filter=("*.md5", ),
filter=[
"",
f"sub?{os.sep}*",
])
caplog.clear()
_cl_verify_filter(a)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(root_dir, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.INFO, f'sub3{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub3{os.sep}sub1{os.sep}file1.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 8.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}new_cshd_3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.WARNING,
f'sub2{os.sep}new_cshd_missing.txt: MISSING'),
('Checksum_Helper', logging.INFO,
f'sub3{os.sep}sub2{os.sep}new_cshd2.txt: SHA512 OK'),
])
assert caplog.record_tuples[17:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 2 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 4 missing files!"
),
]
# ------------ 1 wrong crc, 1 missing, HashFile, md5 filtered ----------
root_dir = test_verify_root
# hash_filename_filter literally only filters out the hashfile if a str of
# hash_filename_filter is in the name of the file without the extension
a = Args(root_dir=root_dir,
discover_hash_files_depth=-1,
hash_filename_filter=("*.md5", ),
filter=[
"*new* ?.txt",
])
caplog.clear()
_cl_verify_filter(a)
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(root_dir, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 8.txt: MISSING'),
])
assert caplog.record_tuples[10:] == [
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 1 files with wrong CRCs!"
),
('Checksum_Helper', logging.WARNING,
f"{root_dir}{os.sep}tt_most_current_{time.strftime('%Y-%m-%d')}.cshd: 1 missing files!"
),
]
def testDocx(self):
mksummery.INPUT_ARGS = Args()
util_docx.write('../out/testdocx.doc')
def test_verify_hfile(caplog):
test_verify_root = os.path.join(TESTS_DIR, "test_verify_files", "tt")
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
caplog.set_level(logging.INFO, logger='Checksum_Helper')
# ------------ 1 wrong crc, no missing ----------
hfile_path = os.path.join(test_verify_root, "sub3", "sub2",
"sub3_sub2.sha512")
a = Args(hash_file_name=[hfile_path])
starting_cwd = os.getcwd()
caplog.clear()
# files_total, nr_matches, nr_missing, nr_crc_errors
assert _cl_verify_hfile(a) == (3, 2, 0, 1)
# cwd hasn't changed
assert starting_cwd == os.getcwd()
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING,
"Found reference beyond the hash file's root dir in file: '%s'. "
"Consider moving/copying the file using ChecksumHelper move/copy "
"to the path that is the most common denominator!" %
os.path.join(test_verify_root, "sub3", "sub2", "sub3_sub2.sha512")),
('Checksum_Helper', logging.INFO, f'..{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'..{os.sep}sub1{os.sep}file1.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'file1.txt: SHA512 OK'),
])
assert caplog.record_tuples[4:] == [
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}sub3{os.sep}sub2{os.sep}sub3_sub2.sha512: 1 files with wrong CRCs!'
),
('Checksum_Helper', logging.INFO,
f'{test_verify_root}{os.sep}sub3{os.sep}sub2{os.sep}sub3_sub2.sha512: No missing files!'
),
]
# ------------ all matching, 1 missing ----------
hfile_path = os.path.join(test_verify_root, "sub1", "sub2",
"sub2_1miss.sha512")
a = Args(hash_file_name=[hfile_path])
starting_cwd = os.getcwd()
caplog.clear()
assert _cl_verify_hfile(a) == (3, 2, 1, 0)
# cwd hasn't changed
assert starting_cwd == os.getcwd()
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING, f'new 8.txt: MISSING'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
])
assert caplog.record_tuples[3:] == [
('Checksum_Helper', logging.INFO,
f'{test_verify_root}{os.sep}sub1{os.sep}sub2{os.sep}sub2_1miss.sha512: All files matching their hashes!'
),
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}sub1{os.sep}sub2{os.sep}sub2_1miss.sha512: 1 missing files!'
),
]
# ----------- no missing all matching ----------
hfile_path = os.path.join(test_verify_root, "sub1", "sub2", "sub2.sha512")
a = Args(hash_file_name=[hfile_path])
starting_cwd = os.getcwd()
caplog.clear()
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
assert _cl_verify_hfile(a) == (3, 3, 0, 0)
# cwd hasn't changed
assert starting_cwd == os.getcwd()
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.INFO, f'new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
])
assert caplog.record_tuples[-1] == (
'Checksum_Helper', logging.INFO,
f'{test_verify_root}{os.sep}sub1{os.sep}sub2{os.sep}sub2.sha512: No missing files and all files matching their hashes'
)
# ----------- 2 missing 2 crc err ----------
hfile_path = os.path.join(test_verify_root, "sub1+2_n3+4.sha512")
a = Args(hash_file_name=[hfile_path])
starting_cwd = os.getcwd()
caplog.clear()
# caplog.set_level sets on root logger by default which is somehow not the logger setup by
# checksum_helper so specify our logger in the kw param
assert _cl_verify_hfile(a) == (11, 7, 2, 2)
# cwd hasn't changed
assert starting_cwd == os.getcwd()
assert x_contains_all_y(caplog.record_tuples, [
('Checksum_Helper', logging.WARNING, f'new 3.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO, f'new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO, f'sub1{os.sep}new 4.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 2.txt: SHA512 OK'),
('Checksum_Helper', logging.INFO,
f'sub1{os.sep}sub2{os.sep}new 3.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub1{os.sep}sub2{os.sep}new 4.txt: SHA512 FAILED'),
('Checksum_Helper', logging.INFO,
f'sub2{os.sep}sub1{os.sep}file1.txt: SHA512 OK'),
('Checksum_Helper', logging.WARNING,
f'sub5{os.sep}sub1{os.sep}file1.txt: MISSING'),
('Checksum_Helper', logging.WARNING,
f'sub6{os.sep}file1.txt: MISSING'),
])
assert caplog.record_tuples[11:] == [
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}sub1+2_n3+4.sha512: 2 files with wrong CRCs!'
),
('Checksum_Helper', logging.WARNING,
f'{test_verify_root}{os.sep}sub1+2_n3+4.sha512: 2 missing files!'),
]
pickle.dump([self.words2ids, self.docs2ids, \
self.ids2docs, self.dft, self.matrix], f)
@property
def N(self):
""" Number of documents. """
return len(self.ids2docs)
if __name__ == '__main__':
import argparse
import os
from utils import Args
parser = argparse.ArgumentParser()
parser.add_argument('--r',
default='custom',
type=str,
help='id of the run')
parser.add_argument('--pc',
type=int,
default=None,
help='__ for development: pc ID')
args_inline = parser.parse_args()
# all arguments for the run
args = Args(args_inline)
# im = IncidenceMatrix.load(args)
im = IncidenceMatrix.create(args)
im.save(args)
from torch.utils.data import DataLoader import torch import torch.nn as nn import numpy as np from sklearn.utils import shuffle from utils import Args, EarlyStopping_unlearning from losses.confusion_loss import confusion_loss from losses.DANN_loss import DANN_loss_three_classes import torch.optim as optim from train_utils import train_unlearn_threedatasets, val_unlearn_threedatasets, train_encoder_unlearn_threedatasets, val_encoder_unlearn_threedatasets import sys ######################################################################################################################## # Create an args class args = Args() args.channels_first = True args.epochs = 300 args.batch_size = 16 args.diff_model_flag = False args.alpha = 1 args.patience = 50 args.learning_rate = 1e-4 args.beta = 10 args.epoch_stage_1 = 100 args.epoch_reached = 1 LOAD_PATH_ENCODER = None LOAD_PATH_REGRESSOR = None LOAD_PATH_DOMAIN = None
if __name__ == '__main__':
import argparse
from logger import *
import subprocess
parser = argparse.ArgumentParser()
parser.add_argument('-d', default='documents_cs.lst', type=str)
parser.add_argument('-q', type=str, default='topics-train_cs.xml', help='Topics list file')
parser.add_argument('-r', type=str, default='run-1_cs', help='Run name')
parser.add_argument('-document_path', type=str, default='documents_cs', help='Run name')
parser.add_argument('-f', type=int)
parser.add_argument('-t', type=int)
args_inline = parser.parse_args()
args = Args(args_inline)
# args._data['o'] = 'results.nosync/res_{}-{}-{}.res'.format(args.lang, args.r, str(9999999))
if args.lang == 'english':
qrels = 'A1/qrels-train_en.txt'
else:
qrels = 'A1/qrels-train_cs.txt'
retrieval = Retrieval(args)
retrieval.search(B=B, K1=K1)
# THIS CODE IS ONLY FOR TRAINING
tags = ['cs', 'lemma2', 'qexp3']
# parameter search
for experiment in range(args.f, args.t):
import subprocess
from logger import *
import regex as re
parser = argparse.ArgumentParser()
parser.add_argument('--pc', type=int, default=None, help='__ for development: pc ID')
parser.add_argument('--start', type=int)
parser.add_argument('--end', type=int)
parser.add_argument('--tag', type=str, default=None, help='tag')
parser.add_argument('-r', type=str, default='run-1_cs', help='Run name')
parser.add_argument('-lang', type=str, default='english', help='Language')
args_inline = parser.parse_args()
args_inline = parser.parse_args()
# all arguments for the run
args = Args(args_inline, False, path=None)
run_im_format = 'x'
im_file = 'nic'
tags = [args.lang] if args.tag is None else [args.lang, args.tag]
for im_version in range(args_inline.start, args_inline.end+1):
run_format = im_version
for arg_file_path in os.listdir(os.path.join('args.nosync', str(im_version))):
path = os.path.join('args.nosync', str(im_version), arg_file_path)
args = Args(args_inline, True, path=path)
logger = NeptuneLogger.new_experiment(tags, args)
# logger.log_hyperparams(args)
logger.log_status('started')
with open(path, 'r') as f:
logger.log_text('args', f.read())
action="store_true",
help="test on a smaller dataset (first 1000 samples)")
line_args = parser.parse_args()
if line_args.stop_at < line_args.n_folds:
print(
"*****************\n\n\tWARNING:\n\n\tthe code is going to run on {} out of {} folds!\n\n*****************"
.format(line_args.stop_at, line_args.n_folds))
small = line_args.small
# All important paths and constants
folds_number = line_args.n_folds
stop_after = line_args.stop_at
args = Args()
args.do_train = line_args.train
args.do_eval = line_args.test
args.do_results = line_args.results
args.use_cuda = not line_args.nocuda
args.small = line_args.small
# aggregation_level = "Chapter"
# aggregation_level = "Block"
# aggregation_level = "Category"
# aggregation_level = "Leaf"
aggregation_level = line_args.level
main_dir = "/mnt/HDD/bportelli/lab_avanzato"
original_data_path = "/mnt/HDD/bportelli/lab_avanzato/beatrice.pkl"
import json
import sys
from vae import Vae
from utils import get_tokenizer, Args, get_dataloader, get_logger, get_callbacks
import pytorch_lightning as pl
import torch
if __name__ == "__main__":
with open(sys.argv[1]) as f:
config = json.load(f)
args = Args(**config)
args.gpus = torch.cuda.device_count()
args.distributed_backend = 'ddp' if args.gpus > 1 else None
tokenizer = get_tokenizer(args.vocab_type)
args.vocab_size = tokenizer.vocab_size
train_dl = get_dataloader(args, tokenizer, type='train')
model = Vae(args)
logger = get_logger(args)
callbacks = get_callbacks(args)
trainer = pl.Trainer(max_steps=args.max_steps,
gpus=args.gpus,
logger=logger,
log_every_n_steps=args.log_steps,
callbacks=callbacks,
distributed_backend=args.distributed_backend)
