def init(filename=None):
rootLogger = logging.getLogger()
if filename is None:
FileUtils.createDir('./logs')
filename = os.path.abspath(
'./logs/' + datetime.now().strftime('%y-%m-%d_auto') + '.log')
if len(rootLogger.handlers) > 0:
if os.path.exists(filename):
return
for each in rootLogger.handlers:
rootLogger.removeHandler(each)
logFormatter = logging.Formatter(
"%(asctime)s [%(threadName)-12.12s] [%(levelname)-5.5s] %(message)s"
)
rootLogger.level = logging.INFO #level
fileHandler = logging.FileHandler(filename)
fileHandler.setFormatter(logFormatter)
rootLogger.addHandler(fileHandler)
consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
parser.add_argument('-debug_batch_count', default=0, type=int) # 0 = release version
parser.add_argument('-embedding_size', default=32, type=int)
parser.add_argument('-gamma', default=0.0, type=float)
parser.add_argument('-C_0', default=0.0, type=float)
parser.add_argument('-C_n', default=5.0, type=float)
parser.add_argument('-C_interval', default=10000, type=int)
parser.add_argument('-C_start', default=0, type=int)
args, args_other = parser.parse_known_args()
path_sequence = f'./results/{args.sequence_name}'
args.run_name += ('-' + datetime.utcnow().strftime(f'%y-%m-%d--%H-%M-%S'))
path_run = f'./results/{args.sequence_name}/{args.run_name}'
FileUtils.createDir(path_run)
path_artifacts = f'./artifacts/{args.sequence_name}/{args.run_name}'
FileUtils.createDir(path_artifacts)
FileUtils.writeJSON(f'{path_run}/args.json', args.__dict__)
CsvUtils2.create_global(path_sequence)
CsvUtils2.create_local(path_sequence, args.run_name)
summary_writer = tensorboard_utils.CustomSummaryWriter(
logdir=path_run
)
rootLogger = logging.getLogger()
logFormatter = logging.Formatter("%(asctime)s [%(process)d] [%(thread)d] [%(levelname)s] %(message)s")
rootLogger.level = logging.INFO #level
default=False,
type=lambda x: (str(x).lower() == 'true'))
args, args_other = parser.parse_known_args()
args = ArgsUtils.add_other_args(args, args_other)
args_other_names = ArgsUtils.extract_other_args_names(args_other)
if args.is_restricted_cpu:
from cgroups import Cgroup # pip install cgroups
# sudo /home/ubuntu/anaconda3/bin/user_cgroups ubuntu
# sudo /home/evalds/.conda/envs/conda_env/bin/user_cgroups evalds
# add all testable parameters to final report header
args.params_report += args_other_names
FileUtils.createDir('./reports')
FileUtils.createDir('./tasks')
FileUtils.createDir('./tasks/' + args.report)
logging_utils = LoggingUtils(filename=os.path.join('reports', args.report +
'.txt'))
ArgsUtils.log_args(args, 'taskgen.py', logging_utils)
task_settings = {'id': 0, 'repeat_id': 0}
tasks_settings_path = os.path.join('tasks', 'tasks.json')
if os.path.exists(tasks_settings_path):
with open(tasks_settings_path, 'r') as outfile:
tasks_settings_loaded = json.load(outfile)
for key in tasks_settings_loaded:
task_settings[key] = tasks_settings_loaded[key]
'score_best',
'loss',
'loss_dqn',
'loss_inverse',
'loss_forward',
'cosine_distance'
]
if not args.params_report is None:
for it in reversed(args.params_report):
if not it in tmp:
tmp.insert(0, it)
args.params_report = tmp
args.params_report_local = args.params_report
FileUtils.createDir('./tasks/' + args.report)
run_path = './tasks/' + args.report + '/runs/' + args.name
if os.path.exists(run_path):
shutil.rmtree(run_path, ignore_errors=True)
time.sleep(3)
while os.path.exists(run_path):
pass
FileUtils.createDir(run_path)
logging_utils = LoggingUtils(filename=os.path.join(run_path, 'log.txt'))
is_logged_cnorm = False
ArgsUtils.log_args(args, 'main.py', logging_utils)
CsvUtils.create_local(args)
parser.add_argument('-hpc_mem',
help='HPC - override mem GB',
default=0,
type=int)
parser.add_argument('-is_hpc',
help='is HPC qsub tasks or local tasks',
default=True,
type=lambda x: (str(x).lower() == 'true'))
parser.add_argument('-hpc_queue', help='hpc queue', default='batch', type=str)
args, args_other = parser.parse_known_args()
FileUtils.createDir('./reports')
FileUtils.createDir('./tasks')
#FileUtils.createDir('./tasks/' + args.report)
if args.is_hpc:
FileUtils.createDir(os.path.expanduser('~') + '/tmp')
logging_utils = LoggingUtils(name=os.path.join('reports', args.report +
'.txt'))
task_settings = {'id': 0, 'repeat_id': 0}
hpc_settings_path = os.path.join('tasks', 'tasks.json')
if os.path.exists(hpc_settings_path):
with open(hpc_settings_path, 'r') as outfile:
hpc_settings_loaded = json.load(outfile)
# /simpsons/test.mmap
# /simpsons/test.json
parser.add_argument('-path_output',
default='/Users/evalds/Downloads/simpsons_x/',
type=str)
# scale and squeeze images to this size
parser.add_argument('-size_img', default=128, type=int)
parser.add_argument('-thread_max', default=10, type=int)
parser.add_argument('-test_split', default=0.2, type=float)
args, args_other = parser.parse_known_args()
FileUtils.createDir(args.path_output)
logging_utils = LoggingUtils(
f"{args.path_output}/simpsons-{datetime.now().strftime('%y-%m-%d_%H-%M-%S')}.log"
)
class_names = []
last_class_name = None
mmap_shape = [0, 3, args.size_img, args.size_img]
logging_utils.info(
f'move test samples into train to change from classification to re-identification task'
)
paths_files = FileUtils.listSubFiles(args.path_input_test)
for path_file in paths_files:
args_other_names = ArgsUtils.extract_other_args_names(args_other)
if len(args.datasource_include_test_class_ids) > 0:
args.datasource_include_test_class_ids = ' '.join(
args.datasource_include_test_class_ids)
if len(args.datasource_exclude_train_class_ids) > 0:
args.datasource_exclude_train_class_ids = ' '.join(
args.datasource_exclude_train_class_ids)
if args.hpc_queue == 'inf':
args.hpc_gpu_max_queue = 0 # for old ones disable GPU
# add all testable parameters to final report header
args.params_report += args_other_names
FileUtils.createDir('./reports')
FileUtils.createDir('./tasks')
FileUtils.createDir('./tasks/' + args.report)
if args.is_hpc:
FileUtils.createDir(os.path.expanduser('~') + '/tmp')
logging_utils = LoggingUtils(filename=os.path.join('reports', args.report +
'.txt'))
ArgsUtils.log_args(args, 'taskgen.py', logging_utils)
task_settings = {'id': 0, 'repeat_id': 0}
hpc_settings_path = os.path.join('tasks', 'tasks.json')
if os.path.exists(hpc_settings_path):
with open(hpc_settings_path, 'r') as outfile:
hpc_settings_loaded = json.load(outfile)
for key in hpc_settings_loaded:
def __init__(self, args, is_test_data):
super().__init__()
self.args = args
self.is_test_data = is_test_data
path_data = f'{self.args.path_data}/{self.args.datasource_type}'
FileUtils.createDir(path_data)
if not os.path.exists(
f'{self.args.path_data}/{self.args.datasource_type}/lock'):
with open(
f'{self.args.path_data}/{self.args.datasource_type}/lock',
'w') as fp_download_lock:
fp_download_lock.write('')
time.sleep(1.0)
with open(f'{self.args.path_data}/{self.args.datasource_type}/lock',
'r+') as fp_download_lock:
FileUtils.lock_file(fp_download_lock)
transform_colors = torchvision.transforms.ToTensor()
if self.args.datasource_is_grayscale:
transform_colors = torchvision.transforms.Compose([
torchvision.transforms.Grayscale(),
torchvision.transforms.ToTensor()
])
if self.args.datasource_type == 'fassion_mnist':
self.dataset = torchvision.datasets.FashionMNIST(
path_data,
download=True,
train=not is_test_data,
transform=torchvision.transforms.ToTensor())
elif self.args.datasource_type == 'mnist':
self.dataset = torchvision.datasets.MNIST(
path_data,
download=True,
train=not is_test_data,
transform=torchvision.transforms.ToTensor())
elif self.args.datasource_type == 'cifar_10':
self.dataset = torchvision.datasets.CIFAR10(
path_data,
download=True,
train=not is_test_data,
transform=transform_colors)
elif self.args.datasource_type == 'cifar_100':
self.dataset = torchvision.datasets.CIFAR100(
path_data,
download=True,
train=not is_test_data,
transform=transform_colors)
elif self.args.datasource_type == 'emnist': # extended mnist https://arxiv.org/pdf/1702.05373.pdf
self.dataset = torchvision.datasets.EMNIST(
path_data,
download=True,
split='balanced',
train=not is_test_data,
transform=torchvision.transforms.Compose([
lambda img: torchvision.transforms.functional.rotate(
img, -90), lambda img: torchvision.transforms.
functional.hflip(img),
torchvision.transforms.ToTensor()
]))
FileUtils.unlock_file(fp_download_lock)
self.classes = np.arange(np.array(self.dataset.targets).max() +
1).tolist()
groups = [{'samples': [], 'counter': 0} for _ in self.classes]
for img, label_idx in self.dataset:
groups[int(label_idx)]['samples'].append(img)
args.input_size = img.size(1) # channels, w, h
args.input_features = img.size(0)
if not is_test_data:
ids = [
int(it) for it in self.args.datasource_exclude_train_class_ids
]
ids = sorted(ids, reverse=True)
for remove_id in ids:
del self.classes[remove_id]
del groups[remove_id]
else:
if len(self.args.datasource_include_test_class_ids):
ids = set(self.classes) - set([
int(it)
for it in self.args.datasource_include_test_class_ids
])
ids = list(ids)
ids = sorted(ids, reverse=True)
for remove_id in ids:
del self.classes[remove_id]
del groups[remove_id]
self.classes = np.array(self.classes, dtype=np.int)
self.size_samples = 0
for idx, group in enumerate(groups):
samples = group['samples']
self.size_samples += len(samples)
self.groups = groups
# for debugging purposes
# DEBUGGING
if self.args.datasource_size_samples > 0:
logging.info(
f'debugging: reduced data size {self.args.datasource_size_samples}'
)
self.size_samples = self.args.datasource_size_samples
logging.info(
f'{self.args.datasource_type} {"test" if is_test_data else "train"}: classes: {len(groups)} total triplets: {self.size_samples}'
)
if not is_test_data:
self.args.datasource_classes_train = len(
groups) # override class count
if self.args.batch_size % self.args.triplet_positives != 0 or self.args.batch_size <= self.args.triplet_positives:
logging.error(
f'batch does not accommodate triplet_positives {self.args.batch_size} {self.args.triplet_positives}'
)
exit()
self.reshuffle()