def __init__(
self,
args,
dataset_split_name: str,
is_training: bool,
name: str,
):
self.name = name
self.args = args
self.dataset_split_name = dataset_split_name
self.is_training = is_training
self.shuffle = self.args.shuffle
self.log = utils.get_logger(self.name)
self.timer = utils.Timer(self.log)
self.dataset_path = Path(self.args.dataset_path)
self.dataset_path_with_split_name = self.dataset_path / self.dataset_split_name
with utils.format_text("yellow", ["underline"]) as fmt:
self.log.info(self.name)
self.log.info(
fmt(f"dataset_path_with_split_name: {self.dataset_path_with_split_name}"
))
self.log.info(
fmt(f"dataset_split_name: {self.dataset_split_name}"))
def __init__(
self,
args,
dataset_split_name: str,
is_training: bool,
name: str,
):
self.name = name
self.args = args
self.dataset_split_name = dataset_split_name
self.is_training = is_training
# args.inference is False by default
self.need_label = not self.args.inference
self.shuffle = self.args.shuffle
self.supported_extensions = [".jpg", ".JPEG", ".png"]
self.log = utils.get_logger(self.name, None)
self.timer = utils.Timer(self.log)
self.dataset_path = Path(self.args.dataset_path)
self.dataset_path_with_split_name = self.dataset_path / self.dataset_split_name
with utils.format_text("yellow", ["underline"]) as fmt:
self.log.info(self.name)
self.log.info(
fmt(f"dataset_path_with_split_name: {self.dataset_path_with_split_name}"
))
self.log.info(
fmt(f"dataset_split_name: {self.dataset_split_name}"))
def test_csv_write_single_process(self):
df = create_dataframe()
total_ms = 0
write_count = 10
for _ in xrange(write_count):
with utils.Timer('csv_write', logger=logger) as tm:
save_dataframe(df)
total_ms += tm.msecs
logger.debug('csv write total time(ms): %d avg(ms): %f', total_ms,
float(total_ms) / write_count)
def test_csv_read(self):
df = create_dataframe()
file_name = save_dataframe(df)
total_ms = 0
read_count = 10
for _ in xrange(read_count):
with utils.Timer('csv_read', logger=logger) as tm:
read_dataframe(file_name)
total_ms += tm.msecs
logger.debug('csv read total time(ms): %d avg(ms): %f', total_ms,
float(total_ms) / read_count)
def test_create_threads(self):
total_ms = 0
thread_count = 50
iter_count = 10
for _ in xrange(iter_count):
with utils.Timer('create_threads', logger=logger) as tm:
workers = create_threads(thread_count)
for worker in workers:
worker.start()
for worker in workers:
worker.join()
total_ms += tm.msecs
logger.debug('%s threads create/run total time(ms): %d avg(ms): %f',
thread_count, total_ms,
float(total_ms) / iter_count)
def test_do_nothing_process_pool(self):
total_ms = 0
process_count = 50
iter_count = 10
proc_ids = [proc_id for proc_id in xrange(process_count)]
for _ in xrange(iter_count):
with utils.Timer('create_pool', logger=logger) as tm:
pool = create_pool(process_count)
pool.map(do_work, proc_ids, chunksize=1)
pool.close()
pool.join()
total_ms += tm.msecs
msg = 'create & run %s processes total time(ms): %d avg(ms): %f'
logger.debug(msg,
process_count,
total_ms,
float(total_ms)/iter_count)
def __init__(self, model, session, args, dataset, dataset_name, name):
self.model = model
self.session = session
self.args = args
self.dataset = dataset
self.dataset_name = dataset_name
self.log = utils.get_logger(name)
self.timer = utils.Timer(self.log)
self.info_red = utils.format_log(self.log.info, "red")
self.info_cyan = utils.format_log(self.log.info, "cyan")
self.info_magenta = utils.format_log(self.log.info, "magenta")
self.info_magenta_reverse = utils.format_log(self.log.info,
"magenta",
attrs=["reverse"])
self.info_cyan_underline = utils.format_log(self.log.info,
"cyan",
attrs=["underline"])
self.debug_red = utils.format_log(self.log.debug, "red")
self._saver = None
# used in `log_step_message` method
self.last_loss = dict()