def setup_loggers(harn):
if harn.train_dpath is None:
harn.warn('harn.train_dpath is None, cannot setup loggers')
return
use_file_logger = True
if use_file_logger and harn.flog is None:
flog_fname = 'fitlog_{}.log'.format(ub.timestamp())
flog_fpath = os.path.join(harn.train_dpath, flog_fname)
flog = logging.getLogger(harn.__class__.__name__)
formatter = logging.Formatter('%(asctime)s : %(message)s')
handler = logging.FileHandler(flog_fpath, mode='w')
handler.setFormatter(formatter)
flog.propagate = False
flog.setLevel(logging.DEBUG)
flog.addHandler(handler)
harn.flog = flog
harn.debug('initialized file logger')
flog_link = os.path.join(harn.train_dpath, 'fit.log')
ub.symlink(flog_fpath, flog_link, overwrite=True)
if tensorboard_logger:
train_base = os.path.dirname(harn.nice_dpath or harn.train_dpath)
harn.log('dont forget to start: tensorboard --logdir ' +
train_base)
harn.log('Initializing tensorboard')
harn.tlog = tensorboard_logger.Logger(harn.train_dpath,
flush_secs=2)
else:
harn.log('Tensorboard is not available')
def on_pick(event, infr=None):
print('ON PICK: %r' % (event, ))
artist = event.artist
plotdat = util.mplutil._get_plotdat_dict(artist)
if plotdat:
if 'node' in plotdat:
all_node_data = util.sort_dict(plotdat['node_data'].copy())
visual_node_data = ub.dict_subset(all_node_data,
infr.visual_node_attrs, None)
node_data = util.delete_dict_keys(all_node_data,
infr.visual_node_attrs)
node = plotdat['node']
node_data['degree'] = infr.graph.degree(node)
node_label = infr.pos_graph.node_label(node)
print('visual_node_data: ' + ub.repr2(visual_node_data, nl=1))
print('node_data: ' + ub.repr2(node_data, nl=1))
util.cprint('node: ' + ub.repr2(plotdat['node']), 'blue')
print('(pcc) node_label = %r' % (node_label, ))
print('artist = %r' % (artist, ))
elif 'edge' in plotdat:
all_edge_data = util.sort_dict(plotdat['edge_data'].copy())
print(infr.repr_edge_data(all_edge_data))
util.cprint('edge: ' + ub.repr2(plotdat['edge']), 'blue')
print('artist = %r' % (artist, ))
else:
print('???: ' + ub.repr2(plotdat))
print(ub.timestamp())
def dump_text(self):
print("Dumping Profile Information")
try:
output_text, summary_text = self.get_text()
except AttributeError:
print('profile is not on')
else:
#profile.dump_stats('out.lprof')
import ubelt as ub
print(summary_text)
ub.writeto('profile_output.txt', output_text + '\n' + summary_text)
ub.writeto('profile_output.%s.txt' % (ub.timestamp()),
output_text + '\n' + summary_text)
def cmake_clean(dpath='.'):
"""
"""
dpath = ub.truepath(dpath)
cmake_cache_fpath = join(dpath, 'CMakeCache.txt')
assert exists(cmake_cache_fpath)
fpath_set = set(glob.glob(join(dpath, '*'))) - {cmake_cache_fpath}
for fpath in list(fpath_set):
if basename(fpath).startswith('_cmake_build_backup_'):
fpath_set.remove(fpath)
backup_dpath = ub.ensuredir(
join(dpath, '_cmake_build_backup_' + ub.timestamp()))
for fpath in ub.ProgIter(fpath_set, 'moving files'):
shutil.move(fpath, backup_dpath)
def dump_text(self):
print("Dumping Profile Information")
try:
output_text, summary_text = self.get_text()
except AttributeError:
print('profile is not on')
else:
#profile.dump_stats('out.lprof')
import ubelt as ub
print(summary_text)
suffix = ub.argval('--profname', default='')
if suffix:
suffix = '_' + suffix
ub.writeto('profile_output{}.txt'.format(suffix), output_text + '\n' + summary_text)
ub.writeto('profile_output{}.{}.txt'.format(suffix, ub.timestamp()),
output_text + '\n' + summary_text)
def save(self, data, cfgstr=None):
"""
Writes data to path specified by `self.fpath(cfgstr)`.
Metadata containing information about the cache will also be appended
to an adjacent file with the `.meta` suffix.
Example:
>>> from ubelt.util_cache import * # NOQA
>>> # Normal functioning
>>> cfgstr = 'long-cfg' * 32
>>> cacher = Cacher('test_enabled_save', cfgstr)
>>> cacher.save('data')
>>> assert exists(cacher.get_fpath()), 'should be enabeled'
>>> assert exists(cacher.get_fpath() + '.meta'), 'missing metadata'
>>> # Setting the cacher as enabled=False turns it off
>>> cacher2 = Cacher('test_disabled_save', 'params', enabled=False)
>>> cacher2.save('data')
>>> assert not exists(cacher2.get_fpath()), 'should be disabled'
"""
import ubelt as ub
if not self.enabled:
return
if self.verbose > 0:
self.log('[cacher] ... {} cache save'.format(self.fname))
cfgstr = self._rectify_cfgstr(cfgstr)
condensed = self._condense_cfgstr(cfgstr)
# Make sure the cache directory exists
ub.ensuredir(self.dpath)
data_fpath = self.get_fpath(cfgstr=cfgstr)
meta_fpath = data_fpath + '.meta'
# Also save metadata file to reconstruct hashing
with open(meta_fpath, 'a') as file_:
# TODO: maybe append this in json format?
file_.write('\n\nsaving {}\n'.format(ub.timestamp()))
file_.write(self.fname + '\n')
file_.write(condensed + '\n')
file_.write(cfgstr + '\n')
file_.write(str(self.meta) + '\n')
with open(data_fpath, 'wb') as file_:
# Use protocol 2 to support python2 and 3
pickle.dump(data, file_, protocol=self.protocol)
def renew(self, cfgstr=None, product=None):
"""
Recertify that the product has been recomputed by writing a new
certificate to disk.
"""
products = self._rectify_products(product)
certificate = {
'timestamp': ub.timestamp(),
'product': products,
}
if products is not None:
if not all(map(exists, products)):
raise IOError(
'The stamped product must exist: {}'.format(products))
certificate['product_file_hash'] = self._product_file_hash(
products)
self.cacher.save(certificate, cfgstr=cfgstr)
def cmake_clean(dpath='.'):
"""
"""
dpath = ub.expandpath(dpath)
cmake_cache_fpath = join(dpath, 'CMakeCache.txt')
if not exists(cmake_cache_fpath):
raise Exception('This does not look like a cmake build directory. '
'No CMakeCache.txt exists')
fpath_set = set(glob.glob(join(dpath, '*'))) - {cmake_cache_fpath}
for fpath in list(fpath_set):
if basename(fpath).startswith('_cmake_build_backup_'):
fpath_set.remove(fpath)
backup_dpath = ub.ensuredir(
join(dpath, '_cmake_build_backup_' + ub.timestamp()))
for fpath in ub.ProgIter(fpath_set, 'moving files'):
shutil.move(fpath, backup_dpath)
def export_model_code(dpath, model, initkw=None):
"""
Exports the class used to define a pytorch model as a new python module.
Exports the minimum amount of code needed to make a self-contained Python
module defining the pytorch model class. This exports the actual source
code. The advantage of using this over pickle is that the original code can
change arbitrarilly because all dependencies on the original code are
removed in the exported code.
Args:
dpath (str): directory to dump the model
model (tuple or type or object): class or class instance (e.g. torch.nn.Module)
name (str): name to use for the file (defaults to the classname)
initkw (dict): if specified, creates the function `make`, which
initializes the network with the specific arguments.
Returns:
str: static_modpath: path to the saved model file.
While you could put the output path in your PYTHONPATH, it is best
to use `ub.import_module_from_path` to "load" the model instead.
CommandLine:
xdoctest -m netharn.export.exporter export_model_code
Example:
>>> from torchvision.models import densenet
>>> from os.path import basename
>>> initkw = {'growth_rate': 16}
>>> model = densenet.DenseNet(**initkw)
>>> dpath = ub.ensure_app_cache_dir('netharn/tests')
>>> static_modpath = export_model_code(dpath, model, initkw)
>>> print('static_modpath = {!r}'.format(static_modpath))
...
>>> print(basename(static_modpath))
DenseNet_256629.py
>>> # now the module can be loaded
>>> module = ub.import_module_from_path(static_modpath)
>>> loaded = module.make()
>>> assert model.features.denseblock1.denselayer1.conv2.out_channels == 16
>>> assert loaded.features.denseblock1.denselayer1.conv2.out_channels == 16
>>> assert model is not loaded
"""
if isinstance(model, type):
model_class = model
else:
model_class = model.__class__
classname = model_class.__name__
if initkw is None:
raise NotImplementedError(
'ERROR: The params passed to the model __init__ must be available')
footer = ''
else:
# First see if we can get away with a simple encoding of initkw
try:
# Do not use repr. The text produced is non-deterministic for
# dictionaries. Instead, use ub.repr2, which is deterministic.
init_text = ub.repr2(initkw, nl=1)
eval(init_text, {})
init_code = ub.codeblock('initkw = {}').format(init_text)
except Exception:
# fallback to pickle
warnings.warn('Initialization params might not be serialized '
'deterministically')
init_bytes = repr(pickle.dumps(initkw, protocol=0))
init_code = ub.codeblock('''
import pickle
initkw = pickle.loads({})
''').format(init_bytes)
init_code = ub.indent(init_code).lstrip()
# create a function to instanciate the class
footer = '\n\n' + ub.codeblock('''
__pt_export_version__ = '{__pt_export_version__}'
def get_initkw():
""" creates an instance of the model """
{init_code}
return initkw
def get_model_cls():
model_cls = {classname}
return model_cls
def make():
""" creates an instance of the model """
initkw = get_initkw()
model_cls = get_model_cls()
model = model_cls(**initkw)
return model
''').format(classname=classname,
init_code=init_code,
__pt_export_version__=__pt_export_version__)
# TODO: assert that the name "make" is not used in the model body
body = closer.source_closure(model_class)
body_footer = body + footer + '\n'
# dont need to hash the header, because comments are removed anyway
hashid = hash_code(body_footer)
header = ub.codeblock('''
"""
This module was autogenerated by netharn/export/exporter.py
original_module={}
classname={}
timestamp={}
hashid={}
"""
''').format(model_class.__module__, classname, ub.timestamp(), hashid)
sourcecode = header + '\n' + body_footer
static_modname = classname + '_' + hashid[0:6]
static_modpath = join(dpath, static_modname + '.py')
with open(static_modpath, 'w') as file:
file.write(sourcecode)
return static_modpath
def timestamp_badmethod():
with pytest.raises(ValueError):
ub.timestamp(method='not real')
def test_timestamp():
stamp = ub.timestamp()
assert re.match(r'\d+-\d+-\d+T\d+[\+\-]\d+', stamp)
def test_timestamp_value_error():
with pytest.raises(ValueError):
ub.timestamp(method='bad-method')
def test_timestamp():
stamp = ub.timestamp()
assert re.match('\d+-\d+-\d+T\d+\+\d+', stamp)
def initialize(harn):
# TODO: Initialize the classes and then have a different function move
# everything to GPU
harn.xpu.set_as_default()
harn.debug('harn.xpu = {!r}'.format(harn.xpu))
if harn.train_dpath is None:
harn.setup_dpath(harn.workdir)
use_file_logger = True
if use_file_logger and harn.flog is None:
flog_fname = 'fitlog_{}.log'.format(ub.timestamp())
flog_fpath = join(harn.train_dpath, flog_fname)
flog = logging.getLogger(harn.__class__.__name__)
formatter = logging.Formatter('%(asctime)s : %(message)s')
handler = logging.FileHandler(flog_fpath, mode='w')
handler.setFormatter(formatter)
flog.propagate = False
flog.setLevel(logging.DEBUG)
flog.addHandler(handler)
harn.flog = flog
harn.debug('initialized file logger')
if tensorboard_logger:
train_base = os.path.dirname(harn.nice_dpath or harn.train_dpath)
harn.log('dont forget to start: tensorboard --logdir ' +
train_base)
harn.log('Initializing tensorboard')
harn.tlogger = tensorboard_logger.Logger(harn.train_dpath,
flush_secs=2)
if harn.dry:
harn.log('Dry run of training harness. xpu={}'.format(harn.xpu))
harn.optimizer = None
else:
prev_states = harn.prev_snapshots()
model_name = harn.hyper.model_cls.__name__
if harn.hyper.criterion_cls:
harn.log('Criterion: {}'.format(
harn.hyper.criterion_cls.__name__))
else:
harn.log('Criterion: Custom')
harn.log('Optimizer: {}'.format(harn.hyper.optimizer_cls.__name__))
if harn.hyper.scheduler_cls:
harn.log('Scheduler: {}'.format(
harn.hyper.scheduler_cls.__name__))
else:
harn.log('No Scheduler')
harn.model = harn.hyper.make_model()
harn.initializer = harn.hyper.make_initializer()
harn.log('Mounting {} model on {}'.format(model_name, harn.xpu))
harn.model = harn.xpu.mount(harn.model)
n_params = number_of_parameters(harn.model)
harn.log('Model has {!r} parameters'.format(n_params))
# more than one criterion? Wrap it in a single criterion OR
# specify a custom batch runner.
if harn.hyper.criterion_cls:
harn.criterion = harn.hyper.criterion_cls(
**harn.hyper.criterion_params)
harn.log('Move {} model to {}'.format(harn.criterion,
harn.xpu))
harn.criterion = harn.xpu.move(harn.criterion)
else:
pass
harn.log('Make optimizer')
harn.optimizer = harn.hyper.make_optimizer(harn.model.parameters())
if harn.hyper.scheduler_cls:
harn.log('Make scheduler')
harn.scheduler = harn.hyper.make_scheduler(harn.optimizer)
needs_init = True
harn.log('There are {} existing snapshots'.format(
len(prev_states)))
if prev_states and not ub.argflag('--reset'):
harn.log('Loading previous states')
# Ignore corrupted snapshots
for load_path in reversed(prev_states):
try:
harn.load_snapshot(load_path)
except RuntimeError:
harn.log(
'Failed to load {}. Skiping.'.format(load_path))
else:
needs_init = False
break
for i, group in enumerate(harn.optimizer.param_groups):
if 'initial_lr' not in group:
raise KeyError(
"param 'initial_lr' is not specified "
"in param_groups[{}] when resuming an optimizer".
format(i))
if needs_init:
harn.log('Initializing new model')
if harn.initializer.__class__.__name__ == 'LSUV':
# harn.model = harn.xpu.mount(harn.model)
# set([p.is_cuda for p in harn.model.parameters()])
#hack LSUV needs a batch of data to run
with grad_context(False):
# import utool
# utool.embed()
loader = harn.loaders['train']
input, labels = next(iter(loader))
data = harn.xpu.variable(input)
harn.initializer(harn.model, data)
else:
harn.initializer(harn.model)
if not harn.dry:
for group in harn.optimizer.param_groups:
group.setdefault('initial_lr', group['lr'])
harn.log(
'Snapshots will save to harn.snapshot_dpath = {!r}'.format(
harn.snapshot_dpath))
def _debug(msg):
msg_ = '[{}, {}, {}] '.format(ub.timestamp(), proc, proc.pid) + msg + '\n'
if is_main:
print(msg_)
with open(cache_gpath + '.atomic.debug', 'a') as f:
f.write(msg_)
def _cog_cache_write(gpath, cache_gpath, config=None):
"""
CommandLine:
xdoctest -m ndsampler.abstract_frames _cog_cache_write
Example:
>>> # xdoctest: +REQUIRES(module:osgeo)
>>> import ndsampler
>>> from ndsampler.abstract_frames import *
>>> import kwcoco
>>> workdir = ub.ensure_app_cache_dir('ndsampler')
>>> dset = kwcoco.CocoDataset.demo()
>>> imgs = dset.images()
>>> id_to_name = imgs.lookup('file_name', keepid=True)
>>> id_to_path = {gid: join(dset.img_root, name)
>>> for gid, name in id_to_name.items()}
>>> self = SimpleFrames(id_to_path, workdir=workdir)
>>> image_id = ub.peek(id_to_name)
>>> #gpath = self._lookup_gpath(image_id)
#### EXIT
# >>> hashid = self._lookup_hashid(image_id)
# >>> cog_gname = '{}_{}.cog.tiff'.format(image_id, hashid)
# >>> cache_gpath = cog_gpath = join(self.cache_dpath, cog_gname)
# >>> _cog_cache_write(gpath, cache_gpath, {})
"""
assert config is not None
# FIXME: if gdal_translate is not installed (because libgdal exists, but
# gdal-bin doesn't) then this seems to fail without throwing an error when
# hack_use_cli=1.
hack_use_cli = config.pop('hack_use_cli', False)
if DEBUG_COG_ATOMIC_WRITE:
import multiprocessing
from multiprocessing import current_process
from threading import current_thread
is_main = (current_thread().name == 'MainThread' and
current_process().name == 'MainProcess')
proc = multiprocessing.current_process()
def _debug(msg):
msg_ = '[{}, {}, {}] '.format(ub.timestamp(), proc, proc.pid) + msg + '\n'
if is_main:
print(msg_)
with open(cache_gpath + '.atomic.debug', 'a') as f:
f.write(msg_)
_debug('attempts aquire'.format())
if not hack_use_cli:
# Load all the image data and dump it to cog format
import kwimage
if DEBUG_COG_ATOMIC_WRITE:
_debug('reading data')
raw_data = kwimage.imread(gpath)
# raw_data = kwimage.atleast_3channels(raw_data, copy=False)
# TODO: THERE HAS TO BE A CORRECT WAY TO DO THIS.
# However, I'm not sure what it is. I extend my appologies to whoever is
# maintaining this code. Note: mode MUST be 'w'
with atomicwrites.atomic_write(cache_gpath + '.atomic', mode='w', overwrite=True) as file:
if DEBUG_COG_ATOMIC_WRITE:
_debug('begin')
_debug('gpath = {}'.format(gpath))
_debug('cache_gpath = {}'.format(cache_gpath))
try:
file.write('begin: {}\n'.format(ub.timestamp()))
file.write('gpath = {}\n'.format(gpath))
file.write('cache_gpath = {}\n'.format(cache_gpath))
if not exists(cache_gpath):
if not hack_use_cli:
util_gdal._imwrite_cloud_optimized_geotiff(
cache_gpath, raw_data, **config)
else:
# The CLI is experimental and might make this pipeline
# faster by avoiding the initial read.
util_gdal._cli_convert_cloud_optimized_geotiff(
gpath, cache_gpath, **config)
if DEBUG_COG_ATOMIC_WRITE:
_debug('finished write: {}\n'.format(ub.timestamp()))
else:
if DEBUG_COG_ATOMIC_WRITE:
_debug('ALREADY EXISTS did not write: {}\n'.format(ub.timestamp()))
file.write('end: {}\n'.format(ub.timestamp()))
except Exception as ex:
file.write('FAILED DUE TO EXCEPTION: {}: {}\n'.format(ex, ub.timestamp()))
if DEBUG_COG_ATOMIC_WRITE:
_debug('FAILED DUE TO EXCEPTION: {}'.format(ex))
raise
finally:
if DEBUG_COG_ATOMIC_WRITE:
_debug('finally')
if RUN_COG_CORRUPTION_CHECKS:
# CHECK THAT THE DATA WAS WRITTEN CORRECTLY
file = util_gdal.LazyGDalFrameFile(cache_gpath)
is_valid = util_gdal.validate_nonzero_data(file)
if not is_valid:
if hack_use_cli:
import kwimage
raw_data = kwimage.imread(gpath)
# The check may fail on zero images, so check that
orig_sum = raw_data.sum()
# cache_sum = file[:].sum()
# if DEBUG:
# _debug('is_valid = {}'.format(is_valid))
# _debug('cache_sum = {}'.format(cache_sum))
if orig_sum > 0:
print('FAILED TO WRITE COG FILE')
print('orig_sum = {!r}'.format(orig_sum))
# print(kwimage.imread(cache_gpath).sum())
if DEBUG_COG_ATOMIC_WRITE:
_debug('FAILED TO WRITE COG FILE')
ub.delete(cache_gpath)
raise CorruptCOG('FAILED TO WRITE COG FILE CORRECTLY')
