def fetch_models(search_dir, registered_models_file=None, filter=None, flash=True,
load_net=False,
show_debug=False,
**kw):
if not registered_models_file:
registered_models_file = 'models-{}.json'.format(gethostname())
if flash:
logging.debug('Flash collecting networks')
try:
rmodels = load_json(search_dir, registered_models_file)
with turnoff_debug(turnoff=not show_debug):
return gather_registered_models(rmodels, filter, load_net=load_net, **kw)
except (FileNotFoundError, NoModelError) as e:
logging.warning('%s not found, will recollect networks', e)
flash = False
if not flash:
logging.debug('Collecting networks')
with turnoff_debug(turnoff=not show_debug):
list_of_networks = collect_models(search_dir,
load_net=False,
**kw)
filter_keys = get_filter_keys()
rmodels = register_models(list_of_networks, *filter_keys)
save_json(rmodels, search_dir, registered_models_file)
return fetch_models(search_dir, registered_models_file, filter=filter, flash=True,
load_net=load_net, **kw)
logging.getLogger().setLevel(logging.WARNING)
if args.verbose > 1:
logging.getLogger().setLevel(logging.INFO)
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
if args.texify:
texify = configparser.ConfigParser()
texify.read(args.texify)
else:
texify = {}
flash = args.flash
registered_models_file = 'models-' + gethostname() + '.json'
all_models = fetch_models(args.job_dir, registered_models_file, load_net=False, flash=flash)
filter_keys = get_filter_keys(args.filters, by='key')
for config_file in args.config_files:
config = configparser.ConfigParser()
config.read(config_file)
which = args.which
if 'all' in which:
which = list(config.keys())
if 'DEFAULT' in which:
which.remove('DEFAULT')
else:
which = [w for w in which if w in config]
parser.add_argument('--job-dir', default='./jobs')
parser.add_argument('--batch-size', type=int, default=32)
parser.add_argument('--num-batch', type=int, default=int(1e6))
parser.add_argument('--device', default='cuda')
parser.add_argument('--saved-samples-per-batch', type=int, default=2)
args_from_file = (
'-vvvv '
# '--jobs 193080 193082'
'--jobs 169381').split()
args, ra = parser.parse_known_args(
None if len(sys.argv) > 1 else args_from_file)
rmodels = load_json(args.job_dir, 'models-{}.json'.format(gethostname()))
wanted = args.when
logging.getLogger().setLevel(40 - 10 * args.v)
if len(args.jobs) < 2:
logging.error('At least two jobs (%d provided)', len(args.jobs))
sys.exit(1)
mdirs_ = {
rmodels[_]['job']: _
for _ in rmodels if rmodels[_]['job'] in args.jobs
}
if len(mdirs_) < len(set(args.jobs)):
logging.error('Jobs not found')
parser.add_argument('--compute', action='store_true')
parser.add_argument('--last', default=0, type=int)
parser.add_argument('--method', default='iws-2s')
parser.add_argument('--tpr', type=float, default=0.95)
parser.add_argument('-v', action='count', default=0)
parser.add_argument('--result-dir', default='/tmp')
parser.add_argument('--agg-type', nargs='*', choices=list(agg_type_letter), default=[])
parser.add_argument('--when', default='last')
parser.add_argument('--plot', nargs='?', const='p')
parser.add_argument('--tex', action='store_true')
parser.add_argument('--sets-to-exclude', nargs='*', default=[])
parser.add_argument('--combos', nargs='+', type=int)
parser.add_argument('--device', default='cpu')
rmodels = load_json('jobs', 'models-{}.json'.format(gethostname()))
col_width = 10
str_col_width = '13.13'
flt_col_width = '5.1f'
_ = np.seterr(divide='ignore', invalid='ignore')
def diff(t):
# return t.diff()
return t[..., 1:] - t[..., :-1]
if __name__ == '__main__':
import sys
import argparse
from utils.parameters import alphanum, list_of_alphanums, get_args, set_log, gethostname
from utils.save_load import collect_models, find_by_job_number, NoModelError
from utils.print_log import EpochOutput
from utils.signaling import SIGHandler
import setproctitle
if __name__ == '__main__':
# sys.argv[0] = 'training'
# setproctitle.setproctitle(' '.join(sys.argv))
setproctitle.setproctitle('training')
hostname = gethostname()
args = get_args(what_for='train')
debug = args.debug
verbose = args.verbose
job_dir = args.job_dir
job_number = args.job_number
log_dir = os.path.join(args.output_dir, 'log')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
log = set_log(verbose, debug, log_dir, job_number=job_number)
log.debug('$ ' + ' '.join(sys.argv))