def _inference_execute(cls, runner_class, **kwargs):
"""Run a lightweight version of the execute with no logging and extra stuff
***NOTE***
TBWriter() will NOT be initialized in this mode!!
Logging will NOT be available
The environment (CUDA devices and the link) is not setup (GUST takes care of it)
Parameters
----------
Specify which is the runner class to select
Returns
-------
payload : dict
Dictionary with the payload to return after inference
"""
def get_all_concrete_subclasses(class_name):
csc = set() # concrete subclasses
if not inspect.isabstract(class_name):
csc.add(class_name)
for c in class_name.__subclasses__():
csc = csc.union(get_all_concrete_subclasses(c))
return csc
# Set up execution environment. Specify CUDA_VISIBLE_DEVICES and seeds
cls._set_up_env(**kwargs)
# Find all subclasses of AbstractRunner and BaseRunner and select the chosen one among them based on -rc
sub_classes = get_all_concrete_subclasses(AbstractRunner)
runner_class = [c for c in sub_classes if to_capital_camel_case(runner_class).lower() == c.__name__.lower()]
assert len(runner_class) == 1
runner_class = runner_class[0]
# Run the actual experiment
start_time = time.time()
return_value = runner_class().single_run(**kwargs)
print(f'Payload (RunMe.py): {return_value}')
print(f'Time taken: {(time.time() - start_time) * 1000:.0f}ms')
return return_value
def _execute(args):
"""
Run an experiment once with the given parameters specified by command line.
This is typical usage scenario.
Parameters
----------
args : dict
Contains all command line arguments parsed.
Returns
-------
train_scores : ndarray[floats] of size (1, `epochs`)
Score values for train split
val_scores : ndarray[floats] of size (1, `epochs`+1)
Score values for validation split
test_scores : float
Score value for test split
"""
# Set up logging
# Don't use args.output_folder as that breaks when using SigOpt
current_log_folder, writer = set_up_logging(parser=RunMe.parser,
args_dict=args.__dict__,
**args.__dict__)
# Copy the code into the output folder
copy_code(output_folder=current_log_folder)
# Check Git status to verify all local changes have been committed
if args.ignoregit:
logging.warning('Git status is ignored!')
else:
try:
local_changes = False
deepdiva_directory = os.path.split(os.getcwd())[0]
git_url = subprocess.check_output(
["git", "config", "--get", "remote.origin.url"])
git_branch = subprocess.check_output(
["git", "rev-parse", "--abbrev-ref", "HEAD"])
git_hash = subprocess.check_output(
["git", "rev-parse", "--short", "HEAD"])
git_status = str(subprocess.check_output(["git", "status"]))
logging.debug(
'DeepDIVA directory is:'.format(deepdiva_directory))
logging.info('Git origin URL is: {}'.format(str(git_url)))
logging.info('Current branch and hash are: {} {}'.format(
str(git_branch), str(git_hash)))
local_changes = "nothing to commit" not in git_status and \
"working directory clean" not in git_status
if local_changes:
logging.warning(
'Running with an unclean working tree branch!')
except Exception as exp:
logging.warning('Git error: {}'.format(exp))
local_changes = True
finally:
if local_changes:
logging.error(
'Errors when acquiring git status. Use --ignoregit to still run.\n'
'This happens when the git folder has not been found on the file system\n'
'or when the code is not the same as the last version on the repository.\n'
'If you are running on a remote machine make sure to sync the .git folder as well.'
)
logging.error(
'Finished with errors. (Log files at {} )'.format(
current_log_folder))
logging.shutdown()
sys.exit(-1)
# Set up execution environment. Specify CUDA_VISIBLE_DEVICES and seeds
set_up_env(**args.__dict__)
# Select with introspection which runner class should be used.
# Default is runner.image_classification.image_classification
runner_class = getattr(
sys.modules["template.runner." + args.runner_class],
args.runner_class).__dict__[to_capital_camel_case(
args.runner_class)]
try:
# Run the actual experiment
start_time = time.time()
if args.multi_run is not None:
train_scores, val_scores, test_scores = RunMe._multi_run(
runner_class=runner_class,
writer=writer,
current_log_folder=current_log_folder,
args=args)
else:
train_scores, val_scores, test_scores = runner_class.single_run(
writer=writer,
current_log_folder=current_log_folder,
**args.__dict__)
end_time = time.time()
logging.info('Time taken for train/eval/test is: {}'.format(
datetime.timedelta(seconds=int(end_time - start_time))))
except Exception as exp:
if args.quiet:
print('Unhandled error: {}'.format(repr(exp)))
logging.error('Unhandled error: %s' % repr(exp))
logging.error(traceback.format_exc())
logging.error('Execution finished with errors :(')
sys.exit(-1)
finally:
# Free logging resources
logging.shutdown()
logging.getLogger().handlers = []
writer.close()
print('All done! (Log files at {} )'.format(current_log_folder))
return train_scores, val_scores, test_scores
def _execute(cls, ignoregit, runner_class, multi_run, quiet, **kwargs):
"""
Run an experiment once with the given parameters specified by command line.
This is typical usage scenario.
Parameters
----------
ignoregit : bool
Flag for verify git status
runner_class : str
Specify which is the runner class to select
multi_run : int
If not None, indicates how many multiple runs needs to be done
quiet : bool
Specify whether to print log to console or only to text file
Returns
-------
return_value : dict
Dictionary with the return value of the runner
"""
def get_all_concrete_subclasses(class_name):
csc = set() # concrete subclasses
if not inspect.isabstract(class_name):
csc.add(class_name)
for c in class_name.__subclasses__():
csc = csc.union(get_all_concrete_subclasses(c))
return csc
# Set up logging
# Don't use args.output_folder as that breaks when using SigOpt
unpacked_args = {'ignoregit': ignoregit, 'runner_class': runner_class, 'multi_run': multi_run, 'quiet': quiet}
current_log_folder = cls._set_up_logging(parser=RunMe.parser, quiet=quiet, args_dict={**kwargs, **unpacked_args},
**kwargs)
# Copy the code into the output folder
cls._copy_code(output_folder=current_log_folder)
# Check Git status to verify all local changes have been committed
if ignoregit:
logging.warning('Git status is ignored!')
else:
cls._verify_git_status(current_log_folder)
# Set up execution environment. Specify CUDA_VISIBLE_DEVICES and seeds
cls._set_up_env(**kwargs)
# kwargs["device"] = torch.device(kwargs["device"] if torch.cuda.is_available() else "cpu")
kwargs["device"] = torch.device('cuda' if torch.cuda.is_available() else "cpu")
# Find all subclasses of AbstractRunner and BaseRunner and select the chosen one among them based on -rc
sub_classes = get_all_concrete_subclasses(AbstractRunner)
runner_class = [c for c in sub_classes if to_capital_camel_case(runner_class).lower() == c.__name__.lower()]
assert len(runner_class) == 1
runner_class = runner_class[0]
# TODO: make this more elegant
# repack the runner_class with the reference for the actual runner, not as a string
unpacked_args['runner_class'] = runner_class
logging.warning(f'Current working directory is: {os.getcwd()}')
# Run the actual experiment
start_time = time.time()
try:
if multi_run is not None:
return_value = cls._multi_run(current_log_folder=current_log_folder, **unpacked_args, **kwargs)
else:
return_value = runner_class().single_run(current_log_folder=current_log_folder, **unpacked_args, **kwargs)
logging.info(f'Time taken: {datetime.timedelta(seconds=time.time() - start_time)}')
except Exception as exp:
if quiet:
print('Unhandled error: {}'.format(repr(exp)))
logging.error('Unhandled error: %s' % repr(exp))
logging.error(traceback.format_exc())
logging.error('Execution finished with errors :(')
raise SystemExit
finally:
# Free logging resources
logging.shutdown()
logging.getLogger().handlers = []
TBWriter().close()
print('All done! (Log files at {} )'.format(current_log_folder))
return return_value