def dump_metadata(metadata_file, additional_metadata=None):
with open(metadata_file, "w") as t:
metadata = { 'timestamp' : timestamp_string(),
'argv' : sys.argv,
'user' : getpass.getuser(),
'cwd' : os.getcwd(),
'host' : {
'name' : socket.gethostname(),
'uptime' : backtick("uptime"),
'free' : backtick("exec 2>/dev/null free"),
'num_cores' : backtick("cat 2>/dev/null /proc/cpuinfo | grep '^processor[[:space:]]' | wc -l"),
'cpu_info' : backtick("cat 2>/dev/null /proc/cpuinfo | grep 'model name[[:space:]]' | uniq | sed 's/.*://' | perl -pi -e 's/\s+/ /g'")
},
'sys' : {
'lsb_release' : backtick("exec 2>/dev/null lsb_release --description --short"),
'uname' : backtick("uname -a")
},
'modules' : {
module : { 'commit' : backtick("git rev-parse HEAD", cwd=path),
'branch' : backtick("git rev-parse --abbrev-ref HEAD", cwd=path),
'status' : backtick("git status", cwd=path),
'diff' : backtick("git diff", cwd=path)
} for module, path in [("sts2", "./")]
},
'additional_metadata': additional_metadata,
}
t.write(json.dumps(metadata, sort_keys=True, indent=2, separators=(',', ": ")) + "\n")
def setup_experiment(args, config):
# Grab parameters
if args.exp_name:
config.exp_name = args.exp_name
# elif not hasattr(config, 'exp_name'):
# config.exp_name = exp_lifecycle.guess_config_name(config)
if not hasattr(config, 'results_dir'):
config.results_dir = "experiments/%s" % config.exp_name
if args.timestamp_results is not None:
# Note that argparse returns a list
config.timestamp_results = args.timestamp_results
if hasattr(config, 'timestamp_results') and config.timestamp_results:
now = timestamp_string()
config.results_dir += "_" + str(now)
# Set up results directory
mkdir_p("./experiments")
create_clean_dir(config.results_dir)
# Make sure that there are no uncommited changes
if args.publish:
exp_lifecycle.publish_prepare(config.exp_name, config.results_dir)
# Record machine information for this experiment
additional_metadata = None
if hasattr(config, "get_additional_metadata"):
additional_metadata = config.get_additional_metadata()
exp_lifecycle.dump_metadata("%s/metadata" % config.results_dir,
additional_metadata=additional_metadata)
def backup(self, incremental=False) -> bool:
ts_string = timestamp_string()
backup_dir = os.path.join(self._backup_dir, ts_string)
logger.info("Starting backup to folder {}".format(backup_dir))
try:
self.__activate_maintenance()
self.__verify_prior_backup()
self.__backup_database(backup_dir=backup_dir)
self.__verify_post_backup(backup_dir=backup_dir)
finally:
self.__deactivate_maintenance()
logger.info("Finished backup to folder {}".format(backup_dir))
return True