def upload_one(exp_path: Path):
"""
Upload one experiment result to database together with the model checkpoints,
the logfile and tensorboardlogs, then delete zipped experiment dir.
"""
is_zip = exp_path.suffix == '.zip'
with tempfile.TemporaryDirectory() as tmpdirname:
tmpdir = Path(tmpdirname) / exp_path.stem
tmpdir.mkdir()
if is_zip:
# unpack zip into tmpdir
log.info(f'Unpacking {exp_path} to {tmpdir}.')
with zipfile.ZipFile(exp_path) as z:
z.extractall(tmpdir)
exp_dir = Path(tmpdir)
else:
exp_dir = exp_path
flags = torch.load(exp_dir / 'flags.rar')
db = MongoDatabase(training=True, flags=flags)
results = {'epoch_results': {}}
epochs = sorted(int(str(epoch.stem)) for epoch in (exp_dir / 'epoch_results').iterdir())
for epoch in epochs:
epoch_str = str(epoch)
epoch_results = (exp_dir / 'epoch_results' / epoch_str).with_suffix('.json')
results['epoch_results'][epoch_str] = json2dict(epoch_results)
db.insert_dict(results)
modalities = [mod_str for mod_str in results['epoch_results'][str(epoch)]['train_results']['log_probs'] if
len(mod_str.split('_')) == 1]
dir_checkpoints = exp_dir / 'checkpoints'
db.save_networks_to_db(
dir_checkpoints=dir_checkpoints,
epoch=max(int(str(d.name)) for d in dir_checkpoints.iterdir()),
modalities=modalities,
)
db.upload_tensorbardlogs(exp_dir / 'logs')
pdf_path = run_notebook_convert(exp_dir)
expvis_url = ppb.upload(pdf_path, plain=True)
db.insert_dict({'expvis_url': expvis_url})
log_file = glob.glob(str(exp_dir) + '/*.log')
if len(log_file):
db.upload_logfile(Path(log_file[0]))
send_msg(f'Uploading of experiment {flags.experiment_uid} has finished. The experiment visualisation can be '
f'found here: {expvis_url}'
)
# delete exp_path
if is_zip:
exp_path.unlink()
else:
shutil.rmtree(exp_path)
def finalize(self, test_results: BaseTestResults, epoch: int, average_epoch_time):
log.info('Finalizing.')
# write results as json to experiment folder
run_metadata = {'end_epoch': epoch, 'experiment_duration': time.time() - self.begin_time,
'mean_epoch_time': self.callback.epoch_time.get_average()}
dict2json(self.flags.dir_experiment_run / 'results.json', test_results.__dict__ | run_metadata)
if self.flags.use_db == 1:
self.exp.experiments_database.insert_dict(run_metadata)
self.exp.experiments_database.save_networks_to_db(dir_checkpoints=self.flags.dir_checkpoints, epoch=epoch,
modalities=self.exp.mm_vae.modalities)
# self.exp.experiments_database.upload_logfile(self.flags.log_file)
self.exp.experiments_database.upload_tensorbardlogs(self.flags.dir_experiment_run / 'logs')
# run jupyter notebook with visualisations
pdf_path = run_notebook_convert(self.flags.dir_experiment_run)
# send alert
if self.flags.norby and self.flags.dataset != 'toy':
import ppb
import norby
expvis_url = ppb.upload(pdf_path, plain=True)
self.exp.experiments_database.insert_dict({'expvis_url': expvis_url})
norby.send_msg(f'Experiment {self.flags.experiment_uid} has finished. The experiment visualisation can be '
f'found here: {expvis_url}')
if self.flags.log_file.exists():
shutil.move(self.flags.log_file, self.flags.dir_experiment_run)
def run_one(exp_dir: str):
exp_dir = Path(exp_dir).expanduser()
log.info(
f'Starting execution of experiment vis for experiment {exp_dir.name}')
pdf_path = run_notebook_convert(exp_dir)
expvis_url = ppb.upload(pdf_path, plain=True)
log.info(f'Uploaded experiment vis to {expvis_url}')
db = MongoDatabase(training=False, _id=exp_dir.name)
db.insert_dict({'expvis_url': expvis_url})
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--zip_flag',
'-zip_flag',
action='store_true',
dest='zip_flag',
help="zip folder and upload it. The folder to be zipped needs to be "
"after the flag -z. Example: ppb -z test")
parser.add_argument(
'--plain',
action='store_true',
dest='plain',
help=
"if this flag is set, no syntax highlighting will be added to the file."
)
parser.add_argument(
'--lifetime',
default=20,
help=
"Time in days after which the file is deleted. Set -1 to never delete de file."
)
parser.add_argument('--description',
'-d',
default='',
type=str,
help="Description stored in the ppb dataframe.")
parser.add_argument('file', type=str, help='file to upload and display')
flags = parser.parse_args()
file_path = Path(flags.file.replace(" ", "\\ "))
upload(file_path=file_path,
zip_flag=flags.zip_flag,
plain=flags.plain,
description=flags.description,
lifetime=flags.lifetime)
def upload_notebook_to_db(experiment_uid: str) -> None:
"""
Run the experiment vis notebook and upload it with ppb to db.
"""
import ppb
with tempfile.TemporaryDirectory() as tmpdirname:
dir_experiment_run = Path(tmpdirname) / experiment_uid
dir_experiment_run.mkdir()
db = MongoDatabase(training=False, _id=experiment_uid)
dict2json(dir_experiment_run / 'flags.json', db.get_experiment_dict()['flags'])
pdf_path = run_notebook_convert(dir_experiment_run=dir_experiment_run)
expvis_url = ppb.upload(pdf_path, plain=True)
log.info(f'Experiment_vis was uploaded to {expvis_url}')
db.insert_dict({'expvis_url': expvis_url})
def test_upload():
test_file = Path(__file__).parent / 'test_file.txt'
url = upload(file_path=test_file)
print(f'This is my url: {url}')