def get_mongodb_config(path=SETTINGS.DATABASE.MONGODB_CONFIG_PATH):
"""Read the MongoDB connection configuration.
Reads the file at the provided path or otherwise {SETTINGS.DATABASE.MONGODB_CONFIG_PATH} to get
- database host
- database port
- database name
- username
- password
Default path is $HOME/.config/seml/mongodb.config.
Config file should be in the format:
username: <your_username>
password: <your_password>
port: <port>
database: <database_name>
host: <host>
Returns
-------
dict
Contains the MongoDB config as detailed above.
"""
access_dict = {}
config_str = "\nPlease run `seml configure` to provide your credentials."
if not path.exists():
raise MongoDBError(
f"MongoDB credentials could not be read at '{path}'.{config_str}")
with open(path, 'r') as f:
for line in f.readlines():
# ignore lines that are empty
if len(line.strip()) > 0:
split = line.split(':')
key = split[0].strip()
value = split[1].strip()
access_dict[key] = value
required_entries = ['username', 'password', 'port', 'host', 'database']
for entry in required_entries:
if entry not in access_dict:
raise MongoDBError(f"No {entry} found in '{path}'.{config_str}")
db_port = access_dict['port']
db_name = access_dict['database']
db_host = access_dict['host']
db_username = access_dict['username']
db_password = access_dict['password']
return {
'password': db_password,
'username': db_username,
'host': db_host,
'db_name': db_name,
'port': db_port
}
def reset_experiments(db_collection_name, sacred_id, filter_states, batch_id, filter_dict):
collection = get_collection(db_collection_name)
if sacred_id is None:
if len({*States.PENDING, *States.RUNNING, *States.KILLED} & set(filter_states)) > 0:
detect_killed(db_collection_name, print_detected=False)
if isinstance(filter_states, str):
filter_states = [filter_states]
filter_dict = build_filter_dict(filter_states, batch_id, filter_dict)
nreset = collection.count_documents(filter_dict)
exps = collection.find(filter_dict)
if nreset >= 10:
if input(f"Resetting the state of {nreset} experiment{s_if(nreset)}. "
f"Are you sure? (y/n) ").lower() != "y":
exit()
else:
logging.info(f"Resetting the state of {nreset} experiment{s_if(nreset)}.")
for exp in exps:
reset_single_experiment(collection, exp)
else:
exp = collection.find_one({'_id': sacred_id})
if exp is None:
raise MongoDBError(f"No experiment found with ID {sacred_id}.")
else:
logging.info(f"Resetting the state of experiment with ID {sacred_id}.")
reset_single_experiment(collection, exp)
def delete_experiments(db_collection_name, sacred_id, filter_states, batch_id, filter_dict):
collection = get_collection(db_collection_name)
if sacred_id is None:
if len({*States.PENDING, *States.RUNNING, *States.KILLED} & set(filter_states)) > 0:
detect_killed(db_collection_name, print_detected=False)
filter_dict = build_filter_dict(filter_states, batch_id, filter_dict)
ndelete = collection.count_documents(filter_dict)
batch_ids = collection.find(filter_dict, {'batch_id'})
batch_ids_in_del = set([x['batch_id'] for x in batch_ids])
if ndelete >= 10:
if input(f"Deleting {ndelete} configuration{s_if(ndelete)} from database collection. "
f"Are you sure? (y/n) ").lower() != "y":
exit()
else:
logging.info(f"Deleting {ndelete} configuration{s_if(ndelete)} from database collection.")
collection.delete_many(filter_dict)
else:
exp = collection.find_one({'_id': sacred_id})
if exp is None:
raise MongoDBError(f"No experiment found with ID {sacred_id}.")
else:
logging.info(f"Deleting experiment with ID {sacred_id}.")
batch_ids_in_del = set([exp['batch_id']])
collection.delete_one({'_id': sacred_id})
if len(batch_ids_in_del) > 0:
# clean up the uploaded sources if no experiments of a batch remain
delete_orphaned_sources(collection, batch_ids_in_del)
def get_command_from_exp(exp, db_collection_name, verbose=False, unobserved=False,
post_mortem=False, debug=False, debug_server=False, print_info=True):
if 'executable' not in exp['seml']:
raise MongoDBError(f"No executable found for experiment {exp['_id']}. Aborting.")
exe = exp['seml']['executable']
config = exp['config']
config['db_collection'] = db_collection_name
if not unobserved:
config['overwrite'] = exp['_id']
config_strings = [f'{key}="{val}"' if type(val) != str else f'{key}="\'{val}\'"' for key, val in config.items()]
if not verbose:
config_strings.append("--force")
if unobserved:
config_strings.append("--unobserved")
if post_mortem:
config_strings.append("--pdb")
if debug:
config_strings.append("--debug")
if debug_server:
ip_address, port = find_free_port()
if print_info:
logging.info(f"Starting debug server with IP '{ip_address}' and port '{port}'. "
f"Experiment will wait for a debug client to attach.")
interpreter = f"python -m debugpy --listen {ip_address}:{port} --wait-for-client"
else:
interpreter = "python"
return interpreter, exe, config_strings
def load_sources_from_db(exp, collection, to_directory):
db = collection.database
fs = gridfs.GridFS(db)
if 'source_files' not in exp['seml']:
raise MongoDBError(
f'No source files found for experiment with ID {exp["_id"]}.')
source_files = exp['seml']['source_files']
for path, _id in source_files:
_dir = f"{to_directory}/{os.path.dirname(path)}"
if not os.path.exists(_dir):
os.makedirs(
_dir,
mode=0o700) # only current user can read, write, or execute
with open(f'{to_directory}/{path}', 'wb') as f:
file = fs.find_one(_id)
if file is None:
raise MongoDBError(
f"Could not find source file with ID '{_id}' for experiment with ID {exp['_id']}."
)
f.write(file.read())