def __init__(self, uri, client_factory=pymongo.MongoClient):
self.chrono = {}
debug('connecting to server')
self.client = client_factory(uri)
# Fetch Database
self.track = self.client.track
# Fetch collections
self.trials = self.track.trials
self.projects = self.track.projects
self.groups = self.track.groups
def new_project(self, project: Project):
debug(f'create new (project: {project.name})')
if project.uid in self.storage.objects:
error(f'Cannot insert project; (uid: {project.uid}) already exists!')
return self.get_project(project)
self.storage.objects[project.uid] = project
self.storage.project_names[project.name] = project.uid
self.storage.projects.add(project.uid)
return project
def set_project(self,
project: Optional[Project] = None,
force: bool = False,
get_only: bool = False,
**kwargs):
"""Set or create a new project
Parameters
----------
project: Optional[Project]
project definition you can use to create or set the project
force: bool
by default once the project is set it cannot be changed.
use force to override this behaviour.
get_only: bool
if true does not insert the project if missing.
default to false
kwargs
arguments used to create a `Project` object if no project object were provided
See :func:`~track.structure.Project` for possible arguments
Returns
-------
returns created project
"""
if self.project is not None and not force:
info('Project is already set, to override use force=True')
return self.project
if project is None:
project = Project(**kwargs)
assert project.name is not None, 'Project name cannot be none'
# does the project exist ?
self.project = self.protocol.get_project(project)
if self.project is not None:
return self.project
if get_only:
raise RuntimeError(
f'Project (name: {project.name}) was not found!')
self.project = self.protocol.new_project(project)
debug(f'set project to (project: {self.project.name})')
return self.project
def _setup(self, client='track_client'):
out = subprocess.check_output(
f'{self.bin} user set {client} --insecure --host={self.addrs}',
shell=True)
debug(out.decode('utf8').strip())
create_db = f"""
CREATE DATABASE IF NOT EXISTS track;
SET DATABASE = track;
GRANT ALL ON DATABASE track TO {client};
CREATE TABLE IF NOT EXISTS track.projects (
uid BYTES PRIMARY KEY,
name STRING,
description STRING,
metadata JSONB,
trial_groups BYTES[],
trials BYTES[]
);
CREATE TABLE IF NOT EXISTS track.trial_groups (
uid BYTES PRIMARY KEY,
name STRING,
description STRING,
metadata JSONB,
trials BYTES[],
project_id BYTES
);
CREATE TABLE IF NOT EXISTS track.trials (
uid BYTES,
hash BYTES,
revision SMALLINT,
name STRING,
description STRING,
tags JSONB,
version BYTES,
group_id BYTES,
project_id BYTES,
parameters JSONB,
metadata JSONB,
metrics JSONB,
chronos JSONB,
status JSONB,
errors JSONB,
PRIMARY KEY (hash, revision)
);""".encode('utf8')
out = subprocess.check_output(
f'{self.bin} sql --insecure --host={self.addrs}',
input=create_db,
shell=True)
debug(out.decode('utf8').strip())
def new_trial_group(self, group: TrialGroup):
debug(f'create new (group: {group.name})')
if group.uid in self.storage.objects:
error(f'Cannot insert group; (uid: {group.uid}) already exists!')
return
project = self.storage.objects.get(group.project_id)
if self.strict:
assert project is not None, 'Cannot create a group without an associated project'
project.groups.add(group)
self.storage.objects[group.uid] = group
self.storage.groups.add(group.uid)
self.storage.group_names[group.name] = group.uid
return group
def __init__(self, uri):
uri = parse_uri(uri)
debug('connecting to server')
self.con = psycopg2.connect(
database='track',
user=uri.get('username', 'track_client'),
password=uri.get('password', 'track_password'),
# sslmode='require',
# sslrootcert='certs/ca.crt',
# sslkey='certs/client.maxroach.key',
# sslcert='certs/client.maxroach.crt',
port=uri['port'],
host=uri['address'])
self.con.set_session(autocommit=True)
debug('get connection cursor')
self.cursor = self.con.cursor()
self.chrono = {}
def get_protocol(backend_name):
""" proto://arg """
arguments = parse_uri(backend_name)
log = _protocols.get(arguments['scheme'])
if log is None:
warning(f'Logger (backend: {backend_name}) was not found!')
log = _protocols.get('__default__')
if log is make_local:
debug('return local protocol')
return log(backend_name)
else:
debug('return multiplexed protocol')
return ProtocolMultiplexer(
# Make a file Protocol to log everything in memory as well as remotely
make_local('file:', strict=False, eager=False),
log(backend_name))
def commit(self, file_name_override=None, **kwargs):
if file_name_override is None:
file_name_override = self.target_file
if file_name_override is None:
debug('No output file target')
return None
# only save top level projects
objects = []
for uid in self._projects:
objects.append(to_json(self._objects[uid]))
# print(json.dumps(objects[-1], indent=2))
file_name = tempfile.mktemp('track_uncommitted')
with open(file_name, 'w') as output:
json.dump(objects, output, indent=2)
# mv is kind of atomic so this prevent generating half generated files
os.rename(file_name, file_name_override)
def new_trial(self, trial: Trial, auto_increment=False):
debug(f'create new (trial: {trial.uid})')
if trial.uid in self.storage.objects:
if not auto_increment:
debug('Found already existing trial')
return None
trials = self.get_trial(trial)
max_rev = 0
for t in trials:
max_rev = max(max_rev, t.revision)
warning(f'Trial was already completed. Increasing revision number (rev={max_rev + 1})')
trial.revision = max_rev + 1
trial._hash = None
self.storage.objects[trial.uid] = trial
self.storage.trials.add(trial.uid)
if trial.project_id is not None:
project = self.storage.objects.get(trial.project_id)
if project is not None or self.strict:
project.trials.add(trial)
else:
warning('Orphan trial')
if trial.group_id is not None:
group = self.storage.objects.get(trial.group_id)
if group is not None or self.strict:
group.trials.add(trial.uid)
trial.metadata['_update_count'] = 0
return trial
def get_project(self, project: Project):
debug(f'look for (project: {project.name})')
return self.storage.objects.get(project.uid)
def __init__(self, uri, client_factory=pymongo.MongoClient):
self.chrono = {}
debug('connecting to server')
self.client = client_factory(uri)