def close(self):
if self.loop is not None:
self.loop.close()
if self.sckt is not None:
info('Shutting down server')
self.sckt.close()
def new_trial(self, trial: Trial):
try:
self.cursor.execute(
"""
INSERT INTO
track.trials (uid, hash, revision, name, description,
tags, metadata, metrics, version, project_id, group_id, parameters, status)
VALUES
(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s);
""",
(trial.uid, trial.hash, trial.revision, trial.name,
trial.description, self.serialize(trial.tags),
self.serialize(trial.metadata), self.serialize(
trial.metrics), self.encode_version(
trial.version), self.encode_uid(trial.project_id),
self.encode_uid(
trial.group_id), self.serialize(
trial.parameters), self.serialize(trial.status)))
return trial
except psycopg2.errors.UniqueViolation:
trial.revision += 1
info(
f'Trial already exist increasing revision (rev: {trial.revision})'
)
return self.new_trial(trial)
def set_trial(self,
trial: Optional[Trial] = None,
force: bool = False,
**kwargs):
"""Set a new trial
Parameters
----------
trial: Optional[Trial]
project definition you can use to create or set the project
force: bool
by default once the trial is set it cannot be changed.
use force to override this behaviour.
kwargs: {uid, hash, revision}
arguments used to create a `Trial` object if no Trial object were provided.
You should specify `uid` or the pair `(hash, revision)`.
See :func:`~track.structure.Trial` for possible arguments
Returns
-------
returns a trial logger
"""
if self.trial is not None and not force:
info('Trial is already set, to override use force=True')
return self.logger
if trial is None:
uhash = kwargs.pop('hash', None)
uid = kwargs.pop('uid', None)
version = kwargs.pop('version', self.version())
trial = Trial(version=version, **kwargs)
if uhash is not None:
trial.hash = uhash
if uid is not None:
trial.uid = uid
try:
if trial.version is None:
trial.version = self.version()
trials = self.protocol.get_trial(trial)
if trials is None:
raise TrialDoesNotExist(
f'Trial (hash: {trial.hash}, v:{trial.version} rev: {trial.revision}) does not exist!'
)
self.trial = trials[0]
self.logger = TrialLogger(self.trial, self.protocol)
return self.logger
except IndexError:
raise TrialDoesNotExist(
f'cannot set trial (id: {trial.uid}, hash:{hash}) it does not exist'
)
def _new_trial(self, force=False, parameters=None, **kwargs):
"""Create a new trial if all the required arguments are satisfied.
To provide a better user experience if not all arguments are provided a delayed trials is created
that holds all the data provided and will create the trial once all arguments are ready.
Currently only `arguments` i.e the parameters of the experience is required. This is
because they are needed to compute the trial uid (which is a hash of the parameters).
If no project is set, the trial is inserted in a catch all project named `orphan`
Parameters
----------
force: bool
by default once the trial is set it cannot be changed.
use force to override this behaviour.
kwargs
See :func:`~track.structure.Trial` for possible arguments
Returns
-------
returns a trial
"""
if isinstance(parameters, Namespace):
parameters = dict(**vars(parameters))
if self.trial is not None and not is_delayed_call(
self.trial) and not force:
info(f'Trial is already set, to override use force=True')
return self.trial
# if arguments are not specified do not create the trial just yet
# wait for the user to be able to specify the parameters so we can have a meaningful hash
if parameters is None:
if is_delayed_call(self.trial):
raise RuntimeError('Trial needs parameters')
self.trial = delay_call(self._new_trial, **kwargs)
# return the logger with the delayed trial
return self.trial
# replace the trial or delayed trial by its actual value
if parameters or is_delayed_call(self.trial):
self.trial = self._make_trial(parameters=parameters, **kwargs)
assert self.trial is not None, f'Trial cant be None because {parameters} or {is_delayed_call(self.trial)}'
if self.project is None:
self.project = self.set_project(name='orphan')
assert self.project is not None, 'Project cant be None'
self.protocol.add_project_trial(self.project, self.trial)
if self.group is not None:
self.protocol.add_group_trial(self.group, self.trial)
return self.trial
def new_trial(self, trial: Trial, auto_increment=None):
try:
self.trials.insert_one(to_json(trial))
return trial
except DuplicateKeyError:
trial.revision += 1
info(f'Trial already exist increasing revision (rev: {trial.revision})')
return self.new_trial(trial)
def __init__(self, uri):
uri = parse_uri(uri)
self.username = uri.get('username')
self.password = uri.get('password')
self.security_layer = uri['query'].get('security_layer')
self.socket = open_socket(uri.get('address'),
int(uri.get('port')),
backend=self.security_layer)
self.token = self._authenticate(uri)
info(f'token: {self.token}')
def get_project(self, project: Project):
kwargs = dict()
kwargs['__rpc__'] = 'get_project'
kwargs['project'] = to_json(project)
info(kwargs)
send(self.socket, kwargs)
p = _check(recv(self.socket))
info(f'got reply {p}')
return p
def run_server(self):
info(f'Server listening to {self.address}:{self.port}')
self.sckt = listen_socket(self.address,
self.port,
backend=self.security_layer)
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.create_task(
asyncio.start_server(self.handle_client, sock=self.sckt))
loop.run_forever()
self.loop = loop
def set_project(self,
project: Optional[Project] = None,
force: bool = False,
get_only: bool = False,
**kwargs):
"""Set or create a new project
Parameters
----------
project: Optional[Project]
project definition you can use to create or set the project
force: bool
by default once the project is set it cannot be changed.
use force to override this behaviour.
get_only: bool
if true does not insert the project if missing.
default to false
kwargs
arguments used to create a `Project` object if no project object were provided
See :func:`~track.structure.Project` for possible arguments
Returns
-------
returns created project
"""
if self.project is not None and not force:
info('Project is already set, to override use force=True')
return self.project
if project is None:
project = Project(**kwargs)
assert project.name is not None, 'Project name cannot be none'
# does the project exist ?
self.project = self.protocol.get_project(project)
if self.project is not None:
return self.project
if get_only:
raise RuntimeError(
f'Project (name: {project.name}) was not found!')
self.project = self.protocol.new_project(project)
debug(f'set project to (project: {self.project.name})')
return self.project
def set_group(self,
group: Optional[TrialGroup] = None,
force: bool = False,
get_only: bool = False,
**kwargs):
"""Set or create a new group
Parameters
----------
group: Optional[TrialGroup]
project definition you can use to create or set the project
force: bool
by default once the trial group is set it cannot be changed.
use force to override this behaviour.
get_only: bool
if true does not insert the group if missing.
default to false
kwargs
arguments used to create a `TrialGroup` object if no TrialGroup object were provided.
See :func:`~track.structure.TrialGroup` for possible arguments
Returns
-------
returns created trial group
"""
if self.group is not None and not force:
info('Group is already set, to override use force=True')
return self.group
if group is None:
group = TrialGroup(**kwargs)
if group.project_id is None:
group.project_id = self.project.uid
self.group = self.protocol.get_trial_group(group)
if self.group is not None:
return self.group
if get_only:
raise RuntimeError(f'Group (name: {group.name}) was not found!')
self.group = self.protocol.new_trial_group(group)
return self.group
def __init__(self, location, addrs, join=None, clean_on_exit=True):
self.location = location
logs = f'{location}/logs'
temp = f'{location}/tmp'
external = f'{location}/extern'
store = location
os.makedirs(logs, exist_ok=True)
os.makedirs(temp, exist_ok=True)
os.makedirs(external, exist_ok=True)
self.location = location
self.addrs = addrs
self.bin = COCKROACH_BIN.get(os.name)
if self.bin is None:
raise RuntimeError('Your OS is not supported')
if not os.path.exists(self.bin):
info('Using system binary')
self.bin = 'cockroach'
else:
hash = COCKROACH_HASH.get(os.name)
if compute_version([self.bin]) != hash:
warning('Binary Hashes do not match')
self.arguments = [
'start', '--insecure', f'--listen-addr={addrs}',
f'--external-io-dir={external}', f'--store={store}',
f'--temp-dir={temp}', f'--log-dir={logs}',
f'--pid-file={location}/cockroach_pid'
]
if join is not None:
self.arguments.append(f'--join={join}')
self.manager: Manager = Manager()
self.properties = self.manager.dict()
self.properties['running'] = False
self.clean_on_exit = clean_on_exit
self._process: Process = None
self.cmd = None
def _look_for_configuration(file_name='track.config'):
config_file = None
paths = {
os.path.dirname(
os.path.realpath(__file__)), # location of the current file
os.getcwd(), # Current working directory
}
files = []
for path in paths:
file = f'{path}/{file_name}'
if os.path.exists(file):
files.append(file)
config_file = file
if len(files) > 1:
warning(f'found multiple configuration file: {", ".join(files)}')
elif config_file is not None:
info(f'loading configuration from {config_file}')
return config_file
def start_track_server(protocol, hostname, port, security_layer=None):
"""Start a track server inside a asyncio loop
Parameters
----------
protocol: str
URI that defines which backend to forward the request to
hostname: str
server host name
port: int
server port to listen to
security_layer: str
backend used for encryption (only AES is supported)
"""
security = ''
if security_layer is not None:
security = f'&security_layer={security_layer}'
server = SocketServer(f'socket://{hostname}:{port}?backend={protocol}' +
security)
_ = ServerSignalHandler(server)
try:
info('Running Server')
server.run_server()
except KeyboardInterrupt as e:
server.close()
raise e
except Exception as e:
server.close()
raise e
async def handle_client(self, reader, writer):
info('Client Connected')
running = True
count = 0
sleep_time = 0
cache = {}
info_proc = throttle_repeated(info, every=10)
while running:
request = await read(reader)
count += 1
if request is None:
time.sleep(0.01)
sleep_time += 0.01
if sleep_time > self.timeout:
info(
f'Client (user: {self.get_username(reader)}) is timing out'
)
await self.close_connection(writer)
self.authentication.pop(reader, None)
return None
continue
proc_name = request.pop('__rpc__', None)
info_proc(
f'Processing Request: {proc_name} for (user: {self.get_username(reader)})'
)
if proc_name is None:
error(f'Could not process message (rpc: {request})')
write(
writer, {
'status': 1,
'error': f'Could not process message (rpc: {request})'
})
continue
elif proc_name == 'authenticate':
request['reader'] = reader
self.exec(reader,
writer,
proc_name,
self.authenticate,
request,
cache=cache)
continue
elif not self.is_authenticated(reader):
error(
f'Client is not authenticated cannot execute (proc: {proc_name})'
)
write(
writer, {
'status':
1,
'error':
f'Client is not authenticated cannot execute (proc: {proc_name})'
})
continue
# Forward request to backend
attr = getattr(self.backend, proc_name)
if attr is None:
error(
f'{self.backend.__name__} does not implement (rpc: {proc_name})'
)
write(
writer, {
'status':
1,
'error':
f'{self.backend.__name__} does not implement (rpc: {proc_name})'
})
continue
self.exec(reader, writer, proc_name, attr, request, cache=cache)
sleep_time = 0
self.authentication.pop(reader, None)