def load_database(json_name):
global _print_warning_once
if json_name is None:
return LocalStorage()
if not os.path.exists(json_name):
if json_name not in _print_warning_once:
warning(f'Local Storage was not found at {json_name}')
_print_warning_once.add(json_name)
return LocalStorage(target_file=json_name)
with open(json_name, 'r') as file:
objects = json.load(file)
db = dict()
projects = set()
project_names = dict()
groups = set()
group_names = dict()
trials = set()
trial_names = dict()
for item in objects:
obj = from_json(item)
if obj.uid in db:
raise RuntimeError('Should be unreachable!')
db[obj.uid] = obj
if isinstance(obj, Project):
projects.add(obj.uid)
if obj.name in project_names:
error('Non unique project names are not supported')
if obj.name is not None:
project_names[obj.name] = obj.uid
for trial in obj.trials:
db[trial.uid] = trial
trials.add(trial.uid)
for group in obj.groups:
db[group.uid] = group
groups.add(group.uid)
elif isinstance(obj, Trial):
trials.add(obj.uid)
if obj.name is not None:
trial_names[obj.name] = obj.uid
elif isinstance(obj, TrialGroup):
groups.add(obj.uid)
if obj.name is not None:
group_names[obj.name] = obj.uid
return LocalStorage(json_name, db, projects, groups, trials, project_names, group_names, trial_names)
def new_project(self, project: Project):
debug(f'create new (project: {project.name})')
if project.uid in self.storage.objects:
error(f'Cannot insert project; (uid: {project.uid}) already exists!')
return self.get_project(project)
self.storage.objects[project.uid] = project
self.storage.project_names[project.name] = project.uid
self.storage.projects.add(project.uid)
return project
def exec(self, reader, writer, proc_name, proc, args, cache=None):
try:
new_args = self.process_args(args)
answer = proc(**new_args)
write(writer, {'status': 0, 'return': to_json(answer)})
# info(f'returned: {answer}')
except Exception as e:
error(f'An exception occurred while processing (rpc: {proc_name}) '
f'for user {self.get_username(reader)[0]}')
error(traceback.format_exc())
write(writer, {'status': 1, 'error': str(e)})
def new_trial_group(self, group: TrialGroup):
if group.uid in self.storage.objects:
error(f'Cannot insert group; (uid: {group.uid}) already exists!')
return
project = self.storage.objects.get(group.project_id)
if self.strict:
assert project is not None, 'Cannot create a group without an associated project'
project.groups.add(group)
self.storage.objects[group.uid] = group
self.storage.groups.add(group.uid)
self.storage.group_names[group.name] = group.uid
return group
def _start(self, properties):
kwargs = dict(args=' '.join([self.bin] + self.arguments),
stdout=subprocess.PIPE,
bufsize=1,
stderr=subprocess.STDOUT)
self.cmd = kwargs['args']
with subprocess.Popen(**kwargs, shell=True) as proc:
try:
properties['running'] = True
properties['pid'] = proc.pid
while properties['running']:
if proc.poll() is None:
line = proc.stdout.readline().decode('utf-8')
if line:
self.parse(properties, line)
else:
properties['running'] = False
properties['exit'] = proc.returncode
except Exception:
error(traceback.format_exc())
def _check(result):
if result['status'] != 0:
error(f'RPC failed with error {result["error"]}')
raise RPCCallFailure(result['error'], result.get('trace'))
return from_json(result['return'])
async def handle_client(self, reader, writer):
info('Client Connected')
running = True
count = 0
sleep_time = 0
cache = {}
info_proc = throttle_repeated(info, every=10)
while running:
request = await read(reader)
count += 1
if request is None:
time.sleep(0.01)
sleep_time += 0.01
if sleep_time > self.timeout:
info(
f'Client (user: {self.get_username(reader)}) is timing out'
)
await self.close_connection(writer)
self.authentication.pop(reader, None)
return None
continue
proc_name = request.pop('__rpc__', None)
info_proc(
f'Processing Request: {proc_name} for (user: {self.get_username(reader)})'
)
if proc_name is None:
error(f'Could not process message (rpc: {request})')
write(
writer, {
'status': 1,
'error': f'Could not process message (rpc: {request})'
})
continue
elif proc_name == 'authenticate':
request['reader'] = reader
self.exec(reader,
writer,
proc_name,
self.authenticate,
request,
cache=cache)
continue
elif not self.is_authenticated(reader):
error(
f'Client is not authenticated cannot execute (proc: {proc_name})'
)
write(
writer, {
'status':
1,
'error':
f'Client is not authenticated cannot execute (proc: {proc_name})'
})
continue
# Forward request to backend
attr = getattr(self.backend, proc_name)
if attr is None:
error(
f'{self.backend.__name__} does not implement (rpc: {proc_name})'
)
write(
writer, {
'status':
1,
'error':
f'{self.backend.__name__} does not implement (rpc: {proc_name})'
})
continue
self.exec(reader, writer, proc_name, attr, request, cache=cache)
sleep_time = 0
self.authentication.pop(reader, None)