def configure(host=None, port=None, key=None):
config = core.read_config('default')
if host is None and port is None and key is None:
return core.pretty_json(core.read_config())
# # print core.pretty_json(core.read_config())
else:
if host:
config['api']['host'] = host
if port:
config['api']['port'] = port
if key:
config['api']['key'] = key
core.write_config('default', config)
return core.pretty_json({'default':config})
def handle(config, conx, name, host, port, key, tag,
upload, file, env, path, obj, group, api):
if config:
print configure(host=host, port=port, key=key)
if conx:
config = core.read_config('default')
if api:
api_path = extensions['api'][alias]['path']
else:
api_path = 'corr.main.api'
api_module = core.extend_load(api_path)
if api_module.api_status(config=config):
## print "OK --- CoRR backend api[{0}:{1}] reached.".format(
# config['api']['host'], config['api']['port'])
print "OK"
else:
## print "KO --- could not reach CoRR backend api[{0}:{1}].".format(
# config['api']['host'], config['api']['port'])
print "KO"
if upload and file:
push_file(path=path, obj=obj, group=group)
if upload and env:
push_env(name=name, tag=tag, path=path)
def tag(name=None, api=None, elnk=None, ctsk=None):
## print ""
## print "Taging the investigation..."
config = core.read_config('default')
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
## print "An investigation with this name/tag has already been registered."
## print "Its name and tag are: [{0}|{1}]".format(registrations[investigations[0]]['name'], registrations[investigations[0]]['tags'])
stamp = core.formated_stamp()
tag = "{0}-tag-{1}".format(name, stamp)
registrations[investigations[0]]['tags'].append(tag)
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
api_module = core.extend_load(api)
api_response = api_module.project_update(
config=config,
project=registrations[investigations[0]]['project'],
description=None,
goals=None,
tags=registrations[investigations[0]]['tags'])
# # print api_response[1]
return [tag, api_response[1]]
else:
## print "No investigation with this name/tag has been registered."
return None
def push_file(path=None, obj=None, group=None):
config = core.read_config()
api_module = core.extend_load(api)
api_response = api_module.upload_file(config=config, path=path, obj=obj, group=group)
if api_response[0]:
## print "File uploaded."
return api_response[1]
else:
## print "File upload failed."
# # print api_response[1]
return api_response[1]
def push_env(name=None, tag=None, path=None):
config = core.read_config()
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name, tag=tag)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
if registrations[investigations[0]]['status']['value'] == 'watching':
## print "Error: Uploading new environment while watching is not possible. Unwatch first."
return registrations[investigations[0]]
else:
api_module = core.extend_load(api)
api_response = api_module.project_env_next(
config=config,
project=registrations[investigations[0]]['project'],
path=path)
if api_response[0]:
## print "Environment successfully pushed."
return api_response[1]
else:
## print "Error: Environment push failed."
# # print api_response[1]
return api_response[1]
def align(api=None, elnk=None, ctsk=None):
config = core.read_config('default')
registrations = core.read_reg('default')
# # print core.read_reg()
# # print registrations
api_module = core.extend_load(api)
api_response = api_module.project_all(config=config)
if api_response[0] == True:
projects_json = api_response[1]
# # print "--> Backend has {0} projects".format(
# projects_json['total_projects'])
projects = projects_json['projects']
for project in projects:
name = project['name']
tags = project['tags']
investigations = find_by(regs=registrations, name=name)
if len(investigations) == 0:
# # print "--> Aligning with remote project [{0}]...".format(name)
investigation = {}
stamp = core.formated_stamp()
investigation['name'] = name
investigation['tags'] = tags
investigation['status'] = {'value':'registered', 'stamp':stamp}
investigation['consistency'] = True
investigation['project'] = project['id']
investigation['history'] = []
investigation['watcher'] = None
registrations.append(investigation)
core.ensure_repo(investigation['name'])
# # print registrations
core.write_reg('default', registrations)
# # print "--> Done."
# # print core.read_reg()
return True
else:
## print "Registrations alignment failed."
# print api_response[1]
return False
def run(self):
found = False
duration = 0
project = None
config = None
while True:
running = False
self.info = self.link.record()
if self.info:
found = True
running = True
core.write_repo(self.name, self.info)
config = core.read_config('default')
registrations = core.read_reg('default')
# # print self.name
# # print self.tag
regs = self.clnk_module.find_by(
regs=registrations,
name=self.name,
tag=self.tag)
# # print "Record: {0}".format(self.record)
# # print registrations
# # print regs
if len(regs) > 0:
project = registrations[regs[0]]['project']
if project:
if self.link.updated:
# print self.tag
for data in self.info['io_files']:
if data[3] in ['r', 'r+', 'a+'] and data[0] not in self.ios['inputs']:
self.ios['inputs'].append(data[0])
if data[3] in ['w', 'w+', 'a', 'a+'] and data[0] not in self.ios['outputs']:
self.ios['outputs'].append(data[0])
try:
self.request['inputs'] = [
{
'input':data
} for data in self.info['io_files'] if data[3] in ['r', 'r+', 'a+']]
except:
self.request['inputs'] = []
try:
self.request['outputs'] = [
{
'output':data
} for data in self.info['io_files'] if data[3] in ['w', 'w+', 'a', 'a+']]
except:
self.request['outputs'] = []
try:
self.request['dependencies'] = [
{
'dependency':data
} for data in self.info['libraries']]
except:
self.request['dependencies'] = []
self.request['status'] = self.info['status']
self.request['extend'] = {}
self.request['extend']['children'] = self.info['children']
self.request['extend']['network'] = self.info['network']
self.request['extend']['cp_purcentage'] = self.info['cp_purcentage']
self.request['extend']['mem_purcentage'] = self.info['mem_purcentage']
self.request['extend']['threads'] = self.info['threads']
api_response = self.api_module.record_update(
config=config,
record=self.record,
request=self.request)
self.records.append(self.request)
# print "Record updated"
if not api_response[0]:
# # print "Error: Watcher recording create process failed."
# # print api_response[1]
pass
else:
self.request['label'] = self.tag
self.request['tags'] = [self.tag]
# print self.tag
self.request['system'] = self.info['computer']
for data in self.info['io_files']:
if data[3] in ['r', 'r+', 'a+'] and data[0] not in self.ios['inputs']:
self.ios['inputs'].append(data[0])
if data[3] in ['w', 'w+', 'a', 'a+'] and data[0] not in self.ios['outputs']:
self.ios['outputs'].append(data[0])
try:
self.request['inputs'] = [
{
'input':data
} for data in self.info['io_files'] if data[3] in ['r', 'r+', 'a+']]
except:
self.request['inputs'] = []
try:
self.request['outputs'] = [
{
'output':data
} for data in self.info['io_files'] if data[3] in ['w', 'w+', 'a', 'a+']]
except:
self.request['outputs'] = []
try:
self.request['dependencies'] = [
{
'dependency':data
} for data in self.info['libraries']]
except:
self.request['dependencies'] = []
self.request['status'] = self.info['status']
self.request['access'] = 'private'
self.request['execution'] = {
'cmdline':self.info['cmdline'],
'executable':self.info['executable'],
'path':self.info['path'],
'name':self.info['name']}
self.request['extend'] = {}
self.request['extend']['children'] = self.info['children']
self.request['extend']['network'] = self.info['network']
self.request['extend']['cp_purcentage'] = self.info['cp_purcentage']
self.request['extend']['mem_purcentage'] = self.info['mem_purcentage']
self.request['extend']['threads'] = self.info['threads']
api_response = self.api_module.record_create(
config=config,
project=project,
request=self.request)
# print "Record created"
self.records.append(self.request)
if api_response[0]:
self.record = api_response[1]['head']['id']
else:
# # print "Error: Watcher recording create process failed."
# print api_response[1]
# pass
pass
if self.info['status'] in ['killed', 'terminated', 'stoped']: #'sleeping',
running = False
else:
# print "Error: Unable to find the project."
pass
else:
# print "No info!!!"
pass
if found and not running:
break
sleep(self.refresh)
duration += self.refresh
if duration >= self.timeout:
break
self.sync_io(config)
return self.records
def sync(name=None, tag=None, force=False, api=None, elnk=None, ctsk=None):
config = core.read_config()
registrations = core.read_reg('default')
if name is None and tag is None:
## print "Syncing all inconsistent registrations..."
for idx, reg in enumerate(registrations):
if not reg['consistency'] or force:
## print "Syncing the registration [{0}]...".format(reg['name'])
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=reg['name'],
description='no description provided.',
goals='no goals set.',
tags=reg['tags'])
if api_response[0] == True:
reg['consistency'] = True
reg['project'] = api_response[1]['id']
core.ensure_repo(reg['name'])
registrations[idx] = reg
core.write_reg('default', registrations)
## print "--> This associated project metadata is now consistent."
# return api_response[1]
# else:
## print "--> Consistency alignment between registration\
#and project metada failed. Please check connectivity\
#and try to sync the investigation again later."
## print api_response[1]
# return api_response[1]
# else:
## print "Registration [{0}] is already consistent.".format(reg['name'])
return registrations
else:
## print "Syncing the registration..."
investigations = find_by(regs=registrations, name=name, tag=tag)
if len(investigations) > 0 and (not registrations[investigations[0]]['consistency'] or force):
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=registrations[investigations[0]]['name'],
description='no description provided.',
goals='no goals set.',
tags=[registrations[investigations[0]]['tags']])
if api_response[0] == True:
registrations[investigations[0]]['consistency'] = True
registrations[investigations[0]]['project'] = api_response[1]['id']
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now consistent."
return api_response[1]
else:
## print "--> Consistency alignment between registration\
# and project metada failed. Please check connectivity\
# and try to sync the investigation again later."
## print api_response[1]
return None
else:
if len(investigations) > 0:
## print "The associated project metadata is already consistent."
return registrations[investigations[0]]
else:
## print "Error: No registration with this name."
return None
def register(name=None, api=None, elnk=None, ctsk=None):
## print ""
## print "Registering the investigation..."
config = core.read_config('default')
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
## print "An investigation with this name/tag has already been registered."
## print "Its name and tag are: [{0}|{1}]".format(registrations[investigations[0]]['name'], registrations[investigations[0]]['tags'])
return registrations[investigations[0]]
else:
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] == 'unregistered':
stamp = core.formated_stamp()
registrations[investigations[0]]['history'].append(registrations[investigations[0]]['status'])
registrations[investigations[0]]['status'] = {'value':'registered', 'stamp':stamp}
registrations[investigations[0]]['consistency'] = False
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "An investigation with this name/tag was\
# unregistered and is now registered back again."
if not registrations[investigations[0]]['consistency']:
api_module = core.extend_load(api)
api_response = api_module.project_create(config=config,
name=registrations[investigations[0]]['name'],
description='no description provided.',
goals='no goals set.',
tags=registrations[investigations[0]]['tags'])
if api_response[0] == True:
registrations[investigations[0]]['consistency'] = True
registrations[investigations[0]]['project'] = api_response[1]['id']
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now consistent."
return registrations[investigations[0]]
else:
## print "Consistency alignment between registration and project metadat failed. \
#Please check connectivity and try to sync the investigation again later."
# print api_response[1]
return registrations[investigations[0]]
else:
return None
else:
investigation = {}
stamp = core.formated_stamp()
investigation['tags'] = ["%s-tag-%s"%(name, stamp)]
if name != None:
investigation['name'] = name
else:
investigation['name'] = "name_%s"%stamp
investigation['status'] = {'value':'registered', 'stamp':stamp}
investigation['consistency'] = False
investigation['history'] = []
investigation['watcher'] = None
exists = False
if not investigation['name'].isalnum():
## print "Registration failed. name has to be alphanumberial."
return None
else:
for reg in registrations:
if reg['name'] == investigation['name']:
exists = True
break
if not exists:
registrations.append(investigation)
core.ensure_repo(investigation['name'])
core.write_reg('default', registrations)
## print "Registration produced tag: {0}".format(investigation['tags'])
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=investigation['name'],
description='no description provided.',
goals='no goals set.',
tags=investigation['tags'])
if api_response[0] == True:
investigation['consistency'] = True
investigation['project'] = api_response[1]['id']
core.ensure_repo(investigation['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now\
#consistent with the registration."
return "Consistent"
else:
## print "Consistency alignment between registration\
#and project metadat failed. Please check connectivity\
#and try to sync the investigation later."
# print api_response[1]
return investigation
else:
## print "Registration failed. A registration already exists\
#with the name: {0}".format(investigation['name'])
return investigation