def test_watch(self):
clnk_module = core.extend_load("corr.main.coreLink")
elnk_module = core.extend_load("corr.main.execLink")
api_module = core.extend_load("corr.main.api")
tag = None
align_resp = coreLink.align(api="corr.main.api", elnk="corr.main.coreLink")
assert align_resp == True
reg_resp = coreLink.register(name="execution", api="corr.main.api", elnk="corr.main.coreLink")
assert reg_resp != None
tag_resp = coreLink.tag(name="execution", api="corr.main.api", elnk="corr.main.coreLink")
assert tag_resp != None
tag = tag_resp[0]
# check what is the latest record now
corr_path = imp.find_module("corr")[1]
task_cmd = []
task_cmd.append("python")
task_cmd.append("{0}/data/execution.py".format(corr_path))
task_cmd.append(tag)
process = subprocess.Popen(task_cmd)
tsk = CoRRTask(
name="execution",
tag=tag,
clnk_module=clnk_module,
api_module=api_module,
elnk_module=elnk_module,
timeout=60,
)
records = tsk.run()
assert len(records) > 0
def test_extend_load(self):
coreLink_module = core.extend_load('corr.main.coreLink')
assert coreLink_module.whois() == "CoreLink"
corr_path = imp.find_module('corr')[1]
extend_coreLink_module = core.extend_load('{0}/data/extend_CoreLink.py'.format(corr_path))
assert extend_coreLink_module.whois() == "extend_CoreLink"
def handle(config, conx, name, host, port, key, tag,
upload, file, env, path, obj, group, api):
if config:
print configure(host=host, port=port, key=key)
if conx:
config = core.read_config('default')
if api:
api_path = extensions['api'][alias]['path']
else:
api_path = 'corr.main.api'
api_module = core.extend_load(api_path)
if api_module.api_status(config=config):
## print "OK --- CoRR backend api[{0}:{1}] reached.".format(
# config['api']['host'], config['api']['port'])
print "OK"
else:
## print "KO --- could not reach CoRR backend api[{0}:{1}].".format(
# config['api']['host'], config['api']['port'])
print "KO"
if upload and file:
push_file(path=path, obj=obj, group=group)
if upload and env:
push_env(name=name, tag=tag, path=path)
def tag(name=None, api=None, elnk=None, ctsk=None):
## print ""
## print "Taging the investigation..."
config = core.read_config('default')
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
## print "An investigation with this name/tag has already been registered."
## print "Its name and tag are: [{0}|{1}]".format(registrations[investigations[0]]['name'], registrations[investigations[0]]['tags'])
stamp = core.formated_stamp()
tag = "{0}-tag-{1}".format(name, stamp)
registrations[investigations[0]]['tags'].append(tag)
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
api_module = core.extend_load(api)
api_response = api_module.project_update(
config=config,
project=registrations[investigations[0]]['project'],
description=None,
goals=None,
tags=registrations[investigations[0]]['tags'])
# # print api_response[1]
return [tag, api_response[1]]
else:
## print "No investigation with this name/tag has been registered."
return None
def push_file(path=None, obj=None, group=None):
config = core.read_config()
api_module = core.extend_load(api)
api_response = api_module.upload_file(config=config, path=path, obj=obj, group=group)
if api_response[0]:
## print "File uploaded."
return api_response[1]
else:
## print "File upload failed."
# # print api_response[1]
return api_response[1]
def handle(name, tag, delay, aid, origin, elnk):
delay_value = 10
aid_value = None
origin_value = None
# # print "Name: {0}".format(name)
# # print "tag: {0}".format(tag)
stamp = str(datetime.datetime.now())
if delay:
delay_value = int(delay)
if name and tag and api and elnk and clnk:
clnk_module = core.extend_load(clnk)
elnk_module = core.extend_load(elnk)
api_module = core.extend_load(api)
task = CoRRTask(name=name, tag=tag, clnk_module=clnk_module, api_module=api_module, elnk_module=elnk_module)
# task.run()
try:
# # print "Loading watcher: {0}".format(task.tag)
with daemon.DaemonContext():
task.run()
except:
pass
def paths(clnk=None, api=None, elnk=None, ctsk=None):
extensions = core.read_extend()
if clnk:
clnk_path = extensions['coreLink'][clnk]['path']
else:
clnk_path = 'corr.main.coreLink'
clnk_module = core.extend_load(clnk_path)
if api:
api_path = extensions['api'][api]['path']
else:
api_path = 'corr.main.api'
if elnk:
elnk_path = extensions['execLink'][elnk]['path']
else:
elnk_path = 'corr.main.execLink'
if ctsk:
ctsk_path = extensions['corrTask'][ctsk]['path']
else:
ctsk_path = 'corr.main.corrTask'
return [clnk_module, api_path, elnk_path, ctsk_path]
def push_env(name=None, tag=None, path=None):
config = core.read_config()
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name, tag=tag)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
if registrations[investigations[0]]['status']['value'] == 'watching':
## print "Error: Uploading new environment while watching is not possible. Unwatch first."
return registrations[investigations[0]]
else:
api_module = core.extend_load(api)
api_response = api_module.project_env_next(
config=config,
project=registrations[investigations[0]]['project'],
path=path)
if api_response[0]:
## print "Environment successfully pushed."
return api_response[1]
else:
## print "Error: Environment push failed."
# # print api_response[1]
return api_response[1]
def align(api=None, elnk=None, ctsk=None):
config = core.read_config('default')
registrations = core.read_reg('default')
# # print core.read_reg()
# # print registrations
api_module = core.extend_load(api)
api_response = api_module.project_all(config=config)
if api_response[0] == True:
projects_json = api_response[1]
# # print "--> Backend has {0} projects".format(
# projects_json['total_projects'])
projects = projects_json['projects']
for project in projects:
name = project['name']
tags = project['tags']
investigations = find_by(regs=registrations, name=name)
if len(investigations) == 0:
# # print "--> Aligning with remote project [{0}]...".format(name)
investigation = {}
stamp = core.formated_stamp()
investigation['name'] = name
investigation['tags'] = tags
investigation['status'] = {'value':'registered', 'stamp':stamp}
investigation['consistency'] = True
investigation['project'] = project['id']
investigation['history'] = []
investigation['watcher'] = None
registrations.append(investigation)
core.ensure_repo(investigation['name'])
# # print registrations
core.write_reg('default', registrations)
# # print "--> Done."
# # print core.read_reg()
return True
else:
## print "Registrations alignment failed."
# print api_response[1]
return False
def handle(config, conx, register, sync, align, unregister,
name, host, port, key, watch, unwatch, list, show, tag,
newtag, upload, file, env, path, obj, group, force, extend,
add, delete, all, alias, impl, clnk, api, elnk, ctsk):
def paths(clnk=None, api=None, elnk=None, ctsk=None):
extensions = core.read_extend()
if clnk:
clnk_path = extensions['coreLink'][clnk]['path']
else:
clnk_path = 'corr.main.coreLink'
clnk_module = core.extend_load(clnk_path)
if api:
api_path = extensions['api'][api]['path']
else:
api_path = 'corr.main.api'
if elnk:
elnk_path = extensions['execLink'][elnk]['path']
else:
elnk_path = 'corr.main.execLink'
if ctsk:
ctsk_path = extensions['corrTask'][ctsk]['path']
else:
ctsk_path = 'corr.main.corrTask'
return [clnk_module, api_path, elnk_path, ctsk_path]
if extend:
extensions = core.read_extend()
if add:
if clnk:
core.extension_add(extensions, 'coreLink')
elif api:
core.extension_add(extensions, 'api')
elif elnk:
core.extension_add(extensions, 'execLink')
elif ctsk:
core.extension_add(extensions, 'corrTask')
else:
# # print "Unknown extension group -- interoerate not supported yet."
pass
if delete:
if clnk:
core.extension_delete(extensions, 'coreLink', alias)
elif api:
core.extension_delete(extensions, 'api', alias)
elif elnk:
core.extension_delete(extensions, 'execLink', alias)
elif ctsk:
core.extension_delete(extensions, 'corrTask', alias)
if all:
core.extension_all(extensions)
elif newtag and name:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].tag(name=name, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif list:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
regs = extends[0].list(api=extends[1], elnk=extends[2], ctsk=extends[3])
for reg in regs:
print "{0}\t{1}\t{2}\t{3}".format(
reg['name'], str(reg['tags']), reg['status']['stamp'],
reg['status']['value'])
elif show and (name or tag):
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
print extends[0].show(name=name, tag=tag, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif align:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].align(api=extends[1], elnk=extends[2], ctsk=extends[3])
elif register and name:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].register(name=name, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif sync:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].sync(name=name, tag=tag, force=force, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif unregister:
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].unregister(name=name, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif watch and (name or tag):
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].watch(name=name, tag=tag, api=extends[1], elnk=extends[2], ctsk=extends[3])
elif unwatch and (name or tag):
extends = paths(clnk=clnk, api=api, elnk=elnk, ctsk=ctsk)
extends[0].unwatch(name=name, tag=tag, api=extends[1], elnk=extends[2], ctsk=extends[3])
else:
if clnk:
extensions = core.read_extend()
clnk_path = extensions['coreLink'][alias]['path']
else:
clnk_path = 'corr.main.coreLink'
clnk_module = core.extend_load(clnk_path)
clnk_module.handle()
def sync(name=None, tag=None, force=False, api=None, elnk=None, ctsk=None):
config = core.read_config()
registrations = core.read_reg('default')
if name is None and tag is None:
## print "Syncing all inconsistent registrations..."
for idx, reg in enumerate(registrations):
if not reg['consistency'] or force:
## print "Syncing the registration [{0}]...".format(reg['name'])
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=reg['name'],
description='no description provided.',
goals='no goals set.',
tags=reg['tags'])
if api_response[0] == True:
reg['consistency'] = True
reg['project'] = api_response[1]['id']
core.ensure_repo(reg['name'])
registrations[idx] = reg
core.write_reg('default', registrations)
## print "--> This associated project metadata is now consistent."
# return api_response[1]
# else:
## print "--> Consistency alignment between registration\
#and project metada failed. Please check connectivity\
#and try to sync the investigation again later."
## print api_response[1]
# return api_response[1]
# else:
## print "Registration [{0}] is already consistent.".format(reg['name'])
return registrations
else:
## print "Syncing the registration..."
investigations = find_by(regs=registrations, name=name, tag=tag)
if len(investigations) > 0 and (not registrations[investigations[0]]['consistency'] or force):
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=registrations[investigations[0]]['name'],
description='no description provided.',
goals='no goals set.',
tags=[registrations[investigations[0]]['tags']])
if api_response[0] == True:
registrations[investigations[0]]['consistency'] = True
registrations[investigations[0]]['project'] = api_response[1]['id']
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now consistent."
return api_response[1]
else:
## print "--> Consistency alignment between registration\
# and project metada failed. Please check connectivity\
# and try to sync the investigation again later."
## print api_response[1]
return None
else:
if len(investigations) > 0:
## print "The associated project metadata is already consistent."
return registrations[investigations[0]]
else:
## print "Error: No registration with this name."
return None
def register(name=None, api=None, elnk=None, ctsk=None):
## print ""
## print "Registering the investigation..."
config = core.read_config('default')
registrations = core.read_reg('default')
investigations = find_by(regs=registrations, name=name)
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] != 'unregistered':
## print "An investigation with this name/tag has already been registered."
## print "Its name and tag are: [{0}|{1}]".format(registrations[investigations[0]]['name'], registrations[investigations[0]]['tags'])
return registrations[investigations[0]]
else:
if len(investigations) > 0 and registrations[investigations[0]]['status']['value'] == 'unregistered':
stamp = core.formated_stamp()
registrations[investigations[0]]['history'].append(registrations[investigations[0]]['status'])
registrations[investigations[0]]['status'] = {'value':'registered', 'stamp':stamp}
registrations[investigations[0]]['consistency'] = False
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "An investigation with this name/tag was\
# unregistered and is now registered back again."
if not registrations[investigations[0]]['consistency']:
api_module = core.extend_load(api)
api_response = api_module.project_create(config=config,
name=registrations[investigations[0]]['name'],
description='no description provided.',
goals='no goals set.',
tags=registrations[investigations[0]]['tags'])
if api_response[0] == True:
registrations[investigations[0]]['consistency'] = True
registrations[investigations[0]]['project'] = api_response[1]['id']
core.ensure_repo(registrations[investigations[0]]['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now consistent."
return registrations[investigations[0]]
else:
## print "Consistency alignment between registration and project metadat failed. \
#Please check connectivity and try to sync the investigation again later."
# print api_response[1]
return registrations[investigations[0]]
else:
return None
else:
investigation = {}
stamp = core.formated_stamp()
investigation['tags'] = ["%s-tag-%s"%(name, stamp)]
if name != None:
investigation['name'] = name
else:
investigation['name'] = "name_%s"%stamp
investigation['status'] = {'value':'registered', 'stamp':stamp}
investigation['consistency'] = False
investigation['history'] = []
investigation['watcher'] = None
exists = False
if not investigation['name'].isalnum():
## print "Registration failed. name has to be alphanumberial."
return None
else:
for reg in registrations:
if reg['name'] == investigation['name']:
exists = True
break
if not exists:
registrations.append(investigation)
core.ensure_repo(investigation['name'])
core.write_reg('default', registrations)
## print "Registration produced tag: {0}".format(investigation['tags'])
api_module = core.extend_load(api)
api_response = api_module.project_create(
config=config,
name=investigation['name'],
description='no description provided.',
goals='no goals set.',
tags=investigation['tags'])
if api_response[0] == True:
investigation['consistency'] = True
investigation['project'] = api_response[1]['id']
core.ensure_repo(investigation['name'])
core.write_reg('default', registrations)
## print "The associated project metadata is now\
#consistent with the registration."
return "Consistent"
else:
## print "Consistency alignment between registration\
#and project metadat failed. Please check connectivity\
#and try to sync the investigation later."
# print api_response[1]
return investigation
else:
## print "Registration failed. A registration already exists\
#with the name: {0}".format(investigation['name'])
return investigation
