async def register(request):
data = await request.json()
etcd_client.write(
"tools/{}".format(data['name']), '{{"author":"{}",\
"image":"{}",\
"data_repo":"{}",\
"code_repo":"{}",\
"artefact":"{}"}}'.format(data['author'], data['image'], data['data_repo'],
data['code_repo'], data['artefact']))
return web.json_response(etcd_client.get("tools/{}".format(data['name'])))
def temp_retrieve(name, nodes):
for node in nodes:
try:
value = get("/data/" + name + '/' + node)
except:
print("No such entry")
return "This name does not correspond to an entry"
try:
#git.Repo.clone_from(value, importdir + "/" + name)
subprocess.run(
f"git clone {value} {importdir + '/' + 'tmp' + '/' + name + '/' + node}",
shell=True)
except:
print("Error cloning data")
collect.consensus(importdir + '/' + 'tmp' + '/' + name)
def retrieve(name, node):
try:
value = get("/data/" + name + '/' + node)
except:
print("No such entry")
return "This name does not correspond to an entry"
try:
#git.Repo.clone_from(value, importdir + "/" + name)
subprocess.run(
f"git clone {value} {importdir + '/' + name + '/' + node}",
shell=True)
if not config.has_option('DATA_REPOS', name):
print("Updating config")
logging.info("updating config")
if not config.has_section('DATA_REPOS'):
config.add_section('DATA_REPOS')
config.set('DATA_REPOS', name, importdir + "/" + name)
with open('config.ini', 'w') as f:
config.write(f)
return "Cloned: {} to {}".format(value, importdir + "/" + name)
except:
print("Error cloning data, trying to pull")
logging.warning("Error cloning data, trying to pull")
try:
repo = git.Repo(importdir + "/" + name)
o = repo.remotes.origin
o.pull()
if not config.has_option('DATA_REPOS', name):
print("Updating config")
if not config.has_section('DATA_REPOS'):
config.add_section('DATA_REPOS')
config.set('DATA_REPOS', name, importdir + "/" + name)
with open('config.ini', 'w') as f:
config.write(f)
return "Pulled: {} to {}".format(value, importdir + "/" + name)
except:
print("Error pulling data.")
logging.error("Error retrieving data.")
return "Error pulling data."
def sync(data):
# Use new scheduler
response = {}
command = []
env = {}
renku = False
blob = get('tools/{}'.format(data['name']))
payload = json.loads(blob)
if 'dataset' in data and 'node' in data:
new_repo = get(f"data/{data['dataset']}/{data['node']}")
payload['data_repo'] = new_repo
tool = payload['image']
print("Tool invoked: " + tool)
response['tool'] = tool
dataset = importdir + "/" + data['name']
print("Data directory: " + dataset)
response['datadir'] = dataset
if 'env' in data:
env = data['env']
if 'command' in data:
command = data['command']
if 'renku' in data:
renku = True
#payload['data_repo'] = data['renku']
# Check if dataset has been cloned already
if not config.has_option('DATA_REPOS', data['name']):
# Clone dataset
print("Cloning dataset from: " + payload['data_repo'] + " to: " +
dataset)
response['dataset'] = payload['data_repo']
print("Updating config")
if not config.has_section('DATA_REPOS'):
config.add_section('DATA_REPOS')
config.set('DATA_REPOS', data['name'], dataset)
with open('config.ini', 'w') as f:
config.write(f)
try:
#git.Repo.clone_from(payload['data_repo'], dataset)
subprocess.run(f"git clone {payload['data_repo']} {dataset}",
shell=True)
except:
print("Error cloning data")
if data['cron']:
freq = data['frequency']
json_out = {
"container": tool,
"tool": data['name'],
"dataset": config['DATA_REPOS'][data['name']],
"cron": data['cron'],
"freq": freq,
"command": command,
"env": env,
"renku": renku
}
else:
json_out = {
"container": tool,
"tool": data['name'],
"dataset": config['DATA_REPOS'][data['name']],
"cron": data['cron'],
"command": command,
"env": env,
"renku": renku
}
print("Message to scheduler: " + json.dumps(json_out))
response['message'] = json.dumps(json_out)
response['scheduler_output'] = schedule.schedule_run(json_out)
return response
async def register(request):
data = await request.json()
etcd_client.write("data/{}".format(data['name']), data['url'])
return web.json_response(etcd_client.get("data/{}".format(data['name'])))
async def write(request):
data = await request.json()
return web.json_response(json.loads(etcd_client.get(data['key'])))
async def write(request):
data = await request.json()
etcd_client.write('raw/' + data['key'], data['value'], ephemeral=True)
return web.json_response(etcd_client.get('raw/' + data['key']))
async def datasets(request):
return web.json_response(
etcd_client.get('data/' + request.match_info['dataset']))
async def regtools(request):
return web.json_response(
etcd_client.get('tools/' + request.match_info['tool']))
async def datasets(request):
return web.json_response(
etcd_client.get(
f"data/{request.match_info['dataset']}/{request.match_info['node']}"
))
