class DataObject(object):
def __init__(self, token=None):
goo_server = settings.GOO_SERVER_URI
self.proxy_token = settings.GOO_SERVER_TOKEN
self.user_token = token
debug = sys.stderr if settings.DEBUG else False
self.server = API(goo_server, debug=debug)
def load(self, oid):
obj = self.server.dataobjects(oid).get(token=self.user_token,
proxy_token=self.proxy_token)
self.sha1 = obj['sha1']
self.data_proxy_servers = obj['data_proxy_servers']
self.name = obj['name']
self.size = obj['size']
self.oid = oid
return self
def file(self):
""" Must call load(oid) load before """
return storage.download(self.sha1)
def save(self, name, req_file, public=False):
self.name = name
self.public = public
# calculate SHA256
digest = hashlib.sha1()
with open(req_file, 'rb') as f:
for chunk in iter(lambda: f.read(128 * digest.block_size), b''):
digest.update(chunk)
self.sha1 = digest.hexdigest()
self.size = os.path.getsize(req_file)
storage.upload(req_file, self.sha1)
values = {
"name": self.name,
"size": self.size,
"sha1": self.sha1,
"public": self.public
}
response = self.server.dataobjects.post(values,
token=self.user_token,
proxy_token=self.proxy_token)
self.oid = response["id"]
self.resource_uri = response["resource_uri"]
def delete(self):
""" Must call load(oid) load before """
# content data deletion
storage.delete(self.sha1)
# metadata deletion
self.server.dataobjects(self.oid).delete(token=self.user_token,
proxy_token=self.proxy_token)
def delete_object(self, args):
object_id = args.object_id
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
dps_api.dataobjects(object_id).delete(token=self.token)
print "Object %s delete with success" % object_id
def delete_object(self, args):
object_id = args.object_id
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
dps_api.dataobjects(object_id).delete(token=self.token)
print "Object %s delete with success" % object_id
class DataObject(object):
def __init__(self, token=None):
goo_server = settings.GOO_SERVER_URI
self.proxy_token = settings.GOO_SERVER_TOKEN
self.user_token = token
debug = sys.stderr if settings.DEBUG else False
self.server = API(goo_server, debug=debug)
def load(self, oid):
obj = self.server.dataobjects(oid).get(token=self.user_token,
proxy_token=self.proxy_token)
self.sha1 = obj['sha1']
self.data_proxy_servers = obj['data_proxy_servers']
self.name = obj['name']
self.size = obj['size']
self.oid = oid
return self
def file(self):
""" Must call load(oid) load before """
return storage.download(self.sha1)
def save(self, name, req_file, public=False):
self.name = name
self.public = public
# calculate SHA256
digest = hashlib.sha1()
with open(req_file,'rb') as f:
for chunk in iter(lambda: f.read(128*digest.block_size), b''):
digest.update(chunk)
self.sha1 = digest.hexdigest()
self.size = os.path.getsize(req_file)
storage.upload(req_file, self.sha1)
values = {"name": self.name,
"size": self.size,
"sha1": self.sha1,
"public": self.public}
response = self.server.dataobjects.post(values, token=self.user_token,
proxy_token=self.proxy_token)
self.oid = response["id"]
self.resource_uri = response["resource_uri"]
def delete(self):
""" Must call load(oid) load before """
# content data deletion
storage.delete(self.sha1)
# metadata deletion
self.server.dataobjects(self.oid).delete(token=self.user_token,
proxy_token=self.proxy_token)
def download_object(self, args):
object_id = args.object_id
# Get object info
obj = self.api.dataobjects(object_id).get(token=self.token)
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
data = dps_api.dataobjects(object_id).get(token=self.token)
f = open(obj['name'], "w+")
f.write(data)
f.close()
print "Finish"
def download_object(self, args):
object_id = args.object_id
# Get object info
obj = self.api.dataobjects(object_id).get(token=self.token)
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
data = dps_api.dataobjects(object_id).get(token=self.token)
f = open(obj['name'], "w+")
f.write(data)
f.close()
print "Finish"
def setUp(self):
super(DataObjectResourceTest, self).setUp()
self.endpoint = 'http://localhost:8001/api/v1/dataproxy/'
self.format = 'json'
self.filename = '/bin/true'
self.object_name = 'true'
debug = sys.stderr if settings.DEBUG else False
self.server = API(self.endpoint, debug=debug)
# Create one token
self.token = '2bae7fa7-5810-4664-8264-725fa3a714b1'
self.bad_token = 'not-a-valid-token'
def create_object(self, args):
inputs = args.inputs
object_name = "%s.zip" % args.name
fd, filepath = tempfile.mkstemp('.zip', self._slugfy(args.name))
# Create package
print 'Creating object package...'
for f in inputs:
zf = zipfile.ZipFile(filepath, mode='a')
try:
print ' adding %s' % f
zf.write(f)
finally:
zf.close()
server_uri = self._get_data_proxy()
f = open(filepath, 'rb')
object_data = {'name': object_name, 'file': f}
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (object_name, object_id)
# Force tempfile to be removed
os.unlink(filepath)
return result['resource_uri']
def setUp(self):
super(DataObjectResourceTest, self).setUp()
self.endpoint = 'http://localhost:8001/api/v1/dataproxy/'
self.format = 'json'
self.filename = '/bin/true'
self.object_name = 'true'
debug = sys.stderr if settings.DEBUG else False
self.server = API(self.endpoint, debug=debug)
# Create one token
self.token = '2bae7fa7-5810-4664-8264-725fa3a714b1'
self.bad_token = 'not-a-valid-token'
def upload_object(self, args):
filename = args.object
object_name = os.path.basename(filename)
if not os.path.exists(filename):
print "Error: Failed to open %s" % filename
print "Aborting..."
sys.exit()
server_uri = self._get_data_proxy()
f = open(filename, 'rb')
object_data = {'name': object_name, 'file': f}
if args.public:
object_data['public'] = 'true'
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (filename, object_id)
def is_authenticated(self, request, **kwargs):
# check if token exists inside the request
try:
token = request.GET['token']
except KeyError:
return False
goo_server = settings.GOO_SERVER_URI
debug = sys.stderr if settings.DEBUG else False
server = API(goo_server, debug=debug)
try:
response = server.token.get(token=token)
if response['expire_time']:
request.token = token
return True
else:
return False # pragma: no cover
except HttpClientError as e:
if e.code == 401:
# maybe a pilot token
try:
response = server.pilot.token.get(token=token)
if response['valid']:
request.token = token
return True
else:
return False
except HttpClientError as e:
if e.code == 401:
return False
else:
raise e # pragma: no cover
else:
raise e # pragma: no cover
return False # pragma: no cover
def __init__(self, token=None):
goo_server = settings.GOO_SERVER_URI
self.proxy_token = settings.GOO_SERVER_TOKEN
self.user_token = token
debug = sys.stderr if settings.DEBUG else False
self.server = API(goo_server, debug=debug)
class GooClient():
def __init__(self, api_uri, debug=False):
self.output = Output()
self.api_uri = api_uri
self.debug = debug
if self.debug:
self.set_debug()
def _slugfy(self, text, separator='-'):
ret = ""
for c in text.lower():
try:
ret += htmlentitydefs.codepoint2name[ord(c)]
except:
ret += c
ret = re.sub("([a-zA-Z])(uml|acute|grave|circ|tilde|cedil)", r"\1", ret)
ret = re.sub("\W", " ", ret)
ret = re.sub(" +", separator, ret)
return ret.strip()
@translate_gooapi_to_gooclient_exception
def create_api(self):
self.api = API(self.api_uri, format="json", debug=self.debug)
@translate_gooapi_to_gooclient_exception
def set_debug(self):
self.debug = sys.stderr
@translate_gooapi_to_gooclient_exception
def _get_data_proxy(self):
servers = self._get_dataproxy_servers()
if len(servers) == 0:
print "Error: No dataproxy servers found"
print "Please contact NCC team"
print "Aborting..."
sys.exit()
# TODO: write a better heuristic, now is the first server.
server_url = servers[0]['url']
server_uri = "%sapi/%s/" % (server_url, CURRENT_API_VERSION)
return server_uri
@translate_gooapi_to_gooclient_exception
def request_token(self, username, password):
api = API(self.api_uri, auth=(username, password), debug=self.debug)
token = api.auth.post({})
self.set_token(token['token'])
return token['token']
def set_token(self, token):
self.token = token
@translate_gooapi_to_gooclient_exception
def get_apps(self, args):
apps = self.api.apps.get(token=self.token)
apps = apps['objects']
# filter out _active = False
apps = filter(lambda x: x['_active'], apps)
# Field name and size in cols
fields = [ {'id': 5},
{'_name': 30},
{'_multi_hosts': 15},
{'_multi_thread': 15}]
self.output.show(fields, apps)
return apps
@translate_gooapi_to_gooclient_exception
def get_jobs(self, args):
jobs = self.api.jobs.get(token=self.token)
jobs = jobs['objects']
# Field name and size in cols
fields = [ {'id': 5},
{'name': 30},
{'status': 7},
{'priority': 10},
{'progress': 10}]
self.output.show(fields, jobs)
return jobs
@translate_gooapi_to_gooclient_exception
def delete_object(self, args):
object_id = args.object_id
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
dps_api.dataobjects(object_id).delete(token=self.token)
print "Object %s delete with success" % object_id
@translate_gooapi_to_gooclient_exception
def download_object(self, args):
object_id = args.object_id
# Get object info
obj = self.api.dataobjects(object_id).get(token=self.token)
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
data = dps_api.dataobjects(object_id).get(token=self.token)
f = open(obj['name'], "w+")
f.write(data)
f.close()
print "Finish"
@translate_gooapi_to_gooclient_exception
def upload_object(self, args):
filename = args.object
object_name = os.path.basename(filename)
if not os.path.exists(filename):
print "Error: Failed to open %s" % filename
print "Aborting..."
sys.exit()
server_uri = self._get_data_proxy()
f = open(filename, 'rb')
object_data = {'name': object_name,
'file': f}
if args.public:
object_data['public'] = 'true'
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (filename, object_id)
@translate_gooapi_to_gooclient_exception
def create_object(self, args):
inputs = args.inputs
object_name = "%s.zip" % args.name
fd, filepath = tempfile.mkstemp('.zip', self._slugfy(args.name))
# Create package
print 'Creating object package...'
for f in inputs:
zf = zipfile.ZipFile(filepath, mode='a')
try:
print ' adding %s' % f
zf.write(f)
finally:
zf.close()
server_uri = self._get_data_proxy()
f = open(filepath, 'rb')
object_data = {'name': object_name,
'file': f}
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (object_name, object_id)
# Force tempfile to be removed
os.unlink(filepath)
return result['resource_uri']
@translate_gooapi_to_gooclient_exception
def get_objects(self, args):
objects = self.api.dataobjects.get(token=self.token)
objects = objects['objects']
# Field name and size in cols
fields = [ {'id': 5},
{'name': 30},
{'size': 7},
{'create_time': 10}]
self.output.show(fields, objects)
return objects
@translate_gooapi_to_gooclient_exception
def _get_dataproxy_servers(self):
servers = self.api.dataproxyserver.get(token=self.token)
servers = servers['objects']
return servers
def show_dataproxy_servers(self, args):
servers = self._get_dataproxy_servers()
fields = [ {'id': 5},
{'name': 30},
{'url': 30}]
self.output.show(fields, servers)
@translate_gooapi_to_gooclient_exception
def get_job_template(self, args):
app_id = args.app_type_id
name = args.name
slug = self._slugfy(name,"-")
app = self.api.apps(app_id).get(token=self.token)
required_fields = app['_required_fields'].split(', ')
constant_fields = app['_constant_fields'].split(', ')
exclude = ('id', 'resource_uri')
print "# Goo template file"
print "# Generated by goo-client at %s" % datetime.datetime.utcnow()
print ""*2
print "# Required fields"
print "name=%s" % name
print "application=/api/%s/apps/%s/" % (CURRENT_API_VERSION, app_id)
for f in required_fields:
print "%s=%s" % (f, app[f])
print ""*2
print "# Optional fields"
print "# Uncomment the follow lines if you want to change the"
print "# default values"
print ""
for v, k in app.items():
if v not in exclude and \
not v.startswith('_') and \
v not in constant_fields and \
v not in required_fields:
print "#%s=%s" % (v, k)
print "#slug=%s" % slug
@translate_gooapi_to_gooclient_exception
def show_job(self, args):
job_id = args.job_id
result = self.api.jobs(job_id).get(token=self.token)
fields = ('id', 'name', 'app_name', 'priority', 'status', 'progress',
'restart', 'maxtime', 'create_time', 'modification_time', 'input_objs', 'output_objs')
width = 52
print "-" * width
print "Job Detail".center(width)
print "-" * width
for f in fields:
print "%20s: " % f,
if f in ('input_objs', 'output_objs'):
output = []
for o in result[f]:
output.append(o['id'])
else:
output = result[f]
print "%s" % output
print "-" * width
@translate_gooapi_to_gooclient_exception
def remove_job(self, args):
job_id = args.job_id
result = self.api.jobs(job_id).delete(token=self.token)
print "Job %s removed" % job_id
@translate_gooapi_to_gooclient_exception
def submit_job(self, args):
template = args.template
if not os.path.exists(template):
print "Error: Failed to open %s" % template
print "Aborting..."
sys.exit()
cp = ConfigParser.ConfigParser()
cp.readfp(FakeSecHead(open(template)))
fields = ('name', 'application', 'hosts', 'cores_per_host', 'priority', 'restart',
'executable', 'args', 'inputs', 'outputs', 'checkpoints',
'input_objs', 'output_objs', 'checkpoint_objs', 'maxtime',
'diskspace', 'memory')
values = {}
inputs = None
for field in fields:
try:
value = cp.get('template', field)
if value is '':
print "Error: Syntax invalid"
print "%s has a empty value" % field
print "Aborting..."
sys.exit()
if field in ('output_objs', 'input_objs', 'checkpoint_objs'):
values[field] = value.split(" ")
elif field is 'inputs':
inputs = value.split(" ")
else:
values[field] = value
except:
pass
if inputs:
obj_name = self._slugfy("%s-inputs" % values['name'])
input_files = []
for i in inputs:
input_files.extend(glob.glob(i))
args = argparse.Namespace(name=obj_name, inputs=input_files)
values['input_objs'] = [self.create_object(args)]
job = self.api.jobs.post(values, token=self.token)
print "Job %s sent to queue (#%d)" % (job['name'], job['id'])
class DataObjectResourceTest(TestCase):
def setUp(self):
super(DataObjectResourceTest, self).setUp()
self.endpoint = 'http://localhost:8001/api/v1/dataproxy/'
self.format = 'json'
self.filename = '/bin/true'
self.object_name = 'true'
debug = sys.stderr if settings.DEBUG else False
self.server = API(self.endpoint, debug=debug)
# Create one token
self.token = '2bae7fa7-5810-4664-8264-725fa3a714b1'
self.bad_token = 'not-a-valid-token'
def test_token_missing(self):
try:
ret = self.server.dataobjects(9999).get()
except HttpClientError as e:
if e.code != 401:
self.fail('Return code != 401')
except Exception as e:
self.fail("Another exception %s" % e)
def test_bad_token(self):
try:
ret = self.server.dataobjects(9999).get(token=self.bad_token)
except HttpClientError as e:
if e.code != 401:
self.fail('Return code != 401')
except Exception as e:
self.fail("Another exception %s" % e)
def test_upload(self):
try:
f = open(self.filename, 'r')
object_data = {'name': "%s" % self.object_name,
'file': f}
result = self.server.dataobjects.post(data=object_data, token=self.token)
f.close()
return result['resource_uri']
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
def _sha1(self, data):
sha1 = hashlib.sha1()
sha1.update(data)
return sha1.hexdigest()
# TODO: do a sha1 or md5 check
def test_download(self):
resource_uri = self.test_upload()
oid = int(resource_uri.split('/')[-2:][0])
f = open(self.filename, "r")
original_sha1 = self._sha1(f.read())
f.close()
try:
result = self.server.dataobjects(oid).get(token=self.token)
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
download_sha1 = self._sha1(result)
if original_sha1 != download_sha1:
self.fail("Sha1 different")
def test_delete(self):
resource_uri = self.test_upload()
oid = int(resource_uri.split('/')[-2:][0])
try:
result = self.server.dataobjects(oid).delete(token=self.token)
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
def __init__(self, token=None):
goo_server = settings.GOO_SERVER_URI
self.proxy_token = settings.GOO_SERVER_TOKEN
self.user_token = token
debug = sys.stderr if settings.DEBUG else False
self.server = API(goo_server, debug=debug)
def create_api(self):
self.api = API(self.api_uri, format="json", debug=self.debug)
def create_api(self):
self.api = API(self.api_uri, format="json", debug=self.debug)
class GooClient():
def __init__(self, api_uri, debug=False):
self.output = Output()
self.api_uri = api_uri
self.debug = debug
if self.debug:
self.set_debug()
def _slugfy(self, text, separator='-'):
ret = ""
for c in text.lower():
try:
ret += htmlentitydefs.codepoint2name[ord(c)]
except:
ret += c
ret = re.sub("([a-zA-Z])(uml|acute|grave|circ|tilde|cedil)", r"\1",
ret)
ret = re.sub("\W", " ", ret)
ret = re.sub(" +", separator, ret)
return ret.strip()
@translate_gooapi_to_gooclient_exception
def create_api(self):
self.api = API(self.api_uri, format="json", debug=self.debug)
@translate_gooapi_to_gooclient_exception
def set_debug(self):
self.debug = sys.stderr
@translate_gooapi_to_gooclient_exception
def _get_data_proxy(self):
servers = self._get_dataproxy_servers()
if len(servers) == 0:
print "Error: No dataproxy servers found"
print "Please contact NCC team"
print "Aborting..."
sys.exit()
# TODO: write a better heuristic, now is the first server.
server_url = servers[0]['url']
server_uri = "%sapi/%s/" % (server_url, CURRENT_API_VERSION)
return server_uri
@translate_gooapi_to_gooclient_exception
def request_token(self, username, password):
api = API(self.api_uri, auth=(username, password), debug=self.debug)
token = api.auth.post({})
self.set_token(token['token'])
return token['token']
def set_token(self, token):
self.token = token
@translate_gooapi_to_gooclient_exception
def get_apps(self, args):
apps = self.api.apps.get(token=self.token)
apps = apps['objects']
# filter out _active = False
apps = filter(lambda x: x['_active'], apps)
# Field name and size in cols
fields = [{
'id': 5
}, {
'_name': 30
}, {
'_multi_hosts': 15
}, {
'_multi_thread': 15
}]
self.output.show(fields, apps)
return apps
@translate_gooapi_to_gooclient_exception
def get_jobs(self, args):
jobs = self.api.jobs.get(token=self.token)
jobs = jobs['objects']
# Field name and size in cols
fields = [{
'id': 5
}, {
'name': 30
}, {
'status': 7
}, {
'priority': 10
}, {
'progress': 10
}]
self.output.show(fields, jobs)
return jobs
@translate_gooapi_to_gooclient_exception
def delete_object(self, args):
object_id = args.object_id
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
dps_api.dataobjects(object_id).delete(token=self.token)
print "Object %s delete with success" % object_id
@translate_gooapi_to_gooclient_exception
def download_object(self, args):
object_id = args.object_id
# Get object info
obj = self.api.dataobjects(object_id).get(token=self.token)
server_uri = self._get_data_proxy()
dps_api = API(server_uri, debug=self.debug)
data = dps_api.dataobjects(object_id).get(token=self.token)
f = open(obj['name'], "w+")
f.write(data)
f.close()
print "Finish"
@translate_gooapi_to_gooclient_exception
def upload_object(self, args):
filename = args.object
object_name = os.path.basename(filename)
if not os.path.exists(filename):
print "Error: Failed to open %s" % filename
print "Aborting..."
sys.exit()
server_uri = self._get_data_proxy()
f = open(filename, 'rb')
object_data = {'name': object_name, 'file': f}
if args.public:
object_data['public'] = 'true'
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (filename, object_id)
@translate_gooapi_to_gooclient_exception
def create_object(self, args):
inputs = args.inputs
object_name = "%s.zip" % args.name
fd, filepath = tempfile.mkstemp('.zip', self._slugfy(args.name))
# Create package
print 'Creating object package...'
for f in inputs:
zf = zipfile.ZipFile(filepath, mode='a')
try:
print ' adding %s' % f
zf.write(f)
finally:
zf.close()
server_uri = self._get_data_proxy()
f = open(filepath, 'rb')
object_data = {'name': object_name, 'file': f}
dps_api = API(server_uri, debug=self.debug)
result = dps_api.dataobjects.post(data=object_data, token=self.token)
f.close()
object_id = re.search(r'/api/v1/dataobjects/(\d+)/',
result['resource_uri']).group(1)
print "%s uploaded with success (#%s)" % (object_name, object_id)
# Force tempfile to be removed
os.unlink(filepath)
return result['resource_uri']
@translate_gooapi_to_gooclient_exception
def get_objects(self, args):
objects = self.api.dataobjects.get(token=self.token)
objects = objects['objects']
# Field name and size in cols
fields = [{'id': 5}, {'name': 30}, {'size': 7}, {'create_time': 10}]
self.output.show(fields, objects)
return objects
@translate_gooapi_to_gooclient_exception
def _get_dataproxy_servers(self):
servers = self.api.dataproxyserver.get(token=self.token)
servers = servers['objects']
return servers
def show_dataproxy_servers(self, args):
servers = self._get_dataproxy_servers()
fields = [{'id': 5}, {'name': 30}, {'url': 30}]
self.output.show(fields, servers)
@translate_gooapi_to_gooclient_exception
def get_job_template(self, args):
app_id = args.app_type_id
name = args.name
slug = self._slugfy(name, "-")
app = self.api.apps(app_id).get(token=self.token)
required_fields = app['_required_fields'].split(', ')
constant_fields = app['_constant_fields'].split(', ')
exclude = ('id', 'resource_uri')
print "# Goo template file"
print "# Generated by goo-client at %s" % datetime.datetime.utcnow()
print "" * 2
print "# Required fields"
print "name=%s" % name
print "application=/api/%s/apps/%s/" % (CURRENT_API_VERSION, app_id)
for f in required_fields:
print "%s=%s" % (f, app[f])
print "" * 2
print "# Optional fields"
print "# Uncomment the follow lines if you want to change the"
print "# default values"
print ""
for v, k in app.items():
if v not in exclude and \
not v.startswith('_') and \
v not in constant_fields and \
v not in required_fields:
print "#%s=%s" % (v, k)
print "#slug=%s" % slug
@translate_gooapi_to_gooclient_exception
def show_job(self, args):
job_id = args.job_id
result = self.api.jobs(job_id).get(token=self.token)
fields = ('id', 'name', 'app_name', 'priority', 'status', 'progress',
'restart', 'maxtime', 'create_time', 'modification_time',
'input_objs', 'output_objs')
width = 52
print "-" * width
print "Job Detail".center(width)
print "-" * width
for f in fields:
print "%20s: " % f,
if f in ('input_objs', 'output_objs'):
output = []
for o in result[f]:
output.append(o['id'])
else:
output = result[f]
print "%s" % output
print "-" * width
@translate_gooapi_to_gooclient_exception
def remove_job(self, args):
job_id = args.job_id
result = self.api.jobs(job_id).delete(token=self.token)
print "Job %s removed" % job_id
@translate_gooapi_to_gooclient_exception
def submit_job(self, args):
template = args.template
if not os.path.exists(template):
print "Error: Failed to open %s" % template
print "Aborting..."
sys.exit()
cp = ConfigParser.ConfigParser()
cp.readfp(FakeSecHead(open(template)))
fields = ('name', 'application', 'hosts', 'cores_per_host', 'priority',
'restart', 'executable', 'args', 'inputs', 'outputs',
'checkpoints', 'input_objs', 'output_objs',
'checkpoint_objs', 'maxtime', 'diskspace', 'memory')
values = {}
inputs = None
for field in fields:
try:
value = cp.get('template', field)
if value is '':
print "Error: Syntax invalid"
print "%s has a empty value" % field
print "Aborting..."
sys.exit()
if field in ('output_objs', 'input_objs', 'checkpoint_objs'):
values[field] = value.split(" ")
elif field is 'inputs':
inputs = value.split(" ")
else:
values[field] = value
except:
pass
if inputs:
obj_name = self._slugfy("%s-inputs" % values['name'])
input_files = []
for i in inputs:
input_files.extend(glob.glob(i))
args = argparse.Namespace(name=obj_name, inputs=input_files)
values['input_objs'] = [self.create_object(args)]
job = self.api.jobs.post(values, token=self.token)
print "Job %s sent to queue (#%d)" % (job['name'], job['id'])
class DataObjectResourceTest(TestCase):
def setUp(self):
super(DataObjectResourceTest, self).setUp()
self.endpoint = 'http://localhost:8001/api/v1/dataproxy/'
self.format = 'json'
self.filename = '/bin/true'
self.object_name = 'true'
debug = sys.stderr if settings.DEBUG else False
self.server = API(self.endpoint, debug=debug)
# Create one token
self.token = '2bae7fa7-5810-4664-8264-725fa3a714b1'
self.bad_token = 'not-a-valid-token'
def test_token_missing(self):
try:
ret = self.server.dataobjects(9999).get()
except HttpClientError as e:
if e.code != 401:
self.fail('Return code != 401')
except Exception as e:
self.fail("Another exception %s" % e)
def test_bad_token(self):
try:
ret = self.server.dataobjects(9999).get(token=self.bad_token)
except HttpClientError as e:
if e.code != 401:
self.fail('Return code != 401')
except Exception as e:
self.fail("Another exception %s" % e)
def test_upload(self):
try:
f = open(self.filename, 'r')
object_data = {'name': "%s" % self.object_name, 'file': f}
result = self.server.dataobjects.post(data=object_data,
token=self.token)
f.close()
return result['resource_uri']
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
def _sha1(self, data):
sha1 = hashlib.sha1()
sha1.update(data)
return sha1.hexdigest()
# TODO: do a sha1 or md5 check
def test_download(self):
resource_uri = self.test_upload()
oid = int(resource_uri.split('/')[-2:][0])
f = open(self.filename, "r")
original_sha1 = self._sha1(f.read())
f.close()
try:
result = self.server.dataobjects(oid).get(token=self.token)
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
download_sha1 = self._sha1(result)
if original_sha1 != download_sha1:
self.fail("Sha1 different")
def test_delete(self):
resource_uri = self.test_upload()
oid = int(resource_uri.split('/')[-2:][0])
try:
result = self.server.dataobjects(oid).delete(token=self.token)
except HttpClientError as e:
self.fail('Return code == %d' % e.code)
except Exception as e:
self.fail("Another exception %s" % e)
def request_token(self, username, password):
api = API(self.api_uri, auth=(username, password), debug=self.debug)
token = api.auth.post({})
self.set_token(token['token'])
return token['token']