def get_usage(self, start_date, end_date, by_group, show_all_users):
"""
Return historical usage
"""
params = {}
params['start'] = start_date
params['end'] = end_date
if by_group:
params['by_group'] = 'true'
if show_all_users:
params['show_all_users'] = 'true'
try:
response = requests.get(self._url + '/accounting', params=params, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.json()
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code == 500:
raise exceptions.UsageError('Unknown error when querying the PROMINENCE server')
else:
if 'error' in response.json():
raise exceptions.UsageError(response.json()['error'])
raise exceptions.UsageError('Unknown error when querying the PROMINENCE server')
return {}
def list_workflows(self, completed=False, all=False, num=1, constraint=None, name_constraint=None):
"""
List running/idle workflows or completed workflows
"""
params = {}
if completed:
params['completed'] = 'true'
if num:
params['num'] = num
if all:
params['all'] = 'true'
if constraint:
params['constraint'] = constraint
if name_constraint:
params['name'] = name_constraint
try:
response = requests.get(self._url + '/workflows', params=params, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.json()
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
raise exceptions.WorkflowGetError(response.json()['error'])
raise exceptions.WorkflowGetError('Unknown error')
def stderr_generic(self, type, id, job=None, instance=-1):
"""
Get standard error from a job
"""
if job is not None:
if instance > -1:
path = '/%s/%d/%s/%d/stderr' % (type, id, job, instance)
else:
path = '/%s/%d/%s/stderr' % (type, id, job)
else:
path = '/%s/%d/stderr' % (type, id)
try:
response = requests.get(self._url + path, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.text
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
else:
if 'error' in response.json():
raise exceptions.StdStreamsError(response.json()['error'])
raise exceptions.StdStreamsError('Unknown error')
def create_snapshot(self, job_id, path):
"""
Create a snapshot of a file or directory in a running job
"""
headers = dict(self._headers)
headers['Content-type'] = 'application/json'
params = {}
params['path'] = path
try:
response = requests.put(self._url + '/jobs/%d/snapshot' % job_id, timeout=self._timeout, headers=headers, params=params, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.text
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif 'error' in response.json():
raise exceptions.SnapshotCreateError(response.json()['error'])
raise exceptions.SnapshotCreateError('Unknown error')
def execute_command(self, job_id, command):
"""
Execute a command inside a job
"""
headers = dict(self._headers)
headers['Content-type'] = 'application/json'
params = {}
params['command'] = ','.join(command)
try:
response = requests.post(self._url + '/jobs/%d/exec' % job_id, timeout=self._timeout, headers=headers, params=params, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.text
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif 'error' in response.json():
raise exceptions.ExecError(response.json()['error'])
raise exceptions.ExecError('Unknown error')
def upload(self, file, filename):
"""
Upload a file to transient cloud storage
"""
# Get S3 URL
data = {'filename':file}
headers = dict(self._headers)
headers['Content-type'] = 'application/json'
try:
response = requests.post(self._url + '/data/upload', data=json.dumps(data), headers=headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 201:
if 'url' in response.json():
url = response.json()['url']
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
else:
if 'error' in response.json():
raise exceptions.FileUploadError(response.json()['error'])
raise exceptions.FileUploadError('Unknown error when querying the PROMINENCE server')
# Upload
try:
with open(filename, 'rb') as file_obj:
response = requests.put(url, data=file_obj, timeout=30)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
except IOError as err:
raise exceptions.FileUploadError(err)
if response.status_code == 200:
return True
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid cloud storage URL, got a 404 not found error from cloud storage')
raise exceptions.FileUploadError('Got status code', response.status_code, 'from cloud storage')
def get_snapshot_url(self, job_id):
"""
Get the URL of the current snapshot
"""
try:
response = requests.get(self._url + '/jobs/%d/snapshot' % job_id, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.json()
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
raise exceptions.SnapshotGetError(response.json()['error'])
raise exceptions.SnapshotGetError('Unknown error')
def rerun_workflow(self, workflow_id):
"""
Rerun any failed jobs from a completed workflow
"""
try:
response = requests.put(self._url + '/workflows/%d' % workflow_id, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
if 'id' in response.json():
return response.json()['id']
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
raise exceptions.WorkflowCreationError(response.json()['error'])
raise exceptions.WorkflowCreationError('Unknown error')
def delete_object(self, object):
"""
Delete an object in cloud storage
"""
url = self._url + '/data/' + object
try:
response = requests.delete(url, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 204:
return True
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
else:
if 'error' in response.json():
raise exceptions.ObjectError(response.json()['error'])
raise exceptions.ObjectError('Unknown error when querying the PROMINENCE server')
return False
def delete_generic(self, id, resource):
"""
Delete the specified job or workflow
"""
try:
response = requests.delete(self._url + '/%s/%d' % (resource, id), timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return True
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code == 500:
raise exceptions.DeletionError('Got a 500 internal server error from PROMINENCE')
else:
if 'error' in response.json():
raise exceptions.DeletionError(response.json()['error'])
raise exceptions.DeletionError('Unknown error')
def list_objects(self, path):
"""
List objects in cloud storage
"""
url = self._url + '/data'
if path:
url += '/%s' % path
try:
response = requests.get(url, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
return response.json()
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
else:
if 'error' in response.json():
raise exceptions.ObjectError(response.json()['error'])
raise exceptions.ObjectError('Unknown error when querying the PROMINENCE server')
def describe_workflow(self, workflow_id):
"""
Describe a specific workflow
"""
try:
response = requests.get(self._url + '/workflows/%d' % workflow_id, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
if response.json():
return response.json()[0]
else:
raise exceptions.JobGetError('No such workflow')
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
return self.Response(return_code=1, data={'error': '%s' % response.json()['error']})
raise exceptions.WorkflowGetError('Unknown error')
def describe_job(self, job_id):
"""
Describe a specific job
"""
try:
response = requests.get(self._url + '/jobs/%d' % job_id, timeout=self._timeout, headers=self._headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 200:
if response.json():
return response.json()[0]
else:
raise exceptions.JobGetError('No such job')
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
raise exceptions.JobGetError(response.json()['error'])
raise exceptions.JobGetError('Unknown error')
def create_workflow(self, workflow):
"""
Create a workflow from a JSON description
"""
headers = dict(self._headers)
headers['Content-type'] = 'application/json'
try:
response = requests.post(self._url + '/workflows', data=json.dumps(workflow), timeout=self._timeout, headers=headers, verify=self._verify)
except requests.exceptions.RequestException as err:
raise exceptions.ConnectionError(err)
if response.status_code == 201:
if 'id' in response.json():
return response.json()['id']
elif response.status_code == 401:
raise exceptions.AuthenticationError()
elif response.status_code == 404:
raise exceptions.ConnectionError('Invalid PROMINENCE URL, got a 404 not found error')
elif response.status_code < 500:
if 'error' in response.json():
raise exceptions.WorkflowCreationError(response.json()['error'])
raise exceptions.WorkflowCreationError('Unknown error')
def authenticate_user(create_client_if_needed=True, token_in_file=True):
"""
Obtain token from OIDC provider
"""
# Create .prominence directory if necessary: this is done automatically if the user registers as a client, but for
# the case of reading the client credentials from environment variables this step will be missed
create_config_dir()
try:
(client_id, client_secret) = get_client()
except exceptions.ClientCredentialsError:
client_id = None
if not client_id:
if create_client_if_needed:
if register_client():
(client_id, client_secret) = get_client()
if not client_id:
raise exceptions.ClientCredentialsError(
'Unable to get a client id and client secret')
else:
raise exceptions.ClientCredentialsError(
'Unable to get a client id and client secret')
else:
raise exceptions.ClientCredentialsError(
'Unable to get a client id and client secret, and automatic registration was disabled'
)
data = {}
data['scope'] = 'openid profile email'
data['client_id'] = client_id
try:
response = requests.post(os.environ.get('PROMINENCE_OIDC_URL',
DEFAULT_PROMINENCE_OIDC_URL) +
'/devicecode',
data=data,
timeout=10,
auth=(client_id, client_secret),
allow_redirects=True)
except requests.exceptions.RequestException as err:
raise exceptions.AuthenticationError(
'Unable to initiate the device code flow: cannot connect to OIDC server'
)
if response.status_code != 200:
raise exceptions.AuthenticationError(
'Unable to initiate the device code flow: got status code %d' %
response.status_code)
device_code_response = response.json()
print(
'To obtain a token, use a web browser to open the page %s and enter the code %s when requested'
% (device_code_response['verification_uri'],
device_code_response['user_code']))
data = {}
data['grant_type'] = 'urn:ietf:params:oauth:grant-type:device_code'
data['device_code'] = device_code_response['device_code']
# Wait for the user to authenticate
current_time = time.time()
authenticated = False
while time.time() < current_time + int(
device_code_response['expires_in']) and not authenticated:
time.sleep(5)
try:
response = requests.post(os.environ.get(
'PROMINENCE_OIDC_URL', DEFAULT_PROMINENCE_OIDC_URL) + '/token',
data=data,
timeout=10,
auth=(client_id, client_secret),
allow_redirects=True)
except requests.exceptions.RequestException:
continue
if response.status_code == 200:
authenticated = True
if token_in_file:
try:
with open(os.path.expanduser('~/.prominence/token'),
'w') as token_file:
json.dump(response.json(), token_file)
os.chmod(os.path.expanduser('~/.prominence/token'), 384)
except IOError:
raise exceptions.AuthenticationError(
'Unable to write to ~/.prominence/token')
else:
return response.json()['access_token']
if not authenticated:
raise exceptions.AuthenticationError('Authentication failed')
return True