def _cancel_jobs(dn):
"""
Cancel all jobs that belong to dn.
Returns the list of affected jobs ids.
"""
jobs = Session.query(Job.job_id).filter(Job.job_state.in_(JobActiveStates),
Job.user_dn == dn,
Job.job_finished == None)
job_ids = map(lambda j: j[0], jobs)
try:
now = datetime.utcnow()
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id).filter(File.file_state.in_(FileActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'User banned',
'finish_time': now
}, synchronize_session=False)
Session.query(Job).filter(Job.job_id == job_id)\
.update({
'job_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now
}, synchronize_session=False)
Session.commit()
Session.expire_all()
return job_ids
except Exception:
Session.rollback()
raise
def get_access_granted(self):
dropbox_user_info = self._get_dropbox_user_info()
if not dropbox_user_info:
raise HTTPBadRequest('No registered user for the service "%s" has been found' % self.service)
dropbox_info = self._get_dropbox_info()
if not dropbox_info:
raise HTTPNotFound('Dropbox info not found in the databse')
access_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/access_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="' +
dropbox_info.app_key + '", oauth_token="' + dropbox_user_info.request_token +
'", oauth_signature="' + dropbox_info.app_secret + '&' + dropbox_user_info.request_token_secret + '"',
None
)
# It returns: oauth_token=<access-token>&oauth_token_secret=<access-token-secret>&uid=<user-id>
access_tokens = access_tokens.split('&')
dropbox_user_info.access_token = access_tokens[1].split('=')[1]
dropbox_user_info.access_token_secret = access_tokens[0].split('=')[1]
try:
Session.add(dropbox_user_info)
Session.commit()
except:
Session.rollback()
raise
return access_tokens
def remove_authz(self, start_response):
"""
Revoke access for a DN for a given operation, or all
"""
input_dict = get_input_as_dict(request, from_query=True)
dn = input_dict.get('dn')
op = input_dict.get('operation')
if not dn:
raise HTTPBadRequest('Missing DN parameter')
to_be_removed = Session.query(AuthorizationByDn).filter(
AuthorizationByDn.dn == dn)
if op:
to_be_removed = to_be_removed.filter(
AuthorizationByDn.operation == op)
try:
to_be_removed.delete()
if op:
audit_configuration('revoke', '%s revoked for "%s"' % (op, dn))
else:
audit_configuration('revoke', 'All revoked for "%s"' % dn)
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def get_access_requested(self):
dropbox_info = self._get_dropbox_info()
request_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/request_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT",'
'oauth_consumer_key="' + dropbox_info.app_key + '", oauth_signature="' + dropbox_info.app_secret + '&"',
None
)
# It returns: oauth_token_secret=b9q1n5il4lcc&oauth_token=mh7an9dkrg59
tokens = request_tokens.split('&')
newuser = CloudStorageUser(
user_dn=self.user_dn,
cloudStorage_name=dropbox_info.cloudStorage_name,
request_token=tokens[1].split('=')[1],
request_token_secret=tokens[0].split('=')[1]
)
try:
Session.add(newuser)
Session.commit()
except:
Session.rollback()
raise
return request_tokens
def get_access_granted(self):
dropbox_user_info = self._get_dropbox_user_info()
if not dropbox_user_info:
raise HTTPBadRequest('No registered user for the service "%s" has been found' % self.service)
dropbox_info = self._get_dropbox_info()
if not dropbox_info:
raise HTTPNotFound('Dropbox info not found in the database')
access_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/access_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="' +
dropbox_info.app_key + '", oauth_token="' + dropbox_user_info.request_token +
'", oauth_signature="' + dropbox_info.app_secret + '&' + dropbox_user_info.request_token_secret + '"',
None
)
# It returns: oauth_token=<access-token>&oauth_token_secret=<access-token-secret>&uid=<user-id>
access_tokens = access_tokens.split('&')
dropbox_user_info.access_token = access_tokens[1].split('=')[1]
dropbox_user_info.access_token_secret = access_tokens[0].split('=')[1]
try:
Session.add(dropbox_user_info)
Session.commit()
except:
Session.rollback()
raise
return access_tokens
def get_access_requested(self):
dropbox_info = self._get_dropbox_info()
request_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/request_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT",'
'oauth_consumer_key="' + dropbox_info.app_key + '", oauth_signature="' + dropbox_info.app_secret + '&"',
None
)
# It returns: oauth_token_secret=b9q1n5il4lcc&oauth_token=mh7an9dkrg59
tokens = request_tokens.split('&')
newuser = CloudStorageUser(
user_dn=self.user_dn,
storage_name=dropbox_info.storage_name,
request_token=tokens[1].split('=')[1],
request_token_secret=tokens[0].split('=')[1],
vo_name=''
)
try:
Session.add(newuser)
Session.commit()
except:
Session.rollback()
raise
return request_tokens
def _cancel_jobs(dn):
"""
Cancel all jobs that belong to dn.
Returns the list of affected jobs ids.
"""
jobs = Session.query(Job.job_id).filter(Job.job_state.in_(JobActiveStates)).filter(Job.user_dn == dn)
job_ids = map(lambda j: j[0], jobs)
try:
now = datetime.utcnow()
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id).filter(File.file_state.in_(FileActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now, 'finish_time': now
}, synchronize_session=False)
Session.query(Job).filter(Job.job_id == job_id)\
.update({
'job_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now, 'finish_time': now
}, synchronize_session=False)
Session.commit()
Session.expire_all()
return job_ids
except Exception:
Session.rollback()
raise
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
if credential_cache is None or credential_cache.cert_request is None:
(x509_request, private_key) = _generate_proxy_request()
credential_cache = CredentialCache(dlg_id=user.delegation_id, dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok', [('X-Delegation-ID', credential_cache.dlg_id),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def delete_share(self, start_response):
"""
Delete a share
"""
input_dict = get_input_as_dict(request, from_query=True)
source = input_dict.get('source')
destination = input_dict.get('destination')
vo = input_dict.get('vo')
if not source or not destination or not vo:
raise HTTPBadRequest('Missing source, destination and/or vo')
try:
share = Session.query(ShareConfig).get((source, destination, vo))
if share:
Session.delete(share)
audit_configuration(
'share-delete', 'Share %s, %s, %s has been deleted' %
(source, destination, vo))
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def delete_app(self, client_id):
"""
Delete an application from the database
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if app is None:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
try:
Session.delete(app)
Session.query(OAuth2Token).filter(
OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(
OAuth2Code.client_id == client_id).delete()
Session.commit()
except:
Session.rollback()
raise
log.info("Application removed: %s" % client_id)
return None
def update_app(self, client_id):
"""
Update an application
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if not app:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
if pylons.request.headers['Content-Type'].startswith('application/json'):
fields = json.loads(pylons.request.body)
else:
fields = pylons.request.POST
try:
if 'delete' not in fields:
app.description = fields.get('description', '')
app.website = fields.get('website', '')
app.redirect_to = fields.get('redirect_to', '')
Session.merge(app)
Session.commit()
redirect(url_for(controller='oauth2', action='get_app'), code=HTTPSeeOther.code)
else:
Session.delete(app)
Session.query(OAuth2Token).filter(OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(OAuth2Code.client_id == client_id).delete()
Session.commit()
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
except:
Session.rollback()
raise
def set_drain(self):
"""
Set the drain status of a server
"""
input_dict = get_input_as_dict(request)
hostname = input_dict.get('hostname', None)
drain = input_dict.get('drain', True)
if not isinstance(drain, bool) or not isinstance(hostname, basestring):
raise HTTPBadRequest('Invalid drain request')
entries = Session.query(Host).filter(Host.hostname == hostname).all()
if not entries:
raise HTTPBadRequest('Host not found')
try:
audit_configuration(
'drain',
'Turning drain %s the drain mode for %s' % (drain, hostname))
for entry in entries:
entry.drain = drain
Session.merge(entry)
Session.commit()
except:
Session.rollback()
raise
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
if credential_cache is None or credential_cache.cert_request is None:
(x509_request, private_key) = _generate_proxy_request()
credential_cache = CredentialCache(dlg_id=user.delegation_id, dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok', [('X-Delegation-ID', str(credential_cache.dlg_id)),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def add_user_to_cloud_storage(self, storage_name, start_response):
"""
Add a user or a VO credentials to the storage
"""
storage = Session.query(CloudStorage).get(storage_name)
if not storage:
raise HTTPNotFound('The storage does not exist')
input_dict = get_input_as_dict(request)
if not input_dict.get('user_dn', None) and not input_dict.get('vo_name', None):
raise HTTPBadRequest('One of user_dn or vo_name must be specified')
elif input_dict.get('user_dn', None) and input_dict.get('vo_name', None):
raise HTTPBadRequest('Only one of user_dn or vo_name must be specified')
cuser = CloudStorageUser(
user_dn=input_dict.get('user_dn', ''),
storage_name=storage_name,
access_token=input_dict.get('access_key', input_dict.get('access_token', None)),
access_token_secret=input_dict.get('secret_key', input_dict.get('access_token_secret', None)),
request_token=input_dict.get('request_token'),
request_token_secret=input_dict.get('request_token_secret'),
vo_name=input_dict.get('vo_name', ''),
)
try:
Session.merge(cuser)
Session.commit()
except:
Session.rollback()
raise
start_response('201 Created', [])
return dict(storage_name=cuser.storage_name, user_dn=cuser.user_dn, vo_name=cuser.vo_name)
def submit(self):
"""
Submits a new job
It returns the information about the new submitted job. To know the format for the
submission, /api-docs/schema/submit gives the expected format encoded as a JSON-schema.
It can be used to validate (i.e in Python, jsonschema.validate)
"""
# First, the request has to be valid JSON
submitted_dict = get_input_as_dict(request)
# The auto-generated delegation id must be valid
user = request.environ['fts3.User.Credentials']
credential = Session.query(Credential).get((user.delegation_id, user.user_dn))
if credential is None:
raise HTTPAuthenticationTimeout('No delegation found for "%s"' % user.user_dn)
if credential.expired():
remaining = credential.remaining()
seconds = abs(remaining.seconds + remaining.days * 24 * 3600)
raise HTTPAuthenticationTimeout(
'The delegated credentials expired %d seconds ago (%s)' % (seconds, user.delegation_id)
)
if user.method != 'oauth2' and credential.remaining() < timedelta(hours=1):
raise HTTPAuthenticationTimeout(
'The delegated credentials has less than one hour left (%s)' % user.delegation_id
)
# Populate the job and files
populated = JobBuilder(user, **submitted_dict)
log.info("%s (%s) is submitting a transfer job" % (user.user_dn, user.vos[0]))
# Insert the job
try:
try:
Session.execute(Job.__table__.insert(), [populated.job])
except IntegrityError:
raise HTTPConflict('The sid provided by the user is duplicated')
if len(populated.files):
Session.execute(File.__table__.insert(), populated.files)
if len(populated.datamanagement):
Session.execute(DataManagement.__table__.insert(), populated.datamanagement)
Session.flush()
Session.commit()
except IntegrityError as err:
Session.rollback()
raise HTTPConflict('The submission is duplicated '+ str(err))
except:
Session.rollback()
raise
# Send messages
# Need to re-query so we get the file ids
job = Session.query(Job).get(populated.job_id)
for i in range(len(job.files)):
try:
submit_state_change(job, job.files[i], populated.files[0]['file_state'])
except Exception, e:
log.warning("Failed to write state message to disk: %s" % e.message)
def register(self):
"""
Register a new third party application
"""
if pylons.request.content_type.split(
';')[0].strip() == 'application/json':
req = json.loads(pylons.request.body)
scopes = req.get('scope', list())
else:
req = pylons.request.POST
scopes = req.getall('scope')
if isinstance(scopes, basestring):
scopes = scopes.split(',')
if not req.get('name', None):
raise HTTPBadRequest('Missing application name')
if not req.get('website', None):
raise HTTPBadRequest('Missing application website')
if not req.get('redirect_to', None):
raise HTTPBadRequest('Missing redirect urls')
for s in scopes:
if str(s) not in VALID_OPERATIONS:
raise HTTPBadRequest('Invalid scope (%s)' % s)
user = pylons.request.environ['fts3.User.Credentials']
app_id = _generate_app_id()
app = OAuth2Application(client_id=app_id,
client_secret=_generate_app_secret(),
name=req['name'],
description=req.get('description', ''),
website=req['website'],
scope=scopes,
redirect_to=req['redirect_to'],
owner=user.user_dn)
try:
Session.merge(app)
Session.commit()
except IntegrityError:
Session.rollback()
raise HTTPForbidden('The name already exists')
except:
Session.rollback()
raise
log.info("New application registered: %s (%s)" % (req['name'], app_id))
if _accept_html(pylons.request.accept):
redirect(url_for(controller='oauth2', action='get_my_apps'),
code=HTTPSeeOther.code)
else:
pylons.response.status_int = HTTPCreated.code
pylons.response.headers['Content-Type'] = 'application/json'
return [to_json(app.client_id)]
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden(
'The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
user_cert = self.certificate()
request_key_len = 2048
if user_cert:
user_key = X509.load_cert_string(user_cert)
request_key_len = user_key.get_pubkey().size() * 8
cached = credential_cache is not None and credential_cache.cert_request is not None
if cached:
cached_key_len = X509.load_request_string(
credential_cache.cert_request).get_pubkey().size() * 8
if cached_key_len != request_key_len:
cached = False
log.debug(
"Invalidating cache due to key length missmatch between client and cached certificates"
)
if not cached:
(x509_request,
private_key) = _generate_proxy_request(request_key_len)
credential_cache = CredentialCache(
dlg_id=user.delegation_id,
dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok',
[('X-Delegation-ID', str(credential_cache.dlg_id)),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def remove_token(self):
info = self._get_dropbox_user_info()
if info is None:
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
try:
Session.delete(info)
Session.commit()
except:
Session.rollback()
raise
def cancel_files(self, job_id, file_ids):
"""
Cancel individual files - comma separated for multiple - within a job
"""
job = self._get_job(job_id)
if job.job_type != 'N':
raise HTTPBadRequest('Multihop or reuse jobs must be cancelled at once (%s)' % str(job.job_type))
file_ids = file_ids.split(',')
changed_states = list()
try:
# Mark files in the list as CANCELED
for file_id in file_ids:
file = Session.query(File).get(file_id)
if not file or file.job_id != job_id:
changed_states.append('File does not belong to the job')
continue
if file.file_state not in FileActiveStates:
changed_states.append(file.file_state)
continue
file.file_state = 'CANCELED'
file.finish_time = datetime.utcnow()
file.dest_surl_uuid = None
changed_states.append('CANCELED')
Session.merge(file)
# Mark job depending on the status of the rest of files
not_changed_states = map(lambda f: f.file_state, filter(lambda f: f.file_id not in file_ids, job.files))
all_states = not_changed_states + changed_states
# All files within the job have been canceled
if len(not_changed_states) == 0:
job.job_state = 'CANCELED'
job.cancel_job = True
job.job_finished = datetime.utcnow()
log.warning('Cancelling all remaining files within the job %s' % job_id)
# No files in non-terminal, mark the job as CANCELED too
elif len(filter(lambda s: s in FileActiveStates, all_states)) == 0:
log.warning('Cancelling a file within a job with others in terminal state (%s)' % job_id)
job.job_state = 'CANCELED'
job.cancel_job = True
job.job_finished = datetime.utcnow()
else:
log.warning('Cancelling files within a job with others still active (%s)' % job_id)
Session.merge(job)
Session.commit()
except:
Session.rollback()
raise
return changed_states if len(changed_states) > 1 else changed_states[0]
def set_se_config(self):
"""
Set the configuration parameters for a given SE
"""
input_dict = get_input_as_dict(request)
try:
for storage, cfg in input_dict.iteritems():
if not storage or storage.isspace():
raise ValueError
se_info = None
se_info_new = cfg.get('se_info', None)
if se_info_new:
se_info = Session.query(Se).get(storage)
if not se_info:
se_info = Se(storage=storage)
for key, value in se_info_new.iteritems():
#value = validate_type(Se, key, value)
setattr(se_info, key, value)
audit_configuration(
'set-se-config',
'Set config %s: %s' % (storage, json.dumps(cfg)))
Session.merge(se_info)
# Operation limits
operations = cfg.get('operations', None)
if operations:
for vo, limits in operations.iteritems():
for op, limit in limits.iteritems():
limit = int(limit)
new_limit = Session.query(OperationConfig).get(
(vo, storage, op))
if limit > 0:
if not new_limit:
new_limit = OperationConfig(
vo_name=vo,
host=storage,
operation=op)
new_limit.concurrent_ops = limit
Session.merge(new_limit)
elif new_limit:
Session.delete(new_limit)
audit_configuration(
'set-se-limits', 'Set limits for %s: %s' %
(storage, json.dumps(operations)))
Session.commit()
except (AttributeError, ValueError):
Session.rollback()
raise HTTPBadRequest('Malformed configuration')
except:
Session.rollback()
raise
return (se_info, operations)
def delete_link_config(self,sym_name, start_response):
"""
Deletes an existing link configuration
"""
try:
sym_name = urllib.unquote(sym_name)
Session.query(LinkConfig).filter(LinkConfig.symbolicname == sym_name).delete()
audit_configuration('link-delete', 'Link %s has been deleted' % sym_name)
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def _ban_dn(dn):
"""
Mark in the db the given DN as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedDN()
banned.dn = dn
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def cancel_all_by_vo(self, vo_name):
"""
Cancel all files by the given vo_name
"""
user = request.environ['fts3.User.Credentials']
now = datetime.utcnow()
if not user.is_root:
raise HTTPForbidden(
'User does not have root privileges'
)
try:
# FTS3 daemon expects finish_time to be NULL in order to trigger the signal
# to fts_url_copy
file_count = Session.query(File).filter(File.vo_name == vo_name)\
.filter(File.file_state.in_(FileActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'Job canceled by the user', 'dest_surl_uuid':None,
'finish_time': None
}, synchronize_session=False)
# However, for data management operations there is nothing to signal, so
# set job_finished
dm_count = Session.query(DataManagement).filter(DataManagement.vo_name == vo_name)\
.filter(DataManagement.file_state.in_(DataManagementActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'Job canceled by the user',
'job_finished': now, 'finish_time': now
}, synchronize_session=False)
job_count = Session.query(Job).filter(Job.vo_name == vo_name)\
.filter(Job.job_state.in_(JobActiveStates))\
.update({
'job_state': 'CANCELED', 'reason': 'Job canceled by the user',
'job_finished': now
}, synchronize_session=False)
Session.commit()
Session.expire_all()
log.info("Active jobs for VO %s canceled" % vo_name)
except:
Session.rollback()
raise
return {
"affected_files": file_count,
"affected_dm": dm_count,
"affected_jobs": job_count
}
def _ban_dn(dn, message):
"""
Mark in the db the given DN as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedDN()
banned.dn = dn
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
banned.message = message
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def _set_to_wait(storage, vo_name):
"""
Updates the transfers that have the given storage either in source or destination,
and belong to the given VO.
"""
try:
job_ids = _set_to_wait_helper(storage, vo_name, 'SUBMITTED', 'ON_HOLD')
job_ids.update(
_set_to_wait_helper(storage, vo_name, 'STAGING',
'ON_HOLD_STAGING'))
Session.commit()
Session.expire_all()
except Exception:
Session.rollback()
raise
return job_ids
def update_app(self, client_id):
"""
Update an application
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if not app:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
if pylons.request.headers['Content-Type'].startswith(
'application/json'):
fields = json.loads(pylons.request.body)
scopes = fields.get('scope', list())
else:
fields = pylons.request.POST
scopes = fields.getall('scope')
if isinstance(scopes, basestring):
scopes = scopes.split(',')
for s in scopes:
if str(s) not in VALID_OPERATIONS:
raise HTTPBadRequest('Invalid scope (%s)' % s)
try:
if 'delete' not in fields:
app.description = fields.get('description', '')
app.website = fields.get('website', '')
app.redirect_to = fields.get('redirect_to', '')
app.scope = scopes
Session.merge(app)
Session.commit()
redirect(url_for(controller='oauth2', action='get_app'),
code=HTTPSeeOther.code)
else:
Session.delete(app)
Session.query(OAuth2Token).filter(
OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(
OAuth2Code.client_id == client_id).delete()
Session.commit()
redirect(url_for(controller='oauth2', action='get_my_apps'),
code=HTTPSeeOther.code)
except:
Session.rollback()
raise
def register(self):
"""
Register a new third party application
"""
if pylons.request.content_type.split(';')[0].strip() == 'application/json':
req = json.loads(pylons.request.body)
else:
req = pylons.request.POST
if not req.get('name', None):
raise HTTPBadRequest('Missing application name')
if not req.get('website', None):
raise HTTPBadRequest('Missing application website')
if not req.get('redirect_to', None):
raise HTTPBadRequest('Missing redirect urls')
user = pylons.request.environ['fts3.User.Credentials']
app_id = _generate_app_id()
app = OAuth2Application(
client_id=app_id,
client_secret=_generate_app_secret(),
name=req['name'],
description=req.get('description', ''),
website=req['website'],
redirect_to=req['redirect_to'],
owner=user.user_dn
)
try:
Session.merge(app)
Session.commit()
except IntegrityError:
Session.rollback()
raise HTTPForbidden('The name already exists')
except:
Session.rollback()
raise
log.info("New application registered: %s (%s)" % (req['name'], app_id))
if _accept_html(pylons.request.accept):
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
else:
pylons.response.status_int = HTTPCreated.code
pylons.response.headers['Content-Type'] = 'application/json'
return to_json(app.client_id)
def delete_activity_shares(self, vo_name, start_response):
"""
Delete an existing activity share
"""
activity_share = Session.query(ActivityShare).get(vo_name)
if activity_share is None:
raise HTTPNotFound('No activity shares for %s' % vo_name)
try:
Session.delete(activity_share)
audit_configuration('activity-share',
'Activity share removed for "%s"' % (vo_name))
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def is_access_requested(self):
info = self._get_dropbox_user_info()
if info is None:
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
if info.is_registered():
res = self._get_content("/")
if res.startswith("401"):
try:
Session.delete(info)
Session.commit()
except:
Session.rollback()
raise
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
return info
def is_access_requested(self):
info = self._get_dropbox_user_info()
if info is None:
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
if info.is_registered():
res = self._get_content("/")
if res.startswith("401"):
try:
Session.delete(info)
Session.commit()
except:
Session.rollback()
raise
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
return info.storage_name
def revoke_token(self, client_id):
"""
Current user revokes all tokens for a given application
"""
user = pylons.request.environ['fts3.User.Credentials']
try:
Session.query(OAuth2Token).filter(
(OAuth2Token.client_id == client_id) & (OAuth2Token.dlg_id == user.delegation_id)
).delete()
Session.query(OAuth2Code).filter(
(OAuth2Code.client_id == client_id) & (OAuth2Code.dlg_id == user.delegation_id)
)
Session.commit()
except:
Session.rollback()
raise
log.warning("User %s revoked application %s" % (user.user_dn, client_id))
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
def revoke_token(self, client_id):
"""
Current user revokes all tokens for a given application
"""
user = pylons.request.environ['fts3.User.Credentials']
try:
Session.query(OAuth2Token).filter(
(OAuth2Token.client_id == client_id) & (OAuth2Token.dlg_id == user.delegation_id)
).delete()
Session.query(OAuth2Code).filter(
(OAuth2Code.client_id == client_id) & (OAuth2Code.dlg_id == user.delegation_id)
)
Session.commit()
except:
Session.rollback()
raise
log.warning("User %s revoked application %s" % (user.user_dn, client_id))
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
def remove_cloud_storage(self, storage_name, start_response):
"""
Remove a registered cloud storage
"""
storage = Session.query(CloudStorage).get(storage_name)
if not storage:
raise HTTPNotFound('The storage does not exist')
try:
Session.query(CloudStorageUser).filter(CloudStorageUser.storage_name == storage_name).delete()
Session.delete(storage)
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def delete_vo_global_config(self, start_response):
"""
Delete the global configuration for the given VO
"""
input_dict = get_input_as_dict(request, from_query=True)
vo_name = input_dict.get('vo_name')
if not vo_name or vo_name == '*':
raise HTTPBadRequest('Missing VO name')
try:
Session.query(ServerConfig).filter(
ServerConfig.vo_name == vo_name).delete()
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
def set_share(self, start_response):
"""
Add or modify a share
"""
input_dict = get_input_as_dict(request)
source = input_dict.get('source')
destination = input_dict.get('destination')
vo = input_dict.get('vo')
try:
share = int(input_dict.get('share'))
if share < 0:
raise HTTPBadRequest('Shares values cannot be negative')
except:
raise HTTPBadRequest('Bad share value')
if not source or not destination or not vo or not share:
raise HTTPBadRequest(
'Missing source, destination, vo and/or share')
source = urlparse(source)
if not source.scheme or not source.hostname:
raise HTTPBadRequest('Invalid source')
source = "%s://%s" % (source.scheme, source.hostname)
destination = urlparse(destination)
if not destination.scheme or not destination.hostname:
raise HTTPBadRequest('Invalid source')
destination = "%s://%s" % (destination.scheme, destination.hostname)
try:
share_cfg = ShareConfig(source=source,
destination=destination,
vo=vo,
share=share)
Session.merge(share_cfg)
audit_configuration(
'share-set', 'Share %s, %s, %s has been set to %d' %
(source, destination, vo, share))
Session.commit()
except:
Session.rollback()
raise
return share
def delete_se_config(self, start_response):
"""
Delete the configuration for a given SE
"""
se = request.params.get('se', None)
if not se:
raise HTTPBadRequest('Missing storage (se)')
try:
Session.query(Se).filter(Se.storage == se).delete()
Session.query(OperationConfig).filter(
OperationConfig.host == se).delete()
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def unban_se(self, start_response):
"""
Unban a storage element
"""
storage = request.params.get('storage', None)
if not storage:
raise HTTPBadRequest('Missing storage parameter')
banned = Session.query(BannedSE).get(storage)
if banned:
try:
Session.delete(banned)
Session.commit()
except Exception:
Session.rollback()
log.warn("Storage %s unbanned" % storage)
else:
log.warn("Unban of storage %s without effect" % storage)
start_response('204 No Content', [])
return ['']
def _ban_se(storage, vo_name, allow_submit, status, timeout):
"""
Mark in the db the given storage as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedSE()
banned.se = storage
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
banned.vo = vo_name
if allow_submit and status == 'WAIT':
banned.status = 'WAIT_AS'
else:
banned.status = status
banned.wait_timeout = timeout
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def delete(self, dlg_id, start_response):
"""
Delete the delegated credentials from the database
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
cred = Session.query(Credential).get((user.delegation_id, user.user_dn))
if not cred:
raise HTTPNotFound('Delegated credentials not found')
else:
try:
Session.delete(cred)
Session.commit()
except Exception:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def _ban_se(storage, vo_name, allow_submit, status, message):
"""
Mark in the db the given storage as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedSE()
banned.se = storage
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
banned.vo = vo_name
banned.message = message
if allow_submit and status == 'WAIT':
banned.status = 'WAIT_AS'
else:
banned.status = status
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def delete(self, dlg_id, start_response):
"""
Delete the delegated credentials from the database
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
cred = Session.query(Credential).get((user.delegation_id, user.user_dn))
if not cred:
raise HTTPNotFound('Delegated credentials not found')
else:
try:
Session.delete(cred)
Session.commit()
except Exception:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def set_cloud_storage(self, start_response):
"""
Add or modify a cloud storage entry
"""
input_dict = get_input_as_dict(request)
if 'storage_name' not in input_dict:
raise HTTPBadRequest('Missing storage name')
storage = CloudStorage(
storage_name=input_dict.get('storage_name'),
app_key=input_dict.get('app_key', None),
app_secret=input_dict.get('app_secret', None),
service_api_url=input_dict.get('service_api_url', None)
)
try:
Session.merge(storage)
Session.commit()
except:
Session.rollback()
raise
start_response('201 Created', [])
return storage.storage_name
def unban_dn(self, start_response):
"""
Unban a user
"""
dn = request.params.get('user_dn', None)
if not dn:
raise HTTPBadRequest('Missing user_dn parameter')
banned = Session.query(BannedDN).get(dn)
if banned:
try:
Session.delete(banned)
Session.commit()
except Exception:
Session.rollback()
log.warn("User %s unbanned" % dn)
else:
log.warn("Unban of user %s without effect" % dn)
start_response('204 No Content', [])
return ['']
def delete_app(self, client_id):
"""
Delete an application from the database
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if app is None:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
try:
Session.delete(app)
Session.query(OAuth2Token).filter(OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(OAuth2Code.client_id == client_id).delete()
Session.commit()
except:
Session.rollback()
raise
log.info("Application removed: %s" % client_id)
return None
def _set_to_wait(storage=None, vo_name=None, timeout=0):
"""
Updates the transfers that have the given storage either in source or destination,
and belong to the given VO.
Returns the list of affected jobs ids.
"""
job_ids = Session.query(distinct(File.job_id))\
.filter((File.source_se == storage) | (File.dest_se == storage)).filter(File.file_state.in_(FileActiveStates))
if vo_name:
job_ids = job_ids.filter(File.vo_name == vo_name)
job_ids = map(lambda j: j[0], job_ids.all())
try:
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id).filter(File.file_state.in_(FileActiveStates))\
.update({'wait_timestamp': datetime.utcnow(), 'wait_timeout': timeout}, synchronize_session=False)
Session.commit()
Session.expire_all()
return job_ids
except Exception:
Session.rollback()
raise
try:
voms_client = voms.VomsClient(credential.proxy)
(new_proxy, new_termination_time) = voms_client.init(voms_list)
except voms.VomsException, e:
# Error generating the proxy because of the request itself
raise HTTPMethodFailure(str(e))
credential.proxy = new_proxy
credential.termination_time = new_termination_time
credential.voms_attrs = ' '.join(voms_list)
try:
Session.merge(credential)
Session.commit()
except Exception:
Session.rollback()
raise
start_response('203 Non-Authoritative Information', [('Content-Type', 'text/plain')])
return [str(new_termination_time)]
@require_certificate
def delegation_page(self):
"""
Render an HTML form to delegate the credentials
"""
user = request.environ['fts3.User.Credentials']
return render(
'/delegation.html', extra_vars={
'user': user,
'vos': self.vo_list,
def _cancel_transfers(storage=None, vo_name=None):
"""
Cancels the transfers that have the given storage either in source or destination,
and belong to the given VO.
Returns the list of affected jobs ids.
"""
affected_job_ids = set()
files = Session.query(File)\
.filter((File.source_se == storage) | (File.dest_se == storage))\
.filter(File.file_state.in_(FileActiveStates + ['NOT_USED']))
if vo_name:
files = files.filter(File.vo_name == vo_name)
now = datetime.utcnow()
try:
for file in files:
affected_job_ids.add(file.job_id)
# Cancel the affected file
file.file_state = 'CANCELED'
file.reason = 'Storage banned'
file.finish_time = now
Session.merge(file)
# If there are alternatives, enable them
Session.query(File).filter(File.job_id == file.job_id)\
.filter(File.file_index == file.file_index)\
.filter(File.file_state == 'NOT_USED').update({'file_state': 'SUBMITTED'})
# Or next queries will not see the changes!
Session.commit()
except Exception:
Session.rollback()
raise
# Set each job terminal state if needed
try:
for job_id in affected_job_ids:
reuse_flag = Session.query(Job.reuse_job).filter(Job.job_id == job_id)[0][0]
n_files = Session.query(func.count(distinct(File.file_id))).filter(File.job_id == job_id).all()[0][0]
n_canceled = Session.query(func.count(distinct(File.file_id)))\
.filter(File.job_id == job_id).filter(File.file_state == 'CANCELED').all()[0][0]
n_finished = Session.query(func.count(distinct(File.file_id)))\
.filter(File.job_id == job_id).filter(File.file_state == 'FINISHED').all()[0][0]
n_failed = Session.query(func.count(distinct(File.file_id)))\
.filter(File.job_id == job_id).filter(File.file_state == 'FAILED').all()[0][0]
n_terminal = n_canceled + n_finished + n_failed
# Job finished!
if n_terminal == n_files:
reason = None
Session.query(Job).filter(Job.job_id == job_id).update({
'job_state': 'CANCELED',
'job_finished': now,
'finish_time': now,
'reason': reason
})
Session.query(File).filter(File.job_id == job_id).update({
'job_finished': now
})
Session.commit()
except Exception:
Session.rollback()
raise
return affected_job_ids
