def _cancel_jobs(dn):
"""
Cancel all jobs that belong to dn.
Returns the list of affected jobs ids.
"""
jobs = Session.query(Job.job_id).filter(Job.job_state.in_(JobActiveStates),
Job.user_dn == dn,
Job.job_finished == None)
job_ids = map(lambda j: j[0], jobs)
try:
now = datetime.utcnow()
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id).filter(File.file_state.in_(FileActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'User banned',
'finish_time': now
}, synchronize_session=False)
Session.query(Job).filter(Job.job_id == job_id)\
.update({
'job_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now
}, synchronize_session=False)
Session.commit()
Session.expire_all()
return job_ids
except Exception:
Session.rollback()
raise
def setUp(self):
super(TestConfigSe, self).setUp()
self.setup_gridsite_environment()
Session.query(Optimizer).delete()
Session.query(ConfigAudit).delete()
Session.query(OperationConfig).delete()
Session.query(Se).delete()
Session.commit()
self.host_config = {
'operations': {
'atlas': {
'delete': 22,
'staging': 32,
},
'dteam': {
'delete': 10,
'staging': 11
}
},
'se_info': {
'ipv6': 1,
'outbound_max_active': 55,
'inbound_max_active': 11,
'inbound_max_throughput': 33,
'se_metadata': 'metadata'
}
}
def get_access_granted(self):
dropbox_user_info = self._get_dropbox_user_info()
if not dropbox_user_info:
raise HTTPBadRequest('No registered user for the service "%s" has been found' % self.service)
dropbox_info = self._get_dropbox_info()
if not dropbox_info:
raise HTTPNotFound('Dropbox info not found in the databse')
access_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/access_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="' +
dropbox_info.app_key + '", oauth_token="' + dropbox_user_info.request_token +
'", oauth_signature="' + dropbox_info.app_secret + '&' + dropbox_user_info.request_token_secret + '"',
None
)
# It returns: oauth_token=<access-token>&oauth_token_secret=<access-token-secret>&uid=<user-id>
access_tokens = access_tokens.split('&')
dropbox_user_info.access_token = access_tokens[1].split('=')[1]
dropbox_user_info.access_token_secret = access_tokens[0].split('=')[1]
try:
Session.add(dropbox_user_info)
Session.commit()
except:
Session.rollback()
raise
return access_tokens
def tearDown(self):
super(TestOAuth2, self).tearDown()
Session.query(OAuth2Application).delete()
Session.query(OAuth2Token).delete()
Session.query(OAuth2Code).delete()
Session.query(AuthorizationByDn).delete()
Session.commit()
def _prepare_and_test_created_jobs_to_cancel(self, files_per_job=8):
"""
Helper function to prepare and test created jobs for cancel tests
"""
job_ids = list()
for i in range(len(FileActiveStates) + len(FileTerminalStates)):
job_ids.append(self._submit(files_per_job))
i = 0
for state in FileActiveStates + FileTerminalStates:
job = Session.query(Job).get(job_ids[i])
i += 1
if state == 'STARTED':
job.job_state = 'STAGING'
else:
job.job_state = state
for f in job.files:
f.file_state = state
Session.merge(job)
Session.commit()
i = 0
for state in FileActiveStates + FileTerminalStates:
job = Session.query(Job).get(job_ids[i])
state_job = state
if state == 'STARTED':
state_job = 'STAGING'
self.assertEqual(job.job_state, state_job)
for f in job.files:
self.assertEqual(f.file_state, state)
i += 1
return job_ids
def _cancel_jobs(dn):
"""
Cancel all jobs that belong to dn.
Returns the list of affected jobs ids.
"""
jobs = Session.query(Job.job_id).filter(Job.job_state.in_(JobActiveStates)).filter(Job.user_dn == dn)
job_ids = map(lambda j: j[0], jobs)
try:
now = datetime.utcnow()
for job_id in job_ids:
Session.query(File).filter(File.job_id == job_id).filter(File.file_state.in_(FileActiveStates))\
.update({
'file_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now, 'finish_time': now
}, synchronize_session=False)
Session.query(Job).filter(Job.job_id == job_id)\
.update({
'job_state': 'CANCELED', 'reason': 'User banned',
'job_finished': now, 'finish_time': now
}, synchronize_session=False)
Session.commit()
Session.expire_all()
return job_ids
except Exception:
Session.rollback()
raise
def get_access_requested(self):
dropbox_info = self._get_dropbox_info()
request_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/request_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT",'
'oauth_consumer_key="' + dropbox_info.app_key + '", oauth_signature="' + dropbox_info.app_secret + '&"',
None
)
# It returns: oauth_token_secret=b9q1n5il4lcc&oauth_token=mh7an9dkrg59
tokens = request_tokens.split('&')
newuser = CloudStorageUser(
user_dn=self.user_dn,
cloudStorage_name=dropbox_info.cloudStorage_name,
request_token=tokens[1].split('=')[1],
request_token_secret=tokens[0].split('=')[1]
)
try:
Session.add(newuser)
Session.commit()
except:
Session.rollback()
raise
return request_tokens
def fill_optimizer():
evolution = OptimizerEvolution(datetime=datetime.datetime.utcnow(),
source_se='http://site01.es',
dest_se='http://dest.ch',
success=90,
active=10,
throughput=10)
Session.add(evolution)
evolution = OptimizerEvolution(datetime=datetime.datetime.utcnow(),
source_se='http://site02.ch',
dest_se='http://dest.ch',
success=95,
active=10,
throughput=15)
Session.add(evolution)
evolution = OptimizerEvolution(datetime=datetime.datetime.utcnow(),
source_se='http://site03.fr',
dest_se='http://dest.ch',
success=100,
active=10,
throughput=20)
Session.add(evolution)
Session.commit()
def fill_activities():
activity = ActivityShare(vo='testvo',
activity_share=json.dumps({
"data brokering":
0.3,
"data consolidation":
0.4,
"default":
0.02,
"express":
0.4,
"functional test":
0.2,
"production":
0.5,
"production input":
0.25,
"production output":
0.25,
"recovery":
0.4,
"staging":
0.5,
"t0 export":
0.7,
"t0 tape":
0.7,
"user subscriptions":
0.1
}))
Session.add(activity)
Session.commit()
def set_activity_shares(self):
"""
Set a new/modify an activity share
"""
input_dict = get_input_as_dict(request)
if not input_dict.get('vo', None):
raise HTTPBadRequest('Missing VO')
if not input_dict.get('share', None):
raise HTTPBadRequest('Missing share')
if 'active' not in input_dict:
input_dict['active'] = True
input_dict['share'] = _normalize_activity_share_format(
input_dict['share'])
# Make sure the share weights are numbers
for entry in input_dict['share']:
for key, value in entry.iteritems():
if not type(value) in (float, int):
raise HTTPBadRequest('Share weight must be a number')
try:
activity_share = ActivityShare(vo=input_dict['vo'],
active=input_dict['active'],
activity_share=input_dict['share'])
Session.merge(activity_share)
audit_configuration('activity-share', json.dumps(input_dict))
Session.commit()
except ValueError, e:
raise HTTPBadRequest(str(e))
def test_expired(self):
"""
Get a token, the token expires, so it should be denied
"""
client_id, access_token, refresh_token, expires = self.test_get_token()
del self.app.extra_environ['GRST_CRED_AURI_0']
response = self.app.get(
url="/whoami",
headers={'Authorization': str('Bearer %s' % access_token)},
status=200
)
whoami = json.loads(response.body)
self.assertEqual('oauth2', whoami['method'])
token = Session.query(OAuth2Token).get((client_id, refresh_token))
token.expires = datetime.utcnow() - timedelta(hours=1)
Session.merge(token)
Session.commit()
response = self.app.get(
url="/whoami",
headers={'Authorization': str('Bearer %s' % access_token)},
status=403
)
def test_set_voms(self):
"""
The server must regenerate a proxy with VOMS extensions
Need a real proxy for this one
"""
self.setup_gridsite_environment()
creds = self.get_user_credentials()
# Need to push a real proxy :/
proxy_pem = self.get_real_x509_proxy()
if proxy_pem is None:
raise SkipTest(
'Could not get a valid real proxy for test_set_voms')
proxy = Credential()
proxy.dn = creds.user_dn
proxy.dlg_id = creds.delegation_id
proxy.termination_time = datetime.utcnow() + timedelta(hours=1)
proxy.proxy = proxy_pem
Session.merge(proxy)
Session.commit()
# Now, request the voms extensions
self.app.post_json(url="/delegation/%s/voms" % creds.delegation_id,
params=['dteam:/dteam/Role=lcgadmin'],
status=203)
# And validate
proxy2 = Session.query(Credential).get(
(creds.delegation_id, creds.user_dn))
self.assertNotEqual(proxy.proxy, proxy2.proxy)
self.assertEqual('dteam:/dteam/Role=lcgadmin', proxy2.voms_attrs)
def delete_share(self, start_response):
"""
Delete a share
"""
input_dict = get_input_as_dict(request, from_query=True)
source = input_dict.get('source')
destination = input_dict.get('destination')
vo = input_dict.get('vo')
if not source or not destination or not vo:
raise HTTPBadRequest('Missing source, destination and/or vo')
try:
share = Session.query(ShareConfig).get((source, destination, vo))
if share:
Session.delete(share)
audit_configuration(
'share-delete', 'Share %s, %s, %s has been deleted' %
(source, destination, vo))
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def setUp(self):
super(TestDrain, self).setUp()
self.setup_gridsite_environment()
Session.add(
Host(hostname='host1.cern.ch',
service_name='fts3',
beat=datetime.utcnow(),
drain=False))
Session.add(
Host(hostname='host1.cern.ch',
service_name='bringonline',
beat=datetime.utcnow(),
drain=False))
Session.add(
Host(hostname='host2.cern.ch',
service_name='fts3',
beat=datetime.utcnow(),
drain=False))
Session.add(
Host(hostname='host2.cern.ch',
service_name='bringonline',
beat=datetime.utcnow(),
drain=False))
Session.commit()
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
if credential_cache is None or credential_cache.cert_request is None:
(x509_request, private_key) = _generate_proxy_request()
credential_cache = CredentialCache(dlg_id=user.delegation_id, dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok', [('X-Delegation-ID', credential_cache.dlg_id),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def delete_app(self, client_id):
"""
Delete an application from the database
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if app is None:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
try:
Session.delete(app)
Session.query(OAuth2Token).filter(
OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(
OAuth2Code.client_id == client_id).delete()
Session.commit()
except:
Session.rollback()
raise
log.info("Application removed: %s" % client_id)
return None
def add_user_to_cloud_storage(self, storage_name, start_response):
"""
Add a user or a VO credentials to the storage
"""
storage = Session.query(CloudStorage).get(storage_name)
if not storage:
raise HTTPNotFound('The storage does not exist')
input_dict = get_input_as_dict(request)
if not input_dict.get('user_dn', None) and not input_dict.get('vo_name', None):
raise HTTPBadRequest('One of user_dn or vo_name must be specified')
elif input_dict.get('user_dn', None) and input_dict.get('vo_name', None):
raise HTTPBadRequest('Only one of user_dn or vo_name must be specified')
cuser = CloudStorageUser(
user_dn=input_dict.get('user_dn', ''),
storage_name=storage_name,
access_token=input_dict.get('access_key', input_dict.get('access_token', None)),
access_token_secret=input_dict.get('secret_key', input_dict.get('access_token_secret', None)),
request_token=input_dict.get('request_token'),
request_token_secret=input_dict.get('request_token_secret'),
vo_name=input_dict.get('vo_name', ''),
)
try:
Session.merge(cuser)
Session.commit()
except:
Session.rollback()
raise
start_response('201 Created', [])
return dict(storage_name=cuser.storage_name, user_dn=cuser.user_dn, vo_name=cuser.vo_name)
def get_access_requested(self):
dropbox_info = self._get_dropbox_info()
request_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/request_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT",'
'oauth_consumer_key="' + dropbox_info.app_key + '", oauth_signature="' + dropbox_info.app_secret + '&"',
None
)
# It returns: oauth_token_secret=b9q1n5il4lcc&oauth_token=mh7an9dkrg59
tokens = request_tokens.split('&')
newuser = CloudStorageUser(
user_dn=self.user_dn,
storage_name=dropbox_info.storage_name,
request_token=tokens[1].split('=')[1],
request_token_secret=tokens[0].split('=')[1],
vo_name=''
)
try:
Session.add(newuser)
Session.commit()
except:
Session.rollback()
raise
return request_tokens
def test_set_voms(self):
"""
The server must regenerate a proxy with VOMS extensions
Need a real proxy for this one
"""
self.setup_gridsite_environment()
creds = self.get_user_credentials()
# Need to push a real proxy :/
proxy_pem = self.get_real_x509_proxy()
if proxy_pem is None:
raise SkipTest('Could not get a valid real proxy for test_set_voms')
proxy = Credential()
proxy.dn = creds.user_dn
proxy.dlg_id = creds.delegation_id
proxy.termination_time = datetime.utcnow() + timedelta(hours=1)
proxy.proxy = proxy_pem
Session.merge(proxy)
Session.commit()
# Now, request the voms extensions
self.app.post(url="/delegation/%s/voms" % creds.delegation_id,
content_type='application/json',
params=json.dumps(['dteam:/dteam/Role=lcgadmin']),
status=203)
# And validate
proxy2 = Session.query(Credential).get((creds.delegation_id, creds.user_dn))
self.assertNotEqual(proxy.proxy, proxy2.proxy)
self.assertEqual('dteam:/dteam/Role=lcgadmin', proxy2.voms_attrs)
def update_app(self, client_id):
"""
Update an application
"""
user = pylons.request.environ['fts3.User.Credentials']
app = Session.query(OAuth2Application).get(client_id)
if not app:
raise HTTPNotFound('Application not found')
if app.owner != user.user_dn:
raise HTTPForbidden()
if pylons.request.headers['Content-Type'].startswith('application/json'):
fields = json.loads(pylons.request.body)
else:
fields = pylons.request.POST
try:
if 'delete' not in fields:
app.description = fields.get('description', '')
app.website = fields.get('website', '')
app.redirect_to = fields.get('redirect_to', '')
Session.merge(app)
Session.commit()
redirect(url_for(controller='oauth2', action='get_app'), code=HTTPSeeOther.code)
else:
Session.delete(app)
Session.query(OAuth2Token).filter(OAuth2Token.client_id == client_id).delete()
Session.query(OAuth2Code).filter(OAuth2Code.client_id == client_id).delete()
Session.commit()
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
except:
Session.rollback()
raise
def setUp(self):
super(TestConfigShares, self).setUp()
self.setup_gridsite_environment()
Session.query(ServerConfig).delete()
Session.query(ConfigAudit).delete()
Session.query(OperationConfig).delete()
Session.commit()
def test_cancel_some_terminal(self):
"""
Cancel a job with some files in terminal state
"""
job_id = self._submit(10)
job = Session.query(Job).get(job_id)
job.job_state = 'ACTIVE'
for f in job.files:
if f.file_id % 2 == 0:
f.file_state = 'FINISHED'
Session.merge(job)
Session.commit()
job = self.app.delete(url="/jobs/%s" % job_id, status=200).json
self.assertEqual(job['job_id'], job_id)
self.assertEqual(job['job_state'], 'CANCELED')
self.assertEqual(job['reason'], 'Job canceled by the user')
# Is it in the database?
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_state, 'CANCELED')
for f in job.files:
if f.file_id % 2 == 0:
self.assertEqual(f.file_state, 'FINISHED')
self.assertNotEqual(f.reason, 'Job canceled by the user')
else:
self.assertEqual(f.file_state, 'CANCELED')
def test_cancel_terminal(self):
"""
Cancel a job with files in terminal state
"""
job_id = self._submit()
job = Session.query(Job).get(job_id)
job.job_state = 'FINISHED'
for f in job.files:
f.file_state = 'FINISHED'
Session.merge(job)
Session.commit()
answer = self.app.delete(url="/jobs/%s" % job_id,
status=200)
job = json.loads(answer.body)
self.assertEqual(job['job_id'], job_id)
self.assertEqual(job['job_state'], 'FINISHED')
self.assertNotEqual(job['reason'], 'Job canceled by the user')
# Is it in the database?
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_state, 'FINISHED')
for f in job.files:
self.assertEqual(f.file_state, 'FINISHED')
def test_expired(self):
"""
Get a token, the token expires, so it should be denied
"""
raise SkipTest('Disabled as code is not used atm')
client_id, access_token, refresh_token, expires = self.test_get_token()
del self.app.extra_environ['GRST_CRED_AURI_0']
whoami = self.app.get(url="/whoami",
headers={
'Authorization':
str('Bearer %s' % access_token)
},
status=200).json
self.assertEqual('oauth2', whoami['method'])
token = Session.query(OAuth2Token).get((client_id, refresh_token))
token.expires = datetime.utcnow() - timedelta(hours=1)
Session.merge(token)
Session.commit()
self.app.get(
url="/whoami",
headers={'Authorization': str('Bearer %s' % access_token)},
status=403)
def remove_authz(self, start_response):
"""
Revoke access for a DN for a given operation, or all
"""
input_dict = get_input_as_dict(request, from_query=True)
dn = input_dict.get('dn')
op = input_dict.get('operation')
if not dn:
raise HTTPBadRequest('Missing DN parameter')
to_be_removed = Session.query(AuthorizationByDn).filter(
AuthorizationByDn.dn == dn)
if op:
to_be_removed = to_be_removed.filter(
AuthorizationByDn.operation == op)
try:
to_be_removed.delete()
if op:
audit_configuration('revoke', '%s revoked for "%s"' % (op, dn))
else:
audit_configuration('revoke', 'All revoked for "%s"' % dn)
Session.commit()
except:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def test_get_retries(self):
"""
Get the retries for a file, forcing one
"""
self.setup_gridsite_environment()
self.push_delegation()
job_id = self._submit()
files = self.app.get(url="/jobs/%s/files" % job_id, status=200).json
file_id = files[0]['file_id']
retry = FileRetryLog()
retry.file_id = file_id
retry.attempt = 1
retry.datetime = datetime.utcnow()
retry.reason = 'Blahblahblah'
Session.merge(retry)
Session.commit()
retries = self.app.get(url="/jobs/%s/files/%d/retries" %
(job_id, file_id),
status=200).json
self.assertEqual(1, len(retries))
self.assertEqual(1, retries[0]['attempt'])
self.assertEqual('Blahblahblah', retries[0]['reason'])
def test_cancel_running(self):
"""
Cancel a job, but the transfer is running (pid is set)
"""
job_id = self._submit()
# Add pid
transfer = Session.query(File).filter(File.job_id == job_id).first()
transfer.pid = 1234
Session.merge(transfer)
Session.commit()
job = self.app.delete(url="/jobs/%s" % job_id, status=200).json
self.assertEqual(job['job_id'], job_id)
self.assertEqual(job['job_state'], 'CANCELED')
self.assertEqual(job['reason'], 'Job canceled by the user')
# Is it in the database?
job = Session.query(Job).get(job_id)
self.assertEqual(job.job_state, 'CANCELED')
self.assertNotEqual(None, job.job_finished)
for f in job.files:
self.assertEqual(f.file_state, 'CANCELED')
self.assertEqual(None, f.finish_time)
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
if credential_cache is None or credential_cache.cert_request is None:
(x509_request, private_key) = _generate_proxy_request()
credential_cache = CredentialCache(dlg_id=user.delegation_id, dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok', [('X-Delegation-ID', str(credential_cache.dlg_id)),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def set_drain(self):
"""
Set the drain status of a server
"""
input_dict = get_input_as_dict(request)
hostname = input_dict.get('hostname', None)
drain = input_dict.get('drain', True)
if not isinstance(drain, bool) or not isinstance(hostname, basestring):
raise HTTPBadRequest('Invalid drain request')
entries = Session.query(Host).filter(Host.hostname == hostname).all()
if not entries:
raise HTTPBadRequest('Host not found')
try:
audit_configuration(
'drain',
'Turning drain %s the drain mode for %s' % (drain, hostname))
for entry in entries:
entry.drain = drain
Session.merge(entry)
Session.commit()
except:
Session.rollback()
raise
def cancel(self, id, **kwargs): """DELETE /jobs/id: Delete an existing item""" job = self._getJob(id) if job.job_state in JobActiveStates: now = datetime.now() job.job_state = 'CANCELED' job.finish_time = now job.job_finished = now job.reason = 'Job canceled by the user' for f in job.files: if f.file_state in JobActiveStates: f.file_state = 'CANCELED' f.job_finished = now f.finish_time = now f.reason = 'Job canceled by the user' Session.merge(job) Session.commit() job = self._getJob(id) files = job.files return job
def test_get_retries(self):
"""
Get the retries for a file, forcing one
"""
self.setup_gridsite_environment()
self.push_delegation()
job_id = self._submit()
answer = self.app.get(url="/jobs/%s/files" % job_id, status=200)
files = json.loads(answer.body)
file_id = files[0]['file_id']
retry = FileRetryLog()
retry.file_id = file_id
retry.attempt = 1
retry.datetime = datetime.utcnow()
retry.reason = 'Blahblahblah'
Session.merge(retry)
Session.commit()
answer = self.app.get(url="/jobs/%s/files/%d/retries" % (job_id, file_id), status=200)
retries = json.loads(answer.body)
self.assertEqual(1, len(retries))
self.assertEqual(1, retries[0]['attempt'])
self.assertEqual('Blahblahblah', retries[0]['reason'])
def popDelegation(self): cred = self.getUserCredentials() if cred and cred.delegation_id: delegated = Session.query(Credential).get((cred.delegation_id, cred.user_dn)) if delegated: Session.delete(delegated) Session.commit()
def get_access_granted(self):
dropbox_user_info = self._get_dropbox_user_info()
if not dropbox_user_info:
raise HTTPBadRequest('No registered user for the service "%s" has been found' % self.service)
dropbox_info = self._get_dropbox_info()
if not dropbox_info:
raise HTTPNotFound('Dropbox info not found in the database')
access_tokens = self._make_call(
dropboxApiEndpoint + "/1/oauth/access_token",
'OAuth oauth_version="1.0", oauth_signature_method="PLAINTEXT", oauth_consumer_key="' +
dropbox_info.app_key + '", oauth_token="' + dropbox_user_info.request_token +
'", oauth_signature="' + dropbox_info.app_secret + '&' + dropbox_user_info.request_token_secret + '"',
None
)
# It returns: oauth_token=<access-token>&oauth_token_secret=<access-token-secret>&uid=<user-id>
access_tokens = access_tokens.split('&')
dropbox_user_info.access_token = access_tokens[1].split('=')[1]
dropbox_user_info.access_token_secret = access_tokens[0].split('=')[1]
try:
Session.add(dropbox_user_info)
Session.commit()
except:
Session.rollback()
raise
return access_tokens
def cancel(self, id, **kwargs):
"""DELETE /jobs/id: Delete an existing item"""
job = self._getJob(id)
if job.job_state in JobActiveStates:
now = datetime.now()
job.job_state = 'CANCELED'
job.finish_time = now
job.job_finished = now
job.reason = 'Job canceled by the user'
for f in job.files:
if f.file_state in JobActiveStates:
f.file_state = 'CANCELED'
f.job_finished = now
f.finish_time = now
f.reason = 'Job canceled by the user'
Session.merge(job)
Session.commit()
job = self._getJob(id)
files = job.files
return job
def submit(self):
"""
Submits a new job
It returns the information about the new submitted job. To know the format for the
submission, /api-docs/schema/submit gives the expected format encoded as a JSON-schema.
It can be used to validate (i.e in Python, jsonschema.validate)
"""
# First, the request has to be valid JSON
submitted_dict = get_input_as_dict(request)
# The auto-generated delegation id must be valid
user = request.environ['fts3.User.Credentials']
credential = Session.query(Credential).get((user.delegation_id, user.user_dn))
if credential is None:
raise HTTPAuthenticationTimeout('No delegation found for "%s"' % user.user_dn)
if credential.expired():
remaining = credential.remaining()
seconds = abs(remaining.seconds + remaining.days * 24 * 3600)
raise HTTPAuthenticationTimeout(
'The delegated credentials expired %d seconds ago (%s)' % (seconds, user.delegation_id)
)
if user.method != 'oauth2' and credential.remaining() < timedelta(hours=1):
raise HTTPAuthenticationTimeout(
'The delegated credentials has less than one hour left (%s)' % user.delegation_id
)
# Populate the job and files
populated = JobBuilder(user, **submitted_dict)
log.info("%s (%s) is submitting a transfer job" % (user.user_dn, user.vos[0]))
# Insert the job
try:
try:
Session.execute(Job.__table__.insert(), [populated.job])
except IntegrityError:
raise HTTPConflict('The sid provided by the user is duplicated')
if len(populated.files):
Session.execute(File.__table__.insert(), populated.files)
if len(populated.datamanagement):
Session.execute(DataManagement.__table__.insert(), populated.datamanagement)
Session.flush()
Session.commit()
except IntegrityError as err:
Session.rollback()
raise HTTPConflict('The submission is duplicated '+ str(err))
except:
Session.rollback()
raise
# Send messages
# Need to re-query so we get the file ids
job = Session.query(Job).get(populated.job_id)
for i in range(len(job.files)):
try:
submit_state_change(job, job.files[i], populated.files[0]['file_state'])
except Exception, e:
log.warning("Failed to write state message to disk: %s" % e.message)
def popDelegation(self):
cred = self.getUserCredentials()
if cred and cred.delegation_id:
delegated = Session.query(Credential).get(
(cred.delegation_id, cred.user_dn))
if delegated:
Session.delete(delegated)
Session.commit()
def request(self, dlg_id, start_response):
"""
First step of the delegation process: get a certificate request
The returned certificate request must be signed with the user's original
credentials.
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden(
'The requested ID and the credentials ID do not match')
credential_cache = Session.query(CredentialCache)\
.get((user.delegation_id, user.user_dn))
user_cert = self.certificate()
request_key_len = 2048
if user_cert:
user_key = X509.load_cert_string(user_cert)
request_key_len = user_key.get_pubkey().size() * 8
cached = credential_cache is not None and credential_cache.cert_request is not None
if cached:
cached_key_len = X509.load_request_string(
credential_cache.cert_request).get_pubkey().size() * 8
if cached_key_len != request_key_len:
cached = False
log.debug(
"Invalidating cache due to key length missmatch between client and cached certificates"
)
if not cached:
(x509_request,
private_key) = _generate_proxy_request(request_key_len)
credential_cache = CredentialCache(
dlg_id=user.delegation_id,
dn=user.user_dn,
cert_request=x509_request.as_pem(),
priv_key=private_key.as_pem(cipher=None),
voms_attrs=' '.join(user.voms_cred))
try:
Session.merge(credential_cache)
Session.commit()
except Exception:
Session.rollback()
raise
log.debug("Generated new credential request for %s" % dlg_id)
else:
log.debug("Using cached request for %s" % dlg_id)
start_response('200 Ok',
[('X-Delegation-ID', str(credential_cache.dlg_id)),
('Content-Type', 'text/plain')])
return [credential_cache.cert_request]
def register(self):
"""
Register a new third party application
"""
if pylons.request.content_type.split(
';')[0].strip() == 'application/json':
req = json.loads(pylons.request.body)
scopes = req.get('scope', list())
else:
req = pylons.request.POST
scopes = req.getall('scope')
if isinstance(scopes, basestring):
scopes = scopes.split(',')
if not req.get('name', None):
raise HTTPBadRequest('Missing application name')
if not req.get('website', None):
raise HTTPBadRequest('Missing application website')
if not req.get('redirect_to', None):
raise HTTPBadRequest('Missing redirect urls')
for s in scopes:
if str(s) not in VALID_OPERATIONS:
raise HTTPBadRequest('Invalid scope (%s)' % s)
user = pylons.request.environ['fts3.User.Credentials']
app_id = _generate_app_id()
app = OAuth2Application(client_id=app_id,
client_secret=_generate_app_secret(),
name=req['name'],
description=req.get('description', ''),
website=req['website'],
scope=scopes,
redirect_to=req['redirect_to'],
owner=user.user_dn)
try:
Session.merge(app)
Session.commit()
except IntegrityError:
Session.rollback()
raise HTTPForbidden('The name already exists')
except:
Session.rollback()
raise
log.info("New application registered: %s (%s)" % (req['name'], app_id))
if _accept_html(pylons.request.accept):
redirect(url_for(controller='oauth2', action='get_my_apps'),
code=HTTPSeeOther.code)
else:
pylons.response.status_int = HTTPCreated.code
pylons.response.headers['Content-Type'] = 'application/json'
return [to_json(app.client_id)]
def pop_delegation(self):
"""
Remove the mock proxy from the database
"""
cred = self.get_user_credentials()
if cred and cred.delegation_id:
delegated = Session.query(Credential).get((cred.delegation_id, cred.user_dn))
if delegated:
Session.delete(delegated)
Session.commit()
def setUp(self):
# Inject a Dropbox app
cs = CloudStorage(storage_name='DROPBOX',
app_key='1234',
app_secret='sssh',
service_api_url='https://api.dropbox.com')
Session.merge(cs)
Session.commit()
self.setup_gridsite_environment()
def remove_token(self):
info = self._get_dropbox_user_info()
if info is None:
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
try:
Session.delete(info)
Session.commit()
except:
Session.rollback()
raise
def cancel_files(self, job_id, file_ids):
"""
Cancel individual files - comma separated for multiple - within a job
"""
job = self._get_job(job_id)
if job.job_type != 'N':
raise HTTPBadRequest('Multihop or reuse jobs must be cancelled at once (%s)' % str(job.job_type))
file_ids = file_ids.split(',')
changed_states = list()
try:
# Mark files in the list as CANCELED
for file_id in file_ids:
file = Session.query(File).get(file_id)
if not file or file.job_id != job_id:
changed_states.append('File does not belong to the job')
continue
if file.file_state not in FileActiveStates:
changed_states.append(file.file_state)
continue
file.file_state = 'CANCELED'
file.finish_time = datetime.utcnow()
file.dest_surl_uuid = None
changed_states.append('CANCELED')
Session.merge(file)
# Mark job depending on the status of the rest of files
not_changed_states = map(lambda f: f.file_state, filter(lambda f: f.file_id not in file_ids, job.files))
all_states = not_changed_states + changed_states
# All files within the job have been canceled
if len(not_changed_states) == 0:
job.job_state = 'CANCELED'
job.cancel_job = True
job.job_finished = datetime.utcnow()
log.warning('Cancelling all remaining files within the job %s' % job_id)
# No files in non-terminal, mark the job as CANCELED too
elif len(filter(lambda s: s in FileActiveStates, all_states)) == 0:
log.warning('Cancelling a file within a job with others in terminal state (%s)' % job_id)
job.job_state = 'CANCELED'
job.cancel_job = True
job.job_finished = datetime.utcnow()
else:
log.warning('Cancelling files within a job with others still active (%s)' % job_id)
Session.merge(job)
Session.commit()
except:
Session.rollback()
raise
return changed_states if len(changed_states) > 1 else changed_states[0]
def tearDown(self):
"""
Called by the test framework at the end of each test
"""
Session.query(Credential).delete()
Session.query(CredentialCache).delete()
Session.query(FileRetryLog).delete()
Session.query(File).delete()
Session.query(Job).delete()
Session.query(OptimizerActive).delete()
Session.commit()
def pushDelegation(self, lifetime = timedelta(hours = 7)): creds = self.getUserCredentials() delegated = Credential() delegated.dlg_id = creds.delegation_id delegated.dn = creds.user_dn delegated.proxy = '-NOT USED-' delegated.voms_attrs = None delegated.termination_time = datetime.now() + lifetime Session.merge(delegated) Session.commit()
def test_ban_dn_submission(self):
"""
If a DN is banned, submissions from this user must not be accepted
"""
banned = BannedDN()
banned.dn = self.get_user_credentials().user_dn
Session.merge(banned)
Session.commit()
self.push_delegation()
self.app.post(url="/jobs", content_type='application/json', params='[]', status=403)
def setUp(self):
# Inject a Dropbox app
cs = CloudStorage(
cloudStorage_name='DROPBOX',
app_key='1234',
app_secret='sssh',
service_api_url='https://api.dropbox.com'
)
Session.merge(cs)
Session.commit()
self.setup_gridsite_environment()
def _ban_dn(dn):
"""
Mark in the db the given DN as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedDN()
banned.dn = dn
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def register(self):
"""
Register a new third party application
"""
if pylons.request.content_type.split(';')[0].strip() == 'application/json':
req = json.loads(pylons.request.body)
else:
req = pylons.request.POST
if not req.get('name', None):
raise HTTPBadRequest('Missing application name')
if not req.get('website', None):
raise HTTPBadRequest('Missing application website')
if not req.get('redirect_to', None):
raise HTTPBadRequest('Missing redirect urls')
user = pylons.request.environ['fts3.User.Credentials']
app_id = _generate_app_id()
app = OAuth2Application(
client_id=app_id,
client_secret=_generate_app_secret(),
name=req['name'],
description=req.get('description', ''),
website=req['website'],
redirect_to=req['redirect_to'],
owner=user.user_dn
)
try:
Session.merge(app)
Session.commit()
except IntegrityError:
Session.rollback()
raise HTTPForbidden('The name already exists')
except:
Session.rollback()
raise
log.info("New application registered: %s (%s)" % (req['name'], app_id))
if _accept_html(pylons.request.accept):
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
else:
pylons.response.status_int = HTTPCreated.code
pylons.response.headers['Content-Type'] = 'application/json'
return to_json(app.client_id)
def is_access_requested(self):
info = self._get_dropbox_user_info()
if info is None:
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
if info.is_registered():
res = self._get_content("/")
if res.startswith("401"):
try:
Session.delete(info)
Session.commit()
except:
Session.rollback()
raise
raise HTTPNotFound('No registered user for the service "%s" has been found' % self.service)
return info
def test_optimizer_respected(self):
"""
Submitting a job with an existing OptimizerActive entry must respect
the existing value
"""
self.test_submit()
# Set active to 20
oa = Session.query(OptimizerActive).get(('root://source.es', 'root://dest.ch'))
oa.active = 20
Session.merge(oa)
Session.flush()
Session.commit()
# Submit a job
self.test_submit()
# Make sure it is still 20!
oa2 = Session.query(OptimizerActive).get(('root://source.es', 'root://dest.ch'))
self.assertEqual(20, oa2.active)
def push_delegation(self, lifetime=timedelta(hours=7)):
"""
Push into the database a mock delegated credential
Args:
lifetime: The mock credential lifetime
"""
creds = self.get_user_credentials()
delegated = Credential()
delegated.dlg_id = creds.delegation_id
delegated.dn = creds.user_dn
delegated.proxy = '-NOT USED-'
delegated.voms_attrs = None
delegated.termination_time = datetime.utcnow() + lifetime
Session.merge(delegated)
Session.commit()
def _insert_job(self):
job = ArchivedJob()
job.job_id = '111-222-333'
job.job_state = 'CANCELED'
archived = ArchivedFile()
archived.job_id = job.job_id
archived.file_id = 1234
archived.file_state = 'CANCELED'
archived.source_se = 'srm://source'
archived.dest_se = 'srm://dest'
Session.merge(job)
Session.merge(archived)
Session.commit()
return job.job_id
def request(self, id, start_response):
user = request.environ['fts3.User.Credentials']
credentialCache = Session.query(CredentialCache).get((id, user.user_dn))
if credentialCache is None:
(proxyRequest, proxyKey) = generateProxyRequest(user.dn)
credentialCache = CredentialCache(dlg_id = user.delegation_id,
dn = user.user_dn,
cert_request = proxyRequest.as_pem(),
priv_key = proxyKey.as_pem(cipher = None),
voms_attrs = ' '.join(user.voms_cred))
Session.add(credentialCache)
Session.commit()
start_response('200 OK', [('X-Delegation-ID', credentialCache.dlg_id)])
return credentialCache.cert_request
def revoke_token(self, client_id):
"""
Current user revokes all tokens for a given application
"""
user = pylons.request.environ['fts3.User.Credentials']
try:
Session.query(OAuth2Token).filter(
(OAuth2Token.client_id == client_id) & (OAuth2Token.dlg_id == user.delegation_id)
).delete()
Session.query(OAuth2Code).filter(
(OAuth2Code.client_id == client_id) & (OAuth2Code.dlg_id == user.delegation_id)
)
Session.commit()
except:
Session.rollback()
raise
log.warning("User %s revoked application %s" % (user.user_dn, client_id))
redirect(url_for(controller='oauth2', action='get_my_apps'), code=HTTPSeeOther.code)
def insert_job(vo, source=None, destination=None, state='SUBMITTED', multiple=None,
duration=None, queued=None, thr=None, reason=None,
user_dn='/DC=ch/DC=cern/CN=Test User'):
assert(multiple is not None or (destination is not None and source is not None))
job = Job()
job.user_dn = user_dn
job.vo_name = vo
job.source_se = source
job.job_state = state
job.submit_time = datetime.utcnow()
if duration and queued:
job.finish_time = job.submit_time + timedelta(seconds=duration+queued)
elif duration:
job.finish_time = job.submit_time + timedelta(seconds=duration)
job.job_id = str(uuid.uuid4())
Session.merge(job)
if multiple is None:
multiple = [(source, destination)]
for (s, d) in multiple:
transfer = File()
transfer.job_id = job.job_id
transfer.vo_name = vo
transfer.source_se = s
transfer.source_surl = s + '/path'
transfer.dest_se = d
transfer.dest_surl = d + '/path'
transfer.file_state = state
if queued:
transfer.start_time = job.submit_time + timedelta(seconds=queued)
if duration:
transfer.tx_duration = duration
if reason:
transfer.reason = reason
if thr:
transfer.throughput = thr
Session.merge(transfer)
Session.commit()
return job.job_id
def unban_se(self, start_response):
"""
Unban a storage element
"""
storage = request.params.get('storage', None)
if not storage:
raise HTTPBadRequest('Missing storage parameter')
banned = Session.query(BannedSE).get(storage)
if banned:
try:
Session.delete(banned)
Session.commit()
except Exception:
Session.rollback()
log.warn("Storage %s unbanned" % storage)
else:
log.warn("Unban of storage %s without effect" % storage)
start_response('204 No Content', [])
return ['']
def credential(self, id, start_response):
user = request.environ['fts3.User.Credentials']
credentialCache = Session.query(CredentialCache).get((id, user.user_dn))
x509ProxyPEM = request.body
x509Proxy = X509.load_cert_string(x509ProxyPEM)
proxyExpirationTime = x509Proxy.get_not_after().get_datetime().replace(tzinfo = None)
x509FullProxyPEM = self._buildFullProxyPEM(x509ProxyPEM, credentialCache.priv_key)
credential = Credential(dlg_id = id,
dn = user.user_dn,
proxy = x509FullProxyPEM,
voms_attrs = credentialCache.voms_attrs,
termination_time = proxyExpirationTime)
Session.merge(credential)
Session.commit()
start_response('201 CREATED', [])
return ''
def _ban_se(storage, vo_name, allow_submit, status, timeout):
"""
Mark in the db the given storage as banned
"""
user = request.environ['fts3.User.Credentials']
banned = BannedSE()
banned.se = storage
banned.addition_time = datetime.utcnow()
banned.admin_dn = user.user_dn
banned.vo = vo_name
if allow_submit and status == 'WAIT':
banned.status = 'WAIT_AS'
else:
banned.status = status
banned.wait_timeout = timeout
try:
Session.merge(banned)
Session.commit()
except Exception:
Session.rollback()
raise
def delete(self, dlg_id, start_response):
"""
Delete the delegated credentials from the database
"""
user = request.environ['fts3.User.Credentials']
if dlg_id != user.delegation_id:
raise HTTPForbidden('The requested ID and the credentials ID do not match')
cred = Session.query(Credential).get((user.delegation_id, user.user_dn))
if not cred:
raise HTTPNotFound('Delegated credentials not found')
else:
try:
Session.delete(cred)
Session.commit()
except Exception:
Session.rollback()
raise
start_response('204 No Content', [])
return ['']
def unban_dn(self, start_response):
"""
Unban a user
"""
dn = request.params.get('user_dn', None)
if not dn:
raise HTTPBadRequest('Missing user_dn parameter')
banned = Session.query(BannedDN).get(dn)
if banned:
try:
Session.delete(banned)
Session.commit()
except Exception:
Session.rollback()
log.warn("User %s unbanned" % dn)
else:
log.warn("Unban of user %s without effect" % dn)
start_response('204 No Content', [])
return ['']