def get_service():
""" Returns core service data.
At the moment this mainly means the central service CA cert.
"""
log = current_app.log
ca_data = None
client_conf = getConfig("client")
ca_cert = client_conf.get("cafile", None)
if ca_cert:
try:
with open(ca_cert, "r") as ca_fd:
ca_data = ca_fd.read()
except Exception as err:
log.error("Failed to read cafile for service endpoint: %s",
str(err))
# Get the user endpoint
ep_conf = getConfig("endpoints")
user_ep = ep_conf.get("users", None)
# Build output dictionary
res = {}
if ca_data:
res["central_ca"] = ca_data
if user_ep:
res["user_ep"] = user_ep
if current_app.vo_list:
res["vos"] = current_app.vo_list
return jsonify(res)
def post_job():
"""Add a job."""
require_attrs('src_siteid')
Job = request.db.tables.Job # pylint: disable=invalid-name
user_id = HRService.check_token()
request.data['user_id'] = user_id
request.data['src_credentials'] = current_app.site_client\
.get_cred(request.data['src_siteid'], user_id)
if request.data['type'] == JobType.COPY:
require_attrs('dst_siteid')
request.data['dst_credentials'] = current_app.site_client\
.get_cred(request.data['dst_siteid'],
user_id)
elif request.data['type'] == JobType.RENAME and\
request.data.get('dst_siteid') != request.data['src_siteid']:
current_app.log.warn("dst_siteid (%s) != src_siteid (%s)",
request.data.get('dst_siteid'),
request.data['src_siteid'])
request.data['dst_siteid'] = request.data['src_siteid']
try:
job = Job(**request.data)
except ValueError as err:
abort(400, description=err.message)
except Exception as err: # pylint: disable=broad-except
abort(500, description=err.message)
try:
job.add()
except Exception as err: # pylint: disable=broad-except
abort(500, description=err.message)
return jsonify(job)
def get_site_list():
""" Get a list of all sites, although actually only returns
entires that the user can see (public + self-owned).
"""
log = current_app.log
db = request.db
user_id = SiteService.get_current_uid()
Site = db.tables.Site
site_list = []
sites = Site.query.all()
for site_entry in sites:
# Ensure user can only see the correct sites
is_owner = False
is_visible = False
if site_entry.site_owner == user_id:
is_owner = True
is_visible = True
elif site_entry.public:
is_visible = True
if not is_visible:
continue
# Build the output object
cur_site = {"is_owner": is_owner, "public": site_entry.public}
for key in ("site_id", "site_name", "site_desc", "def_path"):
cur_site[key] = getattr(site_entry, key)
site_list.append(cur_site)
log.info("Found %u sites for user id %u.", len(site_list), user_id)
return jsonify(site_list)
def get_job_status(job_id):
"""Get job status."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
# pylint: disable=no-member
return jsonify({'jobid': job.id, 'status': JobStatus(job.status).name})
def verify_token():
""" Return whether a client token is valid. """
if request.token_ok:
res = "Token OK! (%s)" % request.token
else:
res = "Token Missing!"
return jsonify(res)
def get_site(site_id):
""" Gets the full details for a given site ID.
Returns 404 if the site doesn't exist or if the user
isn't allowed to view this site.
"""
log = current_app.log
db = request.db
user_id = SiteService.get_current_uid()
Site = db.tables.Site
site = Site.query.filter_by(site_id=site_id).first_or_404()
is_owner = (site.site_owner == user_id)
if not (site.public or is_owner):
log.warn("User %u failed to get site %u details (no permission).",
user_id, site_id)
abort(404) # User isn't allowed to see this site
log.info("User %u got details of site %u (%s).", user_id, site_id,
site.site_name)
# Add the endpoint info
endpoints = []
for ep_info in site.endpoints:
endpoints.append(ep_info.ep_uri)
# Prepare the output data
dict_out = dict(site)
dict_out["is_owner"] = is_owner
dict_out["endpoints"] = endpoints
return jsonify(dict_out)
def verify_user():
"""
Verify users' email address. Data posted is a token which has to ve verified and unpacked.
The email address contained in the token is compared with the email stored in the DB.
User state is changed to 1 (verified) if successful.
:return:
"""
data = json.loads(request.data)
HRService._logger.info("Data received for validation: %s", data)
try:
mtoken = data['mailtoken']
plain = current_app.mail_token_service.check(mtoken)
HRService._logger.info("Mailer token verified OK: %s", plain)
# token checked for integrity, check if not expired
if HRUtils.is_date_passed(plain.get('expiry')):
HRService._logger.error("Email verification token expired.")
abort(400, "Bad token or already verified")
username = plain.get('email')
if not username:
HRService._logger.error(
"Email verification token does not contain user info.")
abort(400, "Bad token or already verified") # 500?
HRService.update_user_status(username, HRServiceUserState.VERIFIED)
response = jsonify({'Verified': True, 'username': username})
response.status_code = 201
return response
except ValueError as ve:
HRService._logger.error(
"Mailer token integrity verification failed (%s)", ve)
abort(400, "Bad token or already verified")
def resend_email():
"""
Re-send a verification email for registered but not verified users.
:return:
"""
data = json.loads(request.data)
if not 'email' in data:
HRService._logger.error("resend email request:no email supplied")
abort(400)
username = data['email']
HRService._logger.info("Re-sending verification email for user %s .",
username)
User = request.db.tables.User
user = User.query.filter_by(email=data['email']).first()
if not user:
# Raise an HTTPException with a 400 bad request status code
HRService._logger.error(
"resending email: requested user for id %s doesn't exist ",
username)
abort(400)
if user.state != 0:
HRService._logger.error(
"resending email: requested user for id %s is already verified ",
username)
abort(400, ' Bad request or user already verified')
HRService.email_user(user.email)
HRService._logger.info("user: %s: verification email sent. ",
user.email)
response = jsonify([{'MailSent': 'OK'}])
response.status_code = 200
return response
def hello():
"""
Ping-like method.
:return: nice greeting ;-)
"""
return jsonify('User Service Desk at your service !\n')
def add_user():
"""
Add a new user.
:return: json document with added user data. An HTTPException with a
403 status code is thrown when the user already exists. User
email address and a password are obligatory (404 is emitted
otherwise).
"""
data = json.loads(request.data)
if not 'email' in data:
HRService._logger.error("add user request:no email supplied")
abort(400)
if not 'password' in data:
HRService._logger.error("add user request:no password supplied")
abort(400)
if not HRService.check_passwd(data['password']):
return "Password supplied is too short (lower character limit is %d)" \
% current_app.pwd_len, 400
data['password'] = hash_pass(data['password'])
User = request.db.tables.User
data.pop('last_login', None)
data.pop('date_created', None)
data.pop('date_modified', func.current_timestamp())
data.pop('state', 0)
# user = User.from_json(json.dumps(data))
user = User(**data)
db = request.db
try:
# user.save(db)
db.session.add(user)
user_id = db.session.query(
User.id).filter_by(email=data['email']).scalar()
# email the user
HRService.email_user(user.email)
HRService._logger.info("user: %s: verification email sent. ",
user.email)
db.session.commit()
except RuntimeError as r_error:
db.session.rollback()
HRService._logger.error(
"Runtime error when trying to send an email\n: %s", r_error)
HRService._logger.error("User %s not added.", user.email)
abort(500, 'The server could not send the verification email.')
except Exception:
HRService._logger.exception("Failed to add user: %s", user.name)
db.session.rollback()
abort(403) # 500 ?
# dict
response = jsonify(user)
response.status_code = 201
return response
def get_element(job_id, element_id):
"""Get all jobs for a user."""
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
element = JobElement.query.filter_by(id=element_id, job_id=job_id)\
.join(JobElement.job)\
.filter_by(user_id=HRService.check_token())\
.first_or_404()
return jsonify(element)
def get_cred(site_id, user_id):
""" Get a credential for a user at a specific site. """
log = current_app.log
db = request.db
Cred = db.tables.Cred
cred = Cred.query.filter_by(cred_owner=user_id,
site_id=site_id).first_or_404()
log.info("Fetched cred for user %u at site %u.", user_id, site_id)
return jsonify(cred.cred_value)
def test_output(self):
# check the token is passed
methodmock = mock.MagicMock()
methodmock.return_value = jsonify({})
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.get_output': methodmock}):
response = self._inst.output(1)
self.assertTrue(methodmock.called)
self.assertEqual(response, {})
def test_remove(self):
args = {'siteid': 12, 'filepath': '/data/somefile'}
removemock = mock.MagicMock()
removemock.return_value = jsonify(args)
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.remove': removemock}):
response = self._inst.remove(**args)
self.assertTrue(removemock.called)
self.assertIsInstance(response, dict)
self.assertEqual(response, args)
def get_output(job_id):
"""Get job output."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
log_filebase = os.path.join(current_app.workqueueservice_workerlogs,
job.log_uid[:2], job.log_uid)
elements_list = []
for element in job.elements:
# pylint: disable=no-member
attempt_output = {
'jobid': job_id,
'elementid': element.id,
'type': JobType(element.type).name
}
attempt_list = []
attempts = element.attempts
element_log_filebase = os.path.join(log_filebase, str(element.id))
for attempt in xrange(
1, attempts): # previous attempts, presumably failed ones
failed_output = dict(attempt_output,
attempt=attempt,
status=JobStatus.FAILED.name)
log_filename = os.path.join(element_log_filebase,
"attempt%i.log" % attempt)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
failed_output.update(log=log)
attempt_list.append(failed_output)
if attempts:
status = JobStatus.FAILED
if element.status == JobStatus.DONE:
status = JobStatus.DONE
last_output = dict(attempt_output,
attempt=attempts,
status=status.name)
log_filename = os.path.join(element_log_filebase,
"attempt%i.log" % attempts)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
last_output.update(log=log)
if status == JobStatus.DONE and element.type == JobType.LIST:
last_output.update(listing=element.listing)
attempt_list.append(last_output)
elements_list.append(attempt_list)
return jsonify(elements_list)
def test_run(self):
workload = [{
'id':
1,
'user_id':
9,
'type':
JobType.LIST,
'status':
JobStatus.SUBMITTED,
'priority':
5,
'protocol':
JobProtocol.DUMMY,
'src_siteid':
12,
'src_filepath':
'/data/somefile',
'src_credentials':
'somesecret',
'dst_credentials':
'someothersecret',
'extra_opts': {},
'elements': [{
"id": 0,
"job_id": 1,
"type": JobType.LIST,
"src_filepath": "/some/file",
"token": 'secret_token'
}]
}]
getjobmock = mock.MagicMock()
outputmock = mock.MagicMock()
getjobmock.return_value = jsonify(workload)
outputmock.return_value = '', 200
with mock.patch.dict(self._service.view_functions, {'WorkqueueService.get_next_job': getjobmock,
'WorkqueueService.return_output': outputmock}),\
mock.patch.object(self._inst._site_client, 'get_endpoints') as mock_get_endpoints,\
mock.patch('pdm.workqueue.Worker.X509Utils.add_ca_to_dir') as mock_ca2dir:
mock_get_endpoints.return_value = {
'endpoints': ['blah1', 'blah2', 'blah3'],
'cas': ['blah blah', 'la la']
}
mock_ca2dir.return_value = '/tmp/somecadir'
self._inst.run()
self.assertTrue(getjobmock.called)
self.assertTrue(outputmock.called)
self.assertTrue(mock_get_endpoints.called)
self.assertEqual(mock_get_endpoints.call_count, 1)
self.assertTrue(mock_ca2dir.called)
def test_copy(self):
args = {
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile'
}
copymock = mock.MagicMock()
copymock.return_value = jsonify(args)
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.copy': copymock}):
response = self._inst.copy(**args)
self.assertTrue(copymock.called)
self.assertIsInstance(response, dict)
self.assertEqual(response, args)
def get_jobs():
"""Get all jobs for a user."""
Job = request.db.tables.Job # pylint: disable=invalid-name
jobs = []
for job in Job.query.filter_by(user_id=HRService.check_token()).all():
elements = job.elements
status_counter = Counter(element.status for element in elements)
new_job = job.encode_for_json()
new_job.update(num_elements=len(elements),
num_new=status_counter[JobStatus.NEW],
num_done=status_counter[JobStatus.DONE],
num_failed=status_counter[JobStatus.FAILED],
num_submitted=status_counter[JobStatus.SUBMITTED],
num_running=status_counter[JobStatus.RUNNING])
jobs.append(new_job)
return jsonify(jobs)
def get_session_info(site_id):
""" Get the session info for the current user at a given site. """
log = current_app.log
db = request.db
Cred = db.tables.Cred
user_id = SiteService.get_current_uid()
cred = Cred.query.filter_by(cred_owner=user_id,
site_id=site_id).first()
res = {'ok': False}
if cred:
res['username'] = cred.cred_username
res['expiry'] = cred.cred_expiry
if cred.cred_expiry > datetime.datetime.utcnow():
res['ok'] = True
log.info("Fetched info for user %u at site %u.", user_id, site_id)
return jsonify(res)
def get_endpoints(site_id):
""" Get a list of all endpoints at a given site_id.
Designed for cert auth.
"""
db = request.db
Site = db.tables.Site
site = Site.query.filter_by(site_id=site_id).first_or_404()
endpoints = []
for ep_info in site.endpoints:
endpoints.append(ep_info.ep_uri)
cas = []
if site.user_ca_cert:
cas.append(site.user_ca_cert)
if site.service_ca_cert:
cas.append(site.service_ca_cert)
# Build response dictionary
res = {'endpoints': endpoints}
if cas:
res['cas'] = cas
return jsonify(res)
def get_element_status(job_id, element_id):
"""Get element status."""
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
element = JobElement.query.filter_by(id=element_id, job_id=job_id)\
.join(JobElement.job)\
.filter_by(user_id=HRService.check_token())\
.first_or_404()
# pylint: disable=no-member
monitoring_info = element.monitoring_info
if monitoring_info is None:
monitoring_info = {}
return jsonify({
'jobid': element.job_id,
'elementid': element.id,
'status': JobStatus(element.status).name,
'attempts': element.attempts,
'transferred': monitoring_info.get('transferred', 'N/A'),
'instant': monitoring_info.get('instant', 'N/A'),
'average': monitoring_info.get('average', 'N/A'),
'elapsed': monitoring_info.get('elapsed', 'N/A')
})
def delete_user():
"""
Delete a user. The user can only delete himself.
:return: response object with code 200 if successful, 404 if the user does not exist
"""
user_id = HRService.check_token()
# exception thrown if no user_id
db = request.db
User = request.db.tables.User
user = User.query.filter_by(id=user_id).first()
if not user:
# Raise an HTTPException with a 404 not found status code
HRService._logger.error(
"GET: requested user for id %s doesn't exist ", user_id)
abort(404)
try:
db.session.delete(user)
current_app.site_client.set_token(request.raw_token)
current_app.site_client.del_user(user_id)
db.session.commit()
HRService._logger.info(" User %s deleted successfully", user_id)
except Exception:
db.session.rollback()
HRService._logger.error(" Failed to delete a user %s (%s)",
user_id, sys.exc_info())
abort(500)
response = jsonify([{
'message':
"user %s deleted successfully" % (user.email, )
}])
response.status_code = 200
return response
def get_next_job():
"""Get the next job."""
current_app.log.debug("Worker requesting job batch, request: %s",
pformat(request.data))
require_attrs('types')
# Job = request.db.tables.Job # pylint: disable=invalid-name
# JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
alg_name = request.data.get('algorithm', 'BY_NUMBER').upper()
elements = Algorithm[alg_name](
**request.data.get('algorithm.args', {}))
# elements = JobElement.query.filter(JobElement.status.in_((JobStatus.NEW, JobStatus.FAILED)),
# JobElement.attempts < JobElement.max_tries)\
# .join(JobElement.job)\
# .filter(Job.type.in_(request.data['types']))\
# .order_by(Job.priority)\
# .order_by(Job.id)\
# .limit(10)\
# .all()
if not elements:
abort(404, description="No work to be done.")
work = []
for job, elements_group in groupby(elements, key=attrgetter('job')):
elements = []
# for element in list(elements_group):
for element in iter(elements_group):
element.status = JobStatus.SUBMITTED
element.update() # should be a bulk update
element_dict = element.asdict()
element_dict['token'] = request.token_svc.issue(
"%d.%d" % (job.id, element.id))
elements.append(element_dict)
job.status = max(ele.status for ele in job.elements)
job_dict = job.asdict()
job_dict['elements'] = elements
work.append(job_dict)
job.update()
current_app.log.debug("Sending worker job batch: %s", pformat(work))
return jsonify(work)
def get_user():
"""
Get user own self based on user_id from the token passed in.
:return: json response with user data or 404 if the user does not exist
"""
user_id = HRService.check_token()
User = request.db.tables.User
# user by id from the token
user = User.query.filter_by(id=user_id).first()
if not user:
# Raise an HTTPException with a 404 not found status code
HRService._logger.error(
"GET: requested user for id %s doesn't exist ", user_id)
abort(404)
response = jsonify(user)
response.status_code = 200
return response
def turtles_get():
""" Get a list of all turtle IDs and names. """
db = request.db
Turtle = db.tables.Turtle
res = {x.id: x.name for x in db.session.query(Turtle).all()}
return jsonify(res)
def hello():
""" Return a test string. """
return jsonify("Hello World!\n")
def get_token():
""" Issue a token to the client. """
token = request.token_svc.issue("Hello")
return jsonify(token)
def add_site():
""" Add a site in the database. """
log = current_app.log
db = request.db
Site = db.tables.Site
Endpoint = db.tables.Endpoint
site_data = {}
endpoints = []
try:
if not request.data:
return "Missing POST data", 400
raw_site_data = json.loads(request.data)
# Required fields
for key in ('site_name', 'site_desc', 'auth_type', 'auth_uri',
'public', 'def_path'):
raw_val = raw_site_data.get(key, None)
if raw_val is None:
return "Required field %s missing" % key, 400
site_data[key] = raw_val
# Optional fields
for key in ('user_ca_cert', 'service_ca_cert'):
raw_val = raw_site_data.get(key, None)
if raw_val:
site_data[key] = raw_val
# Check the auth types
if site_data["auth_type"] not in (0, 1):
log.warn("Unable to add site: Invalid auth_type (%s)",
site_data["auth_type"])
return "Invalid auth_type.", 400
if not SiteService.check_uri(site_data["auth_uri"]):
log.warn("Unable to add site: Invalid auth_uri (%s)",
site_data["auth_uri"])
return "Invalid auth_uri.", 400
# Extra fields
site_data["site_owner"] = SiteService.get_current_uid()
# Endpoints
raw_eps = raw_site_data.get('endpoints', [])
for raw_ep in raw_eps:
if not SiteService.check_uri(raw_ep):
log.warn("Unable to add site: Bad endpoint (%s)", raw_ep)
return "Invalid endpoint format.", 400
endpoints.extend(raw_eps)
except Exception as err:
log.warn("POST data error from client: %s", str(err))
return "Malformed POST data", 400
# Now actually try to create the site
new_site = Site(**site_data)
try:
with managed_session(request) as session:
session.add(new_site)
session.flush() # Ensure new_site gets an ID
site_id = new_site.site_id
# Also create the endpoints
for ep_uri in endpoints:
session.add(Endpoint(site_id=site_id, ep_uri=ep_uri))
except IntegrityError:
# site_name is probably not unique
log.info("Failed to add new non-unique site %s.",
new_site.site_name)
return "site_name is not unique", 409
except Exception as err:
# Some kind of other database error?
log.error("Failed to add new site %s (%s).",
site_data['site_name'], str(err))
return "Failed to add site to DB", 500
log.info("Added site %s with %u endpoints (ID %u).",
new_site.site_name, len(endpoints), new_site.site_id)
return jsonify(new_site.site_id)
def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too-many-branches
"""Get job element output."""
Job = request.db.tables.Job # pylint: disable=invalid-name
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
log_uid, element = Job.query.with_entities(Job.log_uid, JobElement)\
.filter_by(id=job_id, user_id=HRService.check_token())\
.join(Job.elements)\
.filter_by(id=element_id)\
.first_or_404()
log_filebase = os.path.join(current_app.workqueueservice_workerlogs,
log_uid[:2], log_uid, str(element_id))
if element.attempts == 0:
abort(404,
description=
"No attempts have yet been recorded for element %d of "
"job %d. Please try later." % (element_id, job_id))
# pylint: disable=no-member
attempt_output = {
'jobid': job_id,
'elementid': element_id,
'type': JobType(element.type).name
}
if attempt is not None:
try:
attempt = int(
attempt
) # can't use the flask <int:attempt> converter with negatives
except ValueError:
abort(400, description="bad attempt, expected an integer.")
if attempt < 0:
attempt = element.attempts + attempt + 1
if attempt not in xrange(1, element.attempts + 1):
abort(
404,
description=
"Invalid attempt '%s', job.element %s.%s has been tried %s "
"time(s)" %
(attempt, job_id, element_id, element.attempts))
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempt)
if not os.path.exists(log_filename):
abort(
500,
description=
"log directory/file %s not found for job.element %s.%s." %
(log_filename, job_id, element_id))
status = JobStatus.FAILED
if attempt == element.attempts and element.status == JobStatus.DONE:
status = JobStatus.DONE
attempt_output.update(attempt=attempt, status=status.name)
with open(log_filename, 'rb') as logfile:
attempt_output.update(log=logfile.read())
if status == JobStatus.DONE and element.type == JobType.LIST:
attempt_output.update(listing=element.listing)
return jsonify(attempt_output)
attempt_list = []
attempts = element.attempts
for attempt_ in xrange(
1, attempts): # previous attempts, presumably failed ones
failed_output = dict(attempt_output,
attempt=attempt_,
status=JobStatus.FAILED.name)
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempt_)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
failed_output.update(log=log)
attempt_list.append(failed_output)
if attempts:
status = JobStatus.FAILED
if element.status == JobStatus.DONE:
status = JobStatus.DONE
last_output = dict(attempt_output,
attempt=attempts,
status=status.name)
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempts)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
last_output.update(log=log)
if status == JobStatus.DONE and element.type == JobType.LIST:
last_output.update(listing=element.listing)
attempt_list.append(last_output)
return jsonify(attempt_list)
def get_job(job_id):
"""Get job."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
return jsonify(job)
