def test_email_user(self, mcs, smtp_mock):
with self.__service.test_request_context(path="/test"):
with mock.patch.object(pdm.userservicedesk.HRService.current_app,
'mail_token_service') as m_ts:
m_ts.issue = mock.MagicMock(return_value='agfgffsgdf')
HRService.email_user("*****@*****.**")
assert mcs.call_args[0][0] == '*****@*****.**'
assert mcs.call_args[0][1] == 'agfgffsgdf'
def test_compose_and_send_sendmail(self, smtp_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# sendmail errors
mytoken = 'mytoken_abc'
toaddr = "user@remotehost"
body = os.path.join(self._conf['verification_url'], mytoken)
smtp_mock.return_value.sendmail.side_effect = smtplib.SMTPException
with self.assertRaises(RuntimeError):
HRService.compose_and_send(toaddr, mytoken,
datetime.datetime.utcnow())
args = smtp_mock.return_value.sendmail.call_args
assert args[0][0] == self._conf['smtp_server_login']
assert args[0][1] == toaddr
assert body in args[0][2] # check the important part of the email
def get_job_status(job_id):
"""Get job status."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
# pylint: disable=no-member
return jsonify({'jobid': job.id, 'status': JobStatus(job.status).name})
def post_job():
"""Add a job."""
require_attrs('src_siteid')
Job = request.db.tables.Job # pylint: disable=invalid-name
user_id = HRService.check_token()
request.data['user_id'] = user_id
request.data['src_credentials'] = current_app.site_client\
.get_cred(request.data['src_siteid'], user_id)
if request.data['type'] == JobType.COPY:
require_attrs('dst_siteid')
request.data['dst_credentials'] = current_app.site_client\
.get_cred(request.data['dst_siteid'],
user_id)
elif request.data['type'] == JobType.RENAME and\
request.data.get('dst_siteid') != request.data['src_siteid']:
current_app.log.warn("dst_siteid (%s) != src_siteid (%s)",
request.data.get('dst_siteid'),
request.data['src_siteid'])
request.data['dst_siteid'] = request.data['src_siteid']
try:
job = Job(**request.data)
except ValueError as err:
abort(400, description=err.message)
except Exception as err: # pylint: disable=broad-except
abort(500, description=err.message)
try:
job.add()
except Exception as err: # pylint: disable=broad-except
abort(500, description=err.message)
return jsonify(job)
def get_element(job_id, element_id):
"""Get all jobs for a user."""
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
element = JobElement.query.filter_by(id=element_id, job_id=job_id)\
.join(JobElement.job)\
.filter_by(user_id=HRService.check_token())\
.first_or_404()
return jsonify(element)
def test_compose_and_send(self, close_mock, connect_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# force connect to raise the SMTPException derived class. HRService wraps it into
# RuntimeError
connect_mock.return_value = (400, 'cannot connect message'
) # 220 is the success code
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow()
) # timestamp does not matter here
connect_mock.assert_called_with('localhost', None) # from conf{}
# now allow for connect() to raise a socket.error
import socket
connect_mock.side_effect = socket.error
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow())
def get_output(job_id):
"""Get job output."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
log_filebase = os.path.join(current_app.workqueueservice_workerlogs,
job.log_uid[:2], job.log_uid)
elements_list = []
for element in job.elements:
# pylint: disable=no-member
attempt_output = {
'jobid': job_id,
'elementid': element.id,
'type': JobType(element.type).name
}
attempt_list = []
attempts = element.attempts
element_log_filebase = os.path.join(log_filebase, str(element.id))
for attempt in xrange(
1, attempts): # previous attempts, presumably failed ones
failed_output = dict(attempt_output,
attempt=attempt,
status=JobStatus.FAILED.name)
log_filename = os.path.join(element_log_filebase,
"attempt%i.log" % attempt)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
failed_output.update(log=log)
attempt_list.append(failed_output)
if attempts:
status = JobStatus.FAILED
if element.status == JobStatus.DONE:
status = JobStatus.DONE
last_output = dict(attempt_output,
attempt=attempts,
status=status.name)
log_filename = os.path.join(element_log_filebase,
"attempt%i.log" % attempts)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
last_output.update(log=log)
if status == JobStatus.DONE and element.type == JobType.LIST:
last_output.update(listing=element.listing)
attempt_list.append(last_output)
elements_list.append(attempt_list)
return jsonify(elements_list)
def get_jobs():
"""Get all jobs for a user."""
Job = request.db.tables.Job # pylint: disable=invalid-name
jobs = []
for job in Job.query.filter_by(user_id=HRService.check_token()).all():
elements = job.elements
status_counter = Counter(element.status for element in elements)
new_job = job.encode_for_json()
new_job.update(num_elements=len(elements),
num_new=status_counter[JobStatus.NEW],
num_done=status_counter[JobStatus.DONE],
num_failed=status_counter[JobStatus.FAILED],
num_submitted=status_counter[JobStatus.SUBMITTED],
num_running=status_counter[JobStatus.RUNNING])
jobs.append(new_job)
return jsonify(jobs)
def __init__(self, user_token):
"""
Constructor initialises all service clients involved in the transfer management:
EndpointService, CredService and finally the WorkqueueService.
:param user_token: user token
"""
self.__user_token = user_token
# endpoint
self.__site_client = SiteClient()
self.__site_client.set_token(user_token)
self.__sitelist = self.__site_client.get_sites()
# get user id
self.__user_id = HRService.get_token_userid(user_token)
# work queue client
self.__wq_client = WorkqueueClient()
self.__wq_client.set_token(user_token)
def get_element_status(job_id, element_id):
"""Get element status."""
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
element = JobElement.query.filter_by(id=element_id, job_id=job_id)\
.join(JobElement.job)\
.filter_by(user_id=HRService.check_token())\
.first_or_404()
# pylint: disable=no-member
monitoring_info = element.monitoring_info
if monitoring_info is None:
monitoring_info = {}
return jsonify({
'jobid': element.job_id,
'elementid': element.id,
'status': JobStatus(element.status).name,
'attempts': element.attempts,
'transferred': monitoring_info.get('transferred', 'N/A'),
'instant': monitoring_info.get('instant', 'N/A'),
'average': monitoring_info.get('average', 'N/A'),
'elapsed': monitoring_info.get('elapsed', 'N/A')
})
def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too-many-branches
"""Get job element output."""
Job = request.db.tables.Job # pylint: disable=invalid-name
JobElement = request.db.tables.JobElement # pylint: disable=invalid-name
log_uid, element = Job.query.with_entities(Job.log_uid, JobElement)\
.filter_by(id=job_id, user_id=HRService.check_token())\
.join(Job.elements)\
.filter_by(id=element_id)\
.first_or_404()
log_filebase = os.path.join(current_app.workqueueservice_workerlogs,
log_uid[:2], log_uid, str(element_id))
if element.attempts == 0:
abort(404,
description=
"No attempts have yet been recorded for element %d of "
"job %d. Please try later." % (element_id, job_id))
# pylint: disable=no-member
attempt_output = {
'jobid': job_id,
'elementid': element_id,
'type': JobType(element.type).name
}
if attempt is not None:
try:
attempt = int(
attempt
) # can't use the flask <int:attempt> converter with negatives
except ValueError:
abort(400, description="bad attempt, expected an integer.")
if attempt < 0:
attempt = element.attempts + attempt + 1
if attempt not in xrange(1, element.attempts + 1):
abort(
404,
description=
"Invalid attempt '%s', job.element %s.%s has been tried %s "
"time(s)" %
(attempt, job_id, element_id, element.attempts))
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempt)
if not os.path.exists(log_filename):
abort(
500,
description=
"log directory/file %s not found for job.element %s.%s." %
(log_filename, job_id, element_id))
status = JobStatus.FAILED
if attempt == element.attempts and element.status == JobStatus.DONE:
status = JobStatus.DONE
attempt_output.update(attempt=attempt, status=status.name)
with open(log_filename, 'rb') as logfile:
attempt_output.update(log=logfile.read())
if status == JobStatus.DONE and element.type == JobType.LIST:
attempt_output.update(listing=element.listing)
return jsonify(attempt_output)
attempt_list = []
attempts = element.attempts
for attempt_ in xrange(
1, attempts): # previous attempts, presumably failed ones
failed_output = dict(attempt_output,
attempt=attempt_,
status=JobStatus.FAILED.name)
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempt_)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
failed_output.update(log=log)
attempt_list.append(failed_output)
if attempts:
status = JobStatus.FAILED
if element.status == JobStatus.DONE:
status = JobStatus.DONE
last_output = dict(attempt_output,
attempt=attempts,
status=status.name)
log_filename = os.path.join(log_filebase,
"attempt%i.log" % attempts)
log = "log directory/file %s not found for job.element %s.%s."\
% (log_filename, job_id, element.id)
if os.path.exists(log_filename):
with open(log_filename, 'rb') as logfile:
log = logfile.read()
last_output.update(log=log)
if status == JobStatus.DONE and element.type == JobType.LIST:
last_output.update(listing=element.listing)
attempt_list.append(last_output)
return jsonify(attempt_list)
def get_job(job_id):
"""Get job."""
Job = request.db.tables.Job # pylint: disable=invalid-name
job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\
.first_or_404()
return jsonify(job)