def test_init_existingDB(self):
""" Tests that the config step completes correctly if a DB already
exists.
"""
# Create a clean copy of the service
service = FlaskServer("pdm.demo.DemoService")
service.test_mode(DemoService, None)
service.build_db()
# Add a single turtle to the table
db = service.test_db()
new_turtle = db.tables.Turtle(name='Before')
db.session.add(new_turtle)
db.session.commit()
# Continue service start-up
service.before_startup({}, with_test=True)
service.fake_auth("ALL")
client = service.test_client()
# Now check that we only have one turtle
# Rather than the 3 we get by default
res = client.get('/demo/api/v1.0/turtles')
assert(res.status_code == 200)
assert(len(json.loads(res.data)) == 1)
class TestTransferClient(unittest.TestCase):
# @mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient")
# @mock.patch("pdm.userservicedesk.TransferClient.WorkqueueClient")
##@mock.patch.object(HRService, 'check_token')
@mock.patch("pdm.userservicedesk.HRService.SiteClient")
@mock.patch.object(Tokens.TokenService, 'unpack')
@mock.patch("pdm.userservicedesk.TransferClient.SiteClient")
@mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient.__new__")
def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock):
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
site_mock().get_sites.return_value = \
[{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'},
{'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}]
site_mock.return_value.set_token = mock.MagicMock()
self.site_id = site_mock().get_sites()[0]['site_id']
self.site2_id = site_mock().get_sites()[1]['site_id']
conf = {
'CS_secret': 'HJGnbfdsV',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
# HR
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
token = {'id': 1, 'expiry': self.__future_date}
self.__service.fake_auth("TOKEN", token)
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
self.__service.before_startup(conf) # to continue startup
# mock_ct.return_value =1
mocked_unpack.return_value = token
self.__htoken = 'whateverhash'
self.__client = TransferClientFacade(self.__htoken)
assert wq_mock.called
mocked_unpack.assert_called_with(self.__htoken)
def test_list(self):
site = "localhost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt'
assert self.__client.list(site, **{'priority':
2}) == 'root/file.txt'
assert mock_list.called
mock_list.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_list.call_args_list
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt' # event if ...
assert self.__client.list(wrongurl, **{'priority':
2}) == [] # we return []
assert not mock_list.called
def test_sitelist(self):
sites = self.__client.list_sites()
print sites
assert sites[0]['site_name'] == 'localhost'
assert sites[1]['site_name'] == 'remotehost'
assert 'site_id' not in [dd.keys() for dd in sites]
def test_remove(self):
site = "localhost:/root/file.txt"
# mock_remove.return_value = 'root/file.txt'
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'root/file.txt removed'
assert self.__client.remove(site, **{'priority':
2}) == 'root/file.txt removed'
assert mock_remove.called
mock_remove.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_remove.call_args_remove
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'whatever..' # event if ...
assert self.__client.remove(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_remove.called
def test_copy(self):
s_site = "localhost:/root/file.txt"
t_site = "remotehost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'root/file.txt copied'
assert self.__client.copy(s_site, t_site,
**{'priority':
2}) == 'root/file.txt copied'
assert mock_copy.called
mock_copy.assert_called_with(self.site_id,
'/root/file.txt',
self.site2_id,
'/root/file.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'whatever..' # even if ...
assert self.__client.copy(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_copy.called
def test_split_site_path(self):
site = "localhost:/root/file.txt"
malformed_site = "localhost/root/file.txt" # mind a missing colon
multicolon_site = "localhost:/root/file.txt:1"
a, b = TransferClientFacade.split_site_path(site)
assert a == 'localhost'
assert b == '/root/file.txt'
c, d = TransferClientFacade.split_site_path(malformed_site)
assert d is None
assert c is None
e, f = TransferClientFacade.split_site_path(multicolon_site)
assert e == 'localhost'
assert f == '/root/file.txt:1'
def test_mkdir(self):
site = "localhost:/root/subdir"
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'root/subdir created'
assert self.__client.mkdir(site, **{'priority':
2}) == 'root/subdir created'
assert mock_mkdir.called
mock_mkdir.assert_called_with(self.site_id, '/root/subdir', priority=2)
# now unknown site:
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'whatever..' # event if ...
assert self.__client.mkdir(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_mkdir.called
def test_rename(self):
s_site = "localhost:/root/file.txt"
t_site = ":/root/file2.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'root/file.txt renamed'
assert self.__client.rename(s_site, t_site,
**{'priority':
2}) == 'root/file.txt renamed'
assert mock_rename.called
mock_rename.assert_called_with(self.site_id,
'/root/file.txt',
'/root/file2.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'whatever..' # even if ...
assert self.__client.rename(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_rename.called
def tearDown(self):
pass
class TestAlgorithms(unittest.TestCase):
def setUp(self):
conf = {'workerlogs': '/tmp/workers'}
self.__service = FlaskServer("pdm.workqueue.WorkqueueService")
self.__service.test_mode(WorkqueueService,
None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__service.build_db() # build manually
db = self.__service.test_db()
Job = db.tables.Job
JobElement = db.tables.JobElement
db.session.add(
Job(user_id=1,
src_siteid=13,
src_filepath='/data/somefile1',
type=JobType.LIST))
job = Job(user_id=2,
src_siteid=14,
src_filepath='/data/somefile2',
type=JobType.REMOVE)
for i in xrange(1, 6):
job.elements.append(
JobElement(id=i,
job_id=2,
src_siteid=12,
src_filepath='/data/somefile2.%d' % i,
type=JobType.REMOVE,
size=10**i))
db.session.add(job)
j = Job(user_id=3,
type=JobType.COPY,
src_siteid=15,
src_filepath='/data/somefile3',
dst_siteid=16,
dst_filepath='/data/newfile')
for i in xrange(1, 6):
j.elements.append(
JobElement(id=i,
job_id=3,
src_siteid=12,
src_filepath='/data/somefile3.%d' % i,
dst_filepath='/data/newfile.%d' % i,
type=JobType.COPY,
size=10**i))
db.session.add(j)
db.session.commit()
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self.__service.before_startup(conf) # to continue startup
self.__test = self.__service.test_client()
@mock.patch('pdm.workqueue.WorkqueueService.request')
def test_by_number(self, mock_request):
mock_request.db.tables.Job = self.__service.test_db().tables.Job
mock_request.db.tables.JobElement = self.__service.test_db(
).tables.JobElement
mock_request.data = {"types": (JobType.RENAME, JobType.MKDIR)}
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 0)
mock_request.data["types"] = (JobType.LIST, )
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 1)
mock_request.data["types"] = (JobType.LIST, JobType.COPY)
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 7)
mock_request.data["types"] = (JobType.LIST, JobType.COPY,
JobType.REMOVE)
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 13)
self.assertEqual(len(Algorithm["BY_NUMBER"](5)), 5)
self.assertEqual(len(Algorithm["BY_NUMBER"](8)), 8)
@mock.patch('pdm.workqueue.WorkqueueService.request')
def test_by_size(self, mock_request):
mock_request.db.tables.Job = self.__service.test_db().tables.Job
mock_request.db.tables.JobElement = self.__service.test_db(
).tables.JobElement
mock_request.data = {"types": (JobType.RENAME, JobType.MKDIR)}
self.assertEqual(len(Algorithm["BY_SIZE"]()), 0)
mock_request.data["types"] = (JobType.LIST, )
self.assertEqual(len(Algorithm["BY_SIZE"]()), 1)
mock_request.data["types"] = (JobType.LIST, JobType.COPY)
self.assertEqual(len(Algorithm["BY_SIZE"]()), 7)
mock_request.data["types"] = (JobType.LIST, JobType.COPY,
JobType.REMOVE)
self.assertEqual(len(Algorithm["BY_SIZE"]()), 13)
self.assertEqual(len(Algorithm["BY_SIZE"](100)), 3)
self.assertEqual(len(Algorithm["BY_SIZE"](111)), 4)
class TestWorkqueueService(unittest.TestCase):
def setUp(self):
conf = {'workerlogs': '/tmp/workers'}
self.__service = FlaskServer("pdm.workqueue.WorkqueueService")
self.__service.test_mode(WorkqueueService,
None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__service.build_db() # build manually
db = self.__service.test_db()
Job = db.tables.Job
JobElement = db.tables.JobElement
db.session.add(
Job(user_id=1,
src_siteid=13,
src_filepath='/data/somefile1',
type=JobType.LIST))
job = Job(user_id=2,
src_siteid=14,
src_filepath='/data/somefile2',
type=JobType.REMOVE)
for i in xrange(1, 6):
job.elements.append(
JobElement(id=i,
job_id=2,
src_siteid=12,
src_filepath='/data/somefile2.%d' % i,
type=JobType.REMOVE,
size=10**i))
db.session.add(job)
j = Job(user_id=3,
type=JobType.COPY,
src_siteid=15,
src_filepath='/data/somefile3',
dst_siteid=16,
dst_filepath='/data/newfile')
for i in xrange(1, 6):
j.elements.append(
JobElement(id=i,
job_id=3,
src_siteid=12,
src_filepath='/data/somefile3.%d' % i,
dst_filepath='/data/newfile.%d' % i,
type=JobType.COPY,
size=10**i))
db.session.add(j)
db.session.commit()
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self.__service.before_startup(conf) # to continue startup
self.__test = self.__service.test_client()
def test_get_next_job(self):
"""test worker get next job."""
request = self.__test.post('/workqueue/api/v1.0/worker/jobs',
data={'test': 12})
self.assertEqual(request.status_code, 400,
"Expected job with incorrect attrs to fail.")
request = self.__test.post('/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.LIST]})
self.assertEqual(request.status_code, 200,
"Request to get worker job failed.")
work = json.loads(request.data)
self.assertEqual(len(work), 1)
job = work[0]
self.assertEqual(len(job['elements']), 1)
self.assertDictContainsSubset(
{
'status': JobStatus.SUBMITTED,
'dst_credentials': None,
'user_id': 1,
'src_filepath': '/data/somefile1',
'priority': 5,
'dst_siteid': None,
'src_siteid': 13,
'extra_opts': None,
'protocol': 0,
'type': JobType.LIST,
'id': 1,
'src_credentials': None,
'dst_filepath': None
}, job) #, "Job not returned correctly.")
element = job['elements'][0]
self.assertIsInstance(element['token'], basestring)
self.assertDictContainsSubset(
{
'status': JobStatus.SUBMITTED,
'job_id': 1,
'attempts': 0,
'src_filepath': '/data/somefile1',
'listing': None,
'max_tries': 2,
'type': JobType.LIST,
'id': 0,
'dst_filepath': None
}, element)
Job = self.__service.test_db().tables.Job
JobElement = self.__service.test_db().tables.JobElement
#j = Job.query.filter_by(id=job['id']).one()
j = Job.query.filter_by(id=job['id']).one()
je = JobElement.query.filter_by(id=element['id'],
job_id=element['job_id']).one()
self.assertIsNotNone(j)
self.assertIsNotNone(je)
self.assertEqual(j.status, JobStatus.SUBMITTED,
"Job status not updated in DB")
self.assertEqual(je.status, JobStatus.SUBMITTED,
"Job status not updated in DB")
request = self.__test.post(
'/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.COPY, JobType.REMOVE]})
self.assertEqual(request.status_code, 200,
"Failed to get copy or remove job.")
work = json.loads(request.data)
self.assertEqual(len(work), 2)
self.assertEqual(work[0]['type'], JobType.REMOVE)
self.assertEqual(len(work[0]['elements']), 6)
self.assertEqual(work[0]['elements'][0]['type'], JobType.LIST)
for i in xrange(1, 6):
self.assertEqual(work[0]['elements'][i]['type'], JobType.REMOVE)
self.assertEqual(work[1]['type'], JobType.COPY)
self.assertEqual(len(work[1]['elements']), 6)
# up to 10 loaded now at once
#request = self.__test.post('/workqueue/api/v1.0/worker/jobs', data={'types': [JobType.COPY, JobType.REMOVE]})
#self.assertEqual(request.status_code, 200, "Failed to get copy or remove job.")
request = self.__test.post(
'/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.COPY, JobType.REMOVE]})
self.assertEqual(
request.status_code, 404,
"Trying to get a job that doesn't exist should return 404.")
def test_return_output(self):
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 403)
self.__service.fake_auth("TOKEN", "12")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 403)
self.__service.fake_auth("TOKEN", "1.2")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 400)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 404)
self.__service.fake_auth("TOKEN", "1.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 400)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 1,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 200)
Job = self.__service.test_db().tables.Job
JobElement = self.__service.test_db().tables.JobElement
je = JobElement.query.filter_by(job_id=1, id=0).one()
j = Job.query.filter_by(id=1).one()
self.assertIsNotNone(je)
self.assertEqual(je.status, JobStatus.FAILED)
self.assertEqual(je.job.status, JobStatus.FAILED)
self.assertIsNotNone(j)
self.assertEqual(j.status, JobStatus.FAILED)
logfile = os.path.join('/tmp/workers', j.log_uid[:2], j.log_uid,
str(je.id), 'attempt1.log')
self.assertTrue(os.path.isfile(logfile))
expected_log = dedent("""
Job run on host: somehost.domain, returncode: 1, timestamp: timestamp
blah blah
""").strip()
with open(logfile, 'rb') as log:
self.assertEqual(log.read(), expected_log)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'root': []
}
})
self.assertEqual(request.status_code, 200)
je = JobElement.query.filter_by(job_id=1, id=0).one()
j = Job.query.filter_by(id=1).one()
self.assertIsNotNone(je)
self.assertEqual(je.status, JobStatus.DONE)
self.assertEqual(je.job.status, JobStatus.DONE)
self.assertIsNotNone(j)
self.assertEqual(j.status, JobStatus.DONE)
logfile = os.path.join('/tmp/workers', j.log_uid[:2], j.log_uid,
str(je.id), 'attempt2.log')
self.assertTrue(os.path.isfile(logfile))
expected_log = dedent("""
Job run on host: somehost.domain, returncode: 0, timestamp: timestamp
blah blah
""").strip()
with open(logfile, 'rb') as log:
self.assertEqual(log.read(), expected_log)
self.assertEqual(je.listing, {'root': []})
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'root': []
}
})
self.assertEqual(request.status_code, 400,
'Exceeding max tries should give 400')
# Test expansion of COPY jobs
db = self.__service.test_db()
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data/somefile',
dst_siteid=16,
dst_filepath='/site2/data/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data/somefile',
dst_siteid=16,
dst_filepath='~/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='~/somefile',
dst_siteid=16,
dst_filepath='~/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
dst_siteid=16,
dst_filepath='/site2/data/somedir',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
dst_siteid=16,
dst_filepath='~',
type=JobType.COPY))
j = Job.query.filter_by(id=4).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=5).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=6).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=7).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=8).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
self.__service.fake_auth("TOKEN", "4.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/4/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 200,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=4).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath,
'/site2/data/someotherfile')
self.__service.fake_auth("TOKEN", "5.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/5/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 200,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=5).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath, '~/someotherfile')
self.__service.fake_auth("TOKEN", "6.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/6/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'~': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=6).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath, '~/someotherfile')
self.__service.fake_auth("TOKEN", "7.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/7/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=7).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 3)
self.assertEqual(j.elements[1].dst_filepath,
'/site2/data/somedir/somefile')
self.assertEqual(j.elements[2].dst_filepath,
'/site2/data/somedir/someotherdir/someotherfile')
self.__service.fake_auth("TOKEN", "8.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/8/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=8).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 3)
self.assertEqual(j.elements[1].dst_filepath, '~/somefile')
self.assertEqual(j.elements[2].dst_filepath,
'~/someotherdir/someotherfile')
# Test expansion of REMOVE jobs
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
type=JobType.REMOVE))
j = Job.query.filter_by(id=9).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
self.__service.fake_auth("TOKEN", "9.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/9/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=9).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 6)
self.assertEqual(j.elements[1].src_filepath,
'/site1/data/someotherdir/someotherfile')
self.assertEqual(j.elements[2].src_filepath,
'/site1/data/someotherdir/somedir/')
self.assertEqual(j.elements[3].src_filepath, '/site1/data/somefile')
self.assertEqual(j.elements[4].src_filepath,
'/site1/data/someotherdir/')
self.assertEqual(j.elements[5].src_filepath, '/site1/data/')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_post_job(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={'blah': 12})
self.assertEqual(request.status_code, 400)
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.LIST,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.status, JobStatus.NEW)
self.assertEqual(returned_job['status'], 'NEW')
self.assertEqual(job.type, JobType.LIST)
self.assertEqual(returned_job['type'], 'LIST')
self.assertEqual(job.priority, 5)
self.assertEqual(job.src_credentials, 'somesecret')
self.assertEqual(job.protocol, JobProtocol.GRIDFTP)
self.assertEqual(returned_job['protocol'], 'GRIDFTP')
self.assertIsInstance(job.log_uid, basestring)
self.assertEqual(len(job.elements), 1)
element = job.elements[0]
self.assertEqual(element.type, JobType.LIST)
self.assertEqual(element.src_filepath, '/data/somefile')
self.assertIsNone(element.dst_filepath)
self.assertEqual(element.attempts, 0)
self.assertEqual(element.max_tries, 2)
mock_hrservice.return_value = 12
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15
})
self.assertEqual(request.status_code, 400)
mock_siteclient.site_client.get_cred = mock.MagicMock(
side_effect=["somesecret", "someothersecret"])
# mock_siteclient().get_cred = mock.MagicMock(side_effect=["somesecret", "someothersecret"])
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile',
'extra_opts': {},
'attempts': 30,
'max_tries': 3,
'priority': 2,
'protocol': JobProtocol.SSH,
'log_uid': 'my_log_uid'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=12).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.status, JobStatus.NEW)
self.assertEqual(returned_job['status'], 'NEW')
self.assertEqual(job.type, JobType.COPY)
self.assertEqual(returned_job['type'], 'COPY')
self.assertEqual(job.priority, 2)
self.assertEqual(job.src_filepath, '/data/somefile')
self.assertEqual(job.dst_filepath, '/data/someotherfile')
self.assertEqual(job.src_credentials, "somesecret")
self.assertEqual(job.dst_credentials, "someothersecret")
self.assertEqual(job.protocol, JobProtocol.SSH)
self.assertEqual(returned_job['protocol'], 'SSH')
self.assertIsInstance(job.log_uid, basestring)
self.assertNotEqual(job.log_uid, 'my_log_uid')
self.assertEqual(len(job.elements), 1)
element = job.elements[0]
self.assertEqual(element.type, JobType.LIST)
self.assertEqual(element.src_filepath, '/data/somefile')
self.assertEqual(element.dst_filepath, None)
self.assertEqual(element.attempts, 0)
self.assertEqual(element.max_tries, 3)
# @mock.patch.object(HRService.HRService, 'check_token')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_list(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
# mock_siteclient().get_cred = mock.MagicMock(return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/list',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.LIST)
self.assertEqual(returned_job['type'], 'LIST')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_copy(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
side_effect=["somesecret", "someothersecret"])
# mock_siteclient().get_cred = mock.MagicMock(side_effect=["somesecret", "someothersecret"])
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/copy',
data={
'type': JobType.LIST,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.COPY)
self.assertEqual(returned_job['type'], 'COPY')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_remove(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
# mock_siteclient().get_cred = mock.MagicMock(return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/remove',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.REMOVE)
self.assertEqual(returned_job['type'], 'REMOVE')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_jobs(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs')
self.assertEqual(request.status_code, 200)
returned_jobs = json.loads(request.data)
self.assertEqual(returned_jobs, [])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs')
self.assertEqual(request.status_code, 200)
returned_jobs = json.loads(request.data)
self.assertEqual(len(returned_jobs), 1)
self.assertDictContainsSubset(
{
'user_id': 1,
'type': 'LIST',
'status': 'NEW'
}, returned_jobs[0])
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_job(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/2')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/1')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2')
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
self.assertDictContainsSubset(
{
'user_id': 2,
'type': 'REMOVE',
'status': 'NEW'
}, returned_job)
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_elements(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements')
self.assertEqual(request.status_code, 200)
returned_elements = json.loads(request.data)
self.assertEqual(returned_elements, [])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements')
self.assertEqual(request.status_code, 200)
returned_elements = json.loads(request.data)
self.assertEqual(len(returned_elements), 1)
self.assertDictContainsSubset(
{
'job_id': 1,
'type': 'LIST',
'status': 'NEW'
}, returned_elements[0])
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_element(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/0')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements/0')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/33')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/0')
self.assertEqual(request.status_code, 200)
returned_element = json.loads(request.data)
self.assertDictContainsSubset(
{
'job_id': 2,
'type': 'LIST',
'status': 'NEW'
}, returned_element)
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_job_status(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/3/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get('/workqueue/api/v1.0/jobs/1/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get('/workqueue/api/v1.0/jobs/3/status')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, {'jobid': 3, 'status': 'NEW'})
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_element_status(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/17/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/12/elements/0/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get(
'/workqueue/api/v1.0/jobs/3/elements/0/status')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 3,
'elementid': 0,
'status': 'NEW',
'attempts': 0,
'transferred': 'N/A',
'instant': 'N/A',
'average': 'N/A',
'elapsed': 'N/A'
})
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_output(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/1/output')
self.assertEqual(request.status_code, 404,
"invalid userid should give 404")
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/2/output')
self.assertEqual(request.status_code, 404,
"userid not matching job id should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/10/output')
self.assertEqual(request.status_code, 404,
"Unknown element id should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output')
self.assertEqual(request.status_code, 404,
"Status other than DONE or FAILED gives 404")
db = self.__service.test_db()
session = db.session
list_job = db.tables.Job.query.filter_by(user_id=1).one()
remove_job = db.tables.Job.query.filter_by(user_id=2).one()
self.assertIsNotNone(list_job)
self.assertIsNotNone(remove_job)
list_job.status = JobStatus.DONE
remove_job.status = JobStatus.FAILED
list_job.elements[0].status = JobStatus.DONE
remove_job.elements[0].status = JobStatus.DONE
remove_job.elements[1].status = JobStatus.FAILED
session.merge(list_job)
session.merge(remove_job)
session.expunge(list_job)
session.expunge(remove_job)
session.commit()
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output')
self.assertEqual(
request.status_code, 404,
"Element with 0 attempts is not yet ready, should give 404")
list_job.elements[0].attempts = 1
remove_job.elements[0].attempts = 1
remove_job.elements[1].attempts = 1
list_job.elements[0].listing = {'root': [{'name': 'somefile'}]}
remove_job.elements[0].listing = {'root': [{'name': 'somefile'}]}
session.merge(list_job)
session.merge(remove_job)
session.commit()
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output/9')
self.assertEqual(request.status_code, 404,
"Invalid attempt should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output/1')
self.assertEqual(request.status_code, 500,
"Failure to find logfile should give 500")
list_job_dir = os.path.join('/tmp/workers', list_job.log_uid[:2],
list_job.log_uid, '0')
remove_job_dir = os.path.join('/tmp/workers', remove_job.log_uid[:2],
remove_job.log_uid)
remove_job0_dir = os.path.join(remove_job_dir, '0')
remove_job1_dir = os.path.join(remove_job_dir, '1')
list_job_filename = os.path.join(list_job_dir, "attempt1.log")
remove_job0_filename = os.path.join(remove_job0_dir, "attempt1.log")
remove_job1_filename = os.path.join(remove_job1_dir, "attempt1.log")
if not os.path.exists(list_job_dir):
os.makedirs(list_job_dir)
if not os.path.exists(remove_job0_dir):
os.makedirs(remove_job0_dir)
if not os.path.exists(remove_job1_dir):
os.makedirs(remove_job1_dir)
with open(list_job_filename, 'wb') as listlog,\
open(remove_job0_filename, 'wb') as removelog0,\
open(remove_job1_filename, 'wb') as removelog1:
listlog.write('la la la\n')
removelog0.write('blah blah\n')
removelog1.write('tralala\n')
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [[{
'status': 'DONE',
'attempt': 1,
'log': 'blah blah\n',
'jobid': 2,
'listing': {
'root': [{
'name': 'somefile'
}]
},
'type': 'LIST',
'elementid': 0
}],
[{
'status': 'FAILED',
'attempt': 1,
'log': 'tralala\n',
'jobid': 2,
'type': 'REMOVE',
'elementid': 1
}], [], [], [], []])
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [{
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
}])
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output/1')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
})
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output/-1')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
})
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/1/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [{
'jobid': 2,
'elementid': 1,
'type': 'REMOVE',
'attempt': 1,
'status': 'FAILED',
'log': 'tralala\n'
}])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/1/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [[{
'status': 'DONE',
'attempt': 1,
'log': 'la la la\n',
'jobid': 1,
'listing': {
'root': [{
'name': 'somefile'
}]
},
'type': 'LIST',
'elementid': 0
}]])
class TestHRService(unittest.TestCase):
@mock.patch("pdm.userservicedesk.HRService.SiteClient")
def setUp(self, site_mock):
self.__site_mock = site_mock
conf = {
'token_validity': '01:00:00',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
self._conf = copy.deepcopy(conf)
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
self.__past_date = (-datetime.timedelta(0, 60) +
datetime.datetime.utcnow()).isoformat()
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
new_user = db.tables.User(name='John',
surname='Smith',
email='*****@*****.**',
state=HRServiceUserState.VERIFIED,
password=hash_pass('very_secret'))
db.session.add(new_user)
db.session.commit()
self.__service.before_startup(conf) # to continue startup
#
self.__test = self.__service.test_client()
# mail token
time_struct = time.strptime("12:00:00", "%H:%M:%S")
self.token_duration = datetime.timedelta(hours=time_struct.tm_hour,
minutes=time_struct.tm_min,
seconds=time_struct.tm_sec)
self.mail_token_service = TokenService(self._conf['mail_token_secret'])
def test_getUserSelf(self):
"""
GET operation on users/self
:return:
"""
self.__service.fake_auth("TOKEN", {'id': 1})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 500)
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 403) # token expired
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date,
'key': 'unused'
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 200)
user = json.loads(res.data)
assert ('id' not in user)
assert (user['name'] == 'John')
assert (user['surname'] == 'Smith')
assert (user['email'] == '*****@*****.**')
assert (user['state'] == HRServiceUserState.VERIFIED)
assert ('password' not in user)
#
self.__service.fake_auth("TOKEN", {
'id': 2,
'expiry': self.__future_date
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 404)
@mock.patch("pdm.userservicedesk.HRService.HRService.email_user")
def test_addUser(self, email_user_mock):
"""
Testinf user registration. Ignore emailer at this stage.
:return:
"""
self.__service.fake_auth("ALL")
fred = {
'surname': 'Flintstone',
'name': 'Fred',
'email': '*****@*****.**',
'state': 0,
'password': '******'
}
barney = {
'surname': 'Rubble',
'name': 'Barney',
'email': '*****@*****.**',
'state': 0,
'password': '******'
}
res = self.__test.post('/users/api/v1.0/users', data=fred)
assert (res.status_code == 201)
# db
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(email=fred['email']).first()
assert (dbuser.name == fred['name'])
assert (check_hash(dbuser.password, fred['password']))
assert (dbuser.email == fred['email'])
response = json.loads(res.data)
assert (response['name'] == fred['name'])
assert (response['surname'] == fred['surname'])
assert (response['email'] == fred['email'])
assert (response['state'] == fred['state'])
assert ('password' not in response)
# try to duplicate the user:
res = self.__test.post('/users/api/v1.0/users', data=fred)
assert (res.status_code == 403)
# password too short !
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
# barney OK, but verification email sending fails:
barney['password'] = '******'
email_user_mock.side_effect = RuntimeError
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 500)
#
b_email = barney.pop('email')
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
barney['email'] = b_email
password = barney.pop('password')
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
def test_change_password(self):
"""
Test the password changing operation
:return:
"""
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date
})
new_pass_data = {
'passwd': 'very_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 403)
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
new_pass_data = {
'passwd': 'very_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 200)
# check if the password was actually modified:
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert (dbuser.name == "John")
assert (check_hash(dbuser.password, 'even_more_secret'))
#
response = json.loads(res.data)
assert ('password' not in response)
# TODO check last login timestamp later than the time before changing the password.
# wrong password
wrong_pass_data = {
'passwd': 'very_sercet',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=wrong_pass_data)
assert (res.status_code == 403)
# same pass
same_pass_data = {
'passwd': 'even_more_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=same_pass_data)
assert (res.status_code == 400)
# no pass
no_pass = {'passwd': None, 'newpasswd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
no_pass = {'newpasswd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
no_pass = {'passwd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
#
no_npass = {'passwd': 'even_more_secret', 'newpasswd': None}
res = self.__test.put('/users/api/v1.0/passwd', data=no_npass)
assert (res.status_code == 400)
# weak pass
weak_pass = {'passwd': 'even_more_secret', 'newpasswd': 'test'}
res = self.__test.put('/users/api/v1.0/passwd', data=weak_pass)
assert (res.status_code == 400)
# non existing user
self.__service.fake_auth("TOKEN", {
'id': 7,
'expiry': self.__future_date
})
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 403)
# @mock.patch('pdm.userservicedesk.HRService.SiteClient')
def test_delete_user(self):
"""
Test deleting user data
:return:
"""
# not existing user:
self.__service.fake_auth("TOKEN", {
'id': 7,
'expiry': self.__future_date
})
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 404)
assert not self.__site_mock().del_user.called
# attempt to delete Johnny with an expired token
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date,
'key': 'unused'
}) # fake auth John, which is id=1
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 403)
assert not self.__site_mock().del_user.called
# delete poor Johnny ;-(
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date,
'key': 'unused'
}) # fake auth John, which is id=1
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 200)
assert self.__site_mock().del_user.called
def test_deleteUser_SiteService_fail(self):
"""
Test if the user is put back when SiteService fails
:param mock_del_user:
:return:
"""
# delete poor Johnny ;-(
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
self.__site_mock().del_user.side_effect = Exception()
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 500)
assert self.__site_mock().del_user.called
# check if we rolled John back !
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert (dbuser is not None)
@mock.patch('sqlalchemy.orm.scoping.scoped_session.delete')
def test_deleteUser_HR_fail(self, mock_del):
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
mock_del.side_effect = Exception()
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 500)
assert not self.__site_mock().del_user.called
def test_loginUser(self):
"""
Test the user login procedure
:return:
"""
res = self.__test.post('/users/api/v1.0/login') # empty req.
assert (res.status_code == 400)
res = self.__test.post('/users/api/v1.0/login', data=('hulagula'))
assert (res.status_code == 400)
login_creds = {'email': '*****@*****.**', 'passwd': 'very_secret'}
res = self.__test.post('/users/api/v1.0/login', data=login_creds)
assert (res.status_code == 200)
#
token_data = self.__service.token_svc.check(json.loads(res.data))
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert token_data['id'] == 1
isoformat = '%Y-%m-%dT%H:%M:%S.%f'
expiry_date = datetime.datetime.strptime(token_data['expiry'],
isoformat)
# conf gives 1h token validity. Check if we are within 10s
assert abs(
(expiry_date - (datetime.datetime.utcnow() +
datetime.timedelta(0, 3600))).total_seconds()) < 10
login_creds_b = {'email': '*****@*****.**'}
res = self.__test.post('/users/api/v1.0/login', data=login_creds_b)
assert (res.status_code == 400)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': 'very_seCret'
})
assert (res.status_code == 403)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': 'very_secret'
})
assert (res.status_code == 403)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': None
})
assert (res.status_code == 400)
# make Johny unverified ;-(
dbuser.state = HRServiceUserState.REGISTERED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/login', data=login_creds)
assert (res.status_code == 401)
@mock.patch('smtplib.SMTP')
@mock.patch.object(HRService, 'compose_and_send')
def test_email_user(self, mcs, smtp_mock):
with self.__service.test_request_context(path="/test"):
with mock.patch.object(pdm.userservicedesk.HRService.current_app,
'mail_token_service') as m_ts:
m_ts.issue = mock.MagicMock(return_value='agfgffsgdf')
HRService.email_user("*****@*****.**")
assert mcs.call_args[0][0] == '*****@*****.**'
assert mcs.call_args[0][1] == 'agfgffsgdf'
#(ignore the timestamp passed in as the third arg.)
def test_verify_user(self):
# isssue a valid mail token
expiry = datetime.datetime.utcnow() + self.token_duration
plain = {'expiry': expiry.isoformat(), 'email': '*****@*****.**'}
token = self.mail_token_service.issue(plain)
#body = os.path.join(self._conf['verification_url'],token)
# verify takes a token only, not the whole email body at the moment
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
#token tempered with:
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token[1:]})
assert res.status_code == 400
# success ?
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 201
#repeat a verification attempt
dbuser.state = HRServiceUserState.VERIFIED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 400
# expired token
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
expired = datetime.datetime.utcnow() - self.token_duration
e_plain = {
'expiry': expired.isoformat(),
'email': '*****@*****.**'
}
e_token = self.mail_token_service.issue(e_plain)
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': e_token})
assert res.status_code == 400
# non existent user:
plain = {'expiry': expiry.isoformat(), 'email': '*****@*****.**'}
token = self.mail_token_service.issue(plain)
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 400
@mock.patch('smtplib.SMTP')
def test_resend_email(self, smtp_mock):
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
email = '*****@*****.**'
data = {'email': email}
res = self.__test.post('/users/api/v1.0/resend', data=data)
assert res.status_code == 200
dbuser.state = HRServiceUserState.VERIFIED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/resend', data=data)
assert res.status_code == 400
res = self.__test.post('/users/api/v1.0/resend',
data={'email': 'hula@gula'})
assert res.status_code == 400
res = self.__test.post('/users/api/v1.0/resend',
data={'Email': 'hula@gula'})
assert res.status_code == 400
@mock.patch('email.MIMEMultipart.MIMEMultipart')
@mock.patch.object(smtplib.SMTP, 'connect')
@mock.patch.object(smtplib.SMTP, 'close')
def test_compose_and_send(self, close_mock, connect_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# force connect to raise the SMTPException derived class. HRService wraps it into
# RuntimeError
connect_mock.return_value = (400, 'cannot connect message'
) # 220 is the success code
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow()
) # timestamp does not matter here
connect_mock.assert_called_with('localhost', None) # from conf{}
# now allow for connect() to raise a socket.error
import socket
connect_mock.side_effect = socket.error
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow())
@mock.patch('email.MIMEMultipart.MIMEMultipart')
@mock.patch('smtplib.SMTP')
def test_compose_and_send_sendmail(self, smtp_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# sendmail errors
mytoken = 'mytoken_abc'
toaddr = "user@remotehost"
body = os.path.join(self._conf['verification_url'], mytoken)
smtp_mock.return_value.sendmail.side_effect = smtplib.SMTPException
with self.assertRaises(RuntimeError):
HRService.compose_and_send(toaddr, mytoken,
datetime.datetime.utcnow())
args = smtp_mock.return_value.sendmail.call_args
assert args[0][0] == self._conf['smtp_server_login']
assert args[0][1] == toaddr
assert body in args[0][2] # check the important part of the email
def test_hello(self):
res = self.__test.get('/users/api/v1.0/hello')
assert (res.status_code == 200)
res_str = json.loads(res.data)
assert (res_str == 'User Service Desk at your service !\n')