class TestWebPageService(unittest.TestCase):
@mock.patch("pdm.web.WebPageService.HRClient")
@mock.patch("pdm.web.WebPageService.SiteClient")
def setUp(self, hr_mock, ep_mock):
hr_mock.return_value = object() # no MockHRClient available
ep_mock.return_value = object() # no MockSiteClient available
conf = {}
self.__service = FlaskServer("pdm.web.WebPageService")
self.__service.test_mode(WebPageService, conf, with_test=True)
self.__service.fake_auth("ALL")
self.__test = self.__service.test_client()
def test_web(self):
""" Check that the basic website redirect + index page work. """
res = self.__test.get('/')
# 302 == redirect, assertEqual is from unittest
self.assertEqual(res.status_code, 302)
self.assertIn('/web/datamover', res.location)
res = self.__test.get('/web/datamover')
self.assertEqual(res.status_code, 200)
self.assertIn("<!doctype html>", res.data)
def test_about(self):
""" Check that the about page is returned at the correct location. """
res = self.__test.get('/static/about.html')
# check its existence
self.assertEqual(res.status_code, 200)
self.assertIn("About the datamover", res.data)
class TestDemoClient(unittest.TestCase):
def setUp(self):
# Get an instance of DemoService to test against
conf = {'test_param': 1111}
self.__service = FlaskServer("pdm.demo.DemoService")
self.__service.test_mode(DemoService, conf, with_test=True)
self.__service.fake_auth("ALL")
self.__test = self.__service.test_client()
# Create an instance of DemoClient connected to DemoService
patcher, inst = RESTClientTest.patch_client(DemoClient, self.__test,
'/demo/api/v1.0')
self.__patcher = patcher
self.__client = inst
def tearDown(self):
self.__patcher.stop()
def test_hello(self):
assert (self.__client.hello() == 'Hello World!\n')
def test_getTurtles(self):
turtles = self.__client.get_turtles()
assert (len(turtles) == 3)
assert (turtles['1'] == 'Timmy')
def test_addTurtle(self):
new_turtle = self.__client.add_turtle('Test Turtle')
assert (new_turtle['id'] == 4)
assert (new_turtle['name'] == 'Test Turtle')
turtles = self.__client.get_turtles()
assert (len(turtles) == 4)
def test_delTurtle(self):
self.__client.del_turtle(3)
turtles = self.__client.get_turtles()
assert (len(turtles) == 2)
def test_modifyTurtle(self):
tid = 1
modif_turtle = self.__client.modify_turtle(tid, 'TimmyBoy')
assert (modif_turtle)['id'] == tid
assert (modif_turtle)['name'] == 'TimmyBoy'
turtles = self.__client.get_turtles()
assert (turtles['1']) == 'TimmyBoy'
assert (len(turtles) == 3)
def test_init_existingDB(self):
""" Tests that the config step completes correctly if a DB already
exists.
"""
# Create a clean copy of the service
service = FlaskServer("pdm.demo.DemoService")
service.test_mode(DemoService, None)
service.build_db()
# Add a single turtle to the table
db = service.test_db()
new_turtle = db.tables.Turtle(name='Before')
db.session.add(new_turtle)
db.session.commit()
# Continue service start-up
service.before_startup({}, with_test=True)
service.fake_auth("ALL")
client = service.test_client()
# Now check that we only have one turtle
# Rather than the 3 we get by default
res = client.get('/demo/api/v1.0/turtles')
assert(res.status_code == 200)
assert(len(json.loads(res.data)) == 1)
class TestTransferClient(unittest.TestCase):
# @mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient")
# @mock.patch("pdm.userservicedesk.TransferClient.WorkqueueClient")
##@mock.patch.object(HRService, 'check_token')
@mock.patch("pdm.userservicedesk.HRService.SiteClient")
@mock.patch.object(Tokens.TokenService, 'unpack')
@mock.patch("pdm.userservicedesk.TransferClient.SiteClient")
@mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient.__new__")
def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock):
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
site_mock().get_sites.return_value = \
[{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'},
{'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}]
site_mock.return_value.set_token = mock.MagicMock()
self.site_id = site_mock().get_sites()[0]['site_id']
self.site2_id = site_mock().get_sites()[1]['site_id']
conf = {
'CS_secret': 'HJGnbfdsV',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
# HR
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
token = {'id': 1, 'expiry': self.__future_date}
self.__service.fake_auth("TOKEN", token)
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
self.__service.before_startup(conf) # to continue startup
# mock_ct.return_value =1
mocked_unpack.return_value = token
self.__htoken = 'whateverhash'
self.__client = TransferClientFacade(self.__htoken)
assert wq_mock.called
mocked_unpack.assert_called_with(self.__htoken)
def test_list(self):
site = "localhost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt'
assert self.__client.list(site, **{'priority':
2}) == 'root/file.txt'
assert mock_list.called
mock_list.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_list.call_args_list
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt' # event if ...
assert self.__client.list(wrongurl, **{'priority':
2}) == [] # we return []
assert not mock_list.called
def test_sitelist(self):
sites = self.__client.list_sites()
print sites
assert sites[0]['site_name'] == 'localhost'
assert sites[1]['site_name'] == 'remotehost'
assert 'site_id' not in [dd.keys() for dd in sites]
def test_remove(self):
site = "localhost:/root/file.txt"
# mock_remove.return_value = 'root/file.txt'
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'root/file.txt removed'
assert self.__client.remove(site, **{'priority':
2}) == 'root/file.txt removed'
assert mock_remove.called
mock_remove.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_remove.call_args_remove
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'whatever..' # event if ...
assert self.__client.remove(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_remove.called
def test_copy(self):
s_site = "localhost:/root/file.txt"
t_site = "remotehost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'root/file.txt copied'
assert self.__client.copy(s_site, t_site,
**{'priority':
2}) == 'root/file.txt copied'
assert mock_copy.called
mock_copy.assert_called_with(self.site_id,
'/root/file.txt',
self.site2_id,
'/root/file.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'whatever..' # even if ...
assert self.__client.copy(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_copy.called
def test_split_site_path(self):
site = "localhost:/root/file.txt"
malformed_site = "localhost/root/file.txt" # mind a missing colon
multicolon_site = "localhost:/root/file.txt:1"
a, b = TransferClientFacade.split_site_path(site)
assert a == 'localhost'
assert b == '/root/file.txt'
c, d = TransferClientFacade.split_site_path(malformed_site)
assert d is None
assert c is None
e, f = TransferClientFacade.split_site_path(multicolon_site)
assert e == 'localhost'
assert f == '/root/file.txt:1'
def test_mkdir(self):
site = "localhost:/root/subdir"
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'root/subdir created'
assert self.__client.mkdir(site, **{'priority':
2}) == 'root/subdir created'
assert mock_mkdir.called
mock_mkdir.assert_called_with(self.site_id, '/root/subdir', priority=2)
# now unknown site:
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'whatever..' # event if ...
assert self.__client.mkdir(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_mkdir.called
def test_rename(self):
s_site = "localhost:/root/file.txt"
t_site = ":/root/file2.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'root/file.txt renamed'
assert self.__client.rename(s_site, t_site,
**{'priority':
2}) == 'root/file.txt renamed'
assert mock_rename.called
mock_rename.assert_called_with(self.site_id,
'/root/file.txt',
'/root/file2.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'whatever..' # even if ...
assert self.__client.rename(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_rename.called
def tearDown(self):
pass
class TestAlgorithms(unittest.TestCase):
def setUp(self):
conf = {'workerlogs': '/tmp/workers'}
self.__service = FlaskServer("pdm.workqueue.WorkqueueService")
self.__service.test_mode(WorkqueueService,
None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__service.build_db() # build manually
db = self.__service.test_db()
Job = db.tables.Job
JobElement = db.tables.JobElement
db.session.add(
Job(user_id=1,
src_siteid=13,
src_filepath='/data/somefile1',
type=JobType.LIST))
job = Job(user_id=2,
src_siteid=14,
src_filepath='/data/somefile2',
type=JobType.REMOVE)
for i in xrange(1, 6):
job.elements.append(
JobElement(id=i,
job_id=2,
src_siteid=12,
src_filepath='/data/somefile2.%d' % i,
type=JobType.REMOVE,
size=10**i))
db.session.add(job)
j = Job(user_id=3,
type=JobType.COPY,
src_siteid=15,
src_filepath='/data/somefile3',
dst_siteid=16,
dst_filepath='/data/newfile')
for i in xrange(1, 6):
j.elements.append(
JobElement(id=i,
job_id=3,
src_siteid=12,
src_filepath='/data/somefile3.%d' % i,
dst_filepath='/data/newfile.%d' % i,
type=JobType.COPY,
size=10**i))
db.session.add(j)
db.session.commit()
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self.__service.before_startup(conf) # to continue startup
self.__test = self.__service.test_client()
@mock.patch('pdm.workqueue.WorkqueueService.request')
def test_by_number(self, mock_request):
mock_request.db.tables.Job = self.__service.test_db().tables.Job
mock_request.db.tables.JobElement = self.__service.test_db(
).tables.JobElement
mock_request.data = {"types": (JobType.RENAME, JobType.MKDIR)}
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 0)
mock_request.data["types"] = (JobType.LIST, )
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 1)
mock_request.data["types"] = (JobType.LIST, JobType.COPY)
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 7)
mock_request.data["types"] = (JobType.LIST, JobType.COPY,
JobType.REMOVE)
self.assertEqual(len(Algorithm["BY_NUMBER"]()), 13)
self.assertEqual(len(Algorithm["BY_NUMBER"](5)), 5)
self.assertEqual(len(Algorithm["BY_NUMBER"](8)), 8)
@mock.patch('pdm.workqueue.WorkqueueService.request')
def test_by_size(self, mock_request):
mock_request.db.tables.Job = self.__service.test_db().tables.Job
mock_request.db.tables.JobElement = self.__service.test_db(
).tables.JobElement
mock_request.data = {"types": (JobType.RENAME, JobType.MKDIR)}
self.assertEqual(len(Algorithm["BY_SIZE"]()), 0)
mock_request.data["types"] = (JobType.LIST, )
self.assertEqual(len(Algorithm["BY_SIZE"]()), 1)
mock_request.data["types"] = (JobType.LIST, JobType.COPY)
self.assertEqual(len(Algorithm["BY_SIZE"]()), 7)
mock_request.data["types"] = (JobType.LIST, JobType.COPY,
JobType.REMOVE)
self.assertEqual(len(Algorithm["BY_SIZE"]()), 13)
self.assertEqual(len(Algorithm["BY_SIZE"](100)), 3)
self.assertEqual(len(Algorithm["BY_SIZE"](111)), 4)
class TestWorkqueueService(unittest.TestCase):
def setUp(self):
conf = {'workerlogs': '/tmp/workers'}
self.__service = FlaskServer("pdm.workqueue.WorkqueueService")
self.__service.test_mode(WorkqueueService,
None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__service.build_db() # build manually
db = self.__service.test_db()
Job = db.tables.Job
JobElement = db.tables.JobElement
db.session.add(
Job(user_id=1,
src_siteid=13,
src_filepath='/data/somefile1',
type=JobType.LIST))
job = Job(user_id=2,
src_siteid=14,
src_filepath='/data/somefile2',
type=JobType.REMOVE)
for i in xrange(1, 6):
job.elements.append(
JobElement(id=i,
job_id=2,
src_siteid=12,
src_filepath='/data/somefile2.%d' % i,
type=JobType.REMOVE,
size=10**i))
db.session.add(job)
j = Job(user_id=3,
type=JobType.COPY,
src_siteid=15,
src_filepath='/data/somefile3',
dst_siteid=16,
dst_filepath='/data/newfile')
for i in xrange(1, 6):
j.elements.append(
JobElement(id=i,
job_id=3,
src_siteid=12,
src_filepath='/data/somefile3.%d' % i,
dst_filepath='/data/newfile.%d' % i,
type=JobType.COPY,
size=10**i))
db.session.add(j)
db.session.commit()
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self.__service.before_startup(conf) # to continue startup
self.__test = self.__service.test_client()
def test_get_next_job(self):
"""test worker get next job."""
request = self.__test.post('/workqueue/api/v1.0/worker/jobs',
data={'test': 12})
self.assertEqual(request.status_code, 400,
"Expected job with incorrect attrs to fail.")
request = self.__test.post('/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.LIST]})
self.assertEqual(request.status_code, 200,
"Request to get worker job failed.")
work = json.loads(request.data)
self.assertEqual(len(work), 1)
job = work[0]
self.assertEqual(len(job['elements']), 1)
self.assertDictContainsSubset(
{
'status': JobStatus.SUBMITTED,
'dst_credentials': None,
'user_id': 1,
'src_filepath': '/data/somefile1',
'priority': 5,
'dst_siteid': None,
'src_siteid': 13,
'extra_opts': None,
'protocol': 0,
'type': JobType.LIST,
'id': 1,
'src_credentials': None,
'dst_filepath': None
}, job) #, "Job not returned correctly.")
element = job['elements'][0]
self.assertIsInstance(element['token'], basestring)
self.assertDictContainsSubset(
{
'status': JobStatus.SUBMITTED,
'job_id': 1,
'attempts': 0,
'src_filepath': '/data/somefile1',
'listing': None,
'max_tries': 2,
'type': JobType.LIST,
'id': 0,
'dst_filepath': None
}, element)
Job = self.__service.test_db().tables.Job
JobElement = self.__service.test_db().tables.JobElement
#j = Job.query.filter_by(id=job['id']).one()
j = Job.query.filter_by(id=job['id']).one()
je = JobElement.query.filter_by(id=element['id'],
job_id=element['job_id']).one()
self.assertIsNotNone(j)
self.assertIsNotNone(je)
self.assertEqual(j.status, JobStatus.SUBMITTED,
"Job status not updated in DB")
self.assertEqual(je.status, JobStatus.SUBMITTED,
"Job status not updated in DB")
request = self.__test.post(
'/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.COPY, JobType.REMOVE]})
self.assertEqual(request.status_code, 200,
"Failed to get copy or remove job.")
work = json.loads(request.data)
self.assertEqual(len(work), 2)
self.assertEqual(work[0]['type'], JobType.REMOVE)
self.assertEqual(len(work[0]['elements']), 6)
self.assertEqual(work[0]['elements'][0]['type'], JobType.LIST)
for i in xrange(1, 6):
self.assertEqual(work[0]['elements'][i]['type'], JobType.REMOVE)
self.assertEqual(work[1]['type'], JobType.COPY)
self.assertEqual(len(work[1]['elements']), 6)
# up to 10 loaded now at once
#request = self.__test.post('/workqueue/api/v1.0/worker/jobs', data={'types': [JobType.COPY, JobType.REMOVE]})
#self.assertEqual(request.status_code, 200, "Failed to get copy or remove job.")
request = self.__test.post(
'/workqueue/api/v1.0/worker/jobs',
data={'types': [JobType.COPY, JobType.REMOVE]})
self.assertEqual(
request.status_code, 404,
"Trying to get a job that doesn't exist should return 404.")
def test_return_output(self):
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 403)
self.__service.fake_auth("TOKEN", "12")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 403)
self.__service.fake_auth("TOKEN", "1.2")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'returncode': 0,
'host': 'somehost.domain'
})
self.assertEqual(request.status_code, 400)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/2',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 404)
self.__service.fake_auth("TOKEN", "1.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 400)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 1,
'host': 'somehost.domain',
'timestamp': 'timestamp'
})
self.assertEqual(request.status_code, 200)
Job = self.__service.test_db().tables.Job
JobElement = self.__service.test_db().tables.JobElement
je = JobElement.query.filter_by(job_id=1, id=0).one()
j = Job.query.filter_by(id=1).one()
self.assertIsNotNone(je)
self.assertEqual(je.status, JobStatus.FAILED)
self.assertEqual(je.job.status, JobStatus.FAILED)
self.assertIsNotNone(j)
self.assertEqual(j.status, JobStatus.FAILED)
logfile = os.path.join('/tmp/workers', j.log_uid[:2], j.log_uid,
str(je.id), 'attempt1.log')
self.assertTrue(os.path.isfile(logfile))
expected_log = dedent("""
Job run on host: somehost.domain, returncode: 1, timestamp: timestamp
blah blah
""").strip()
with open(logfile, 'rb') as log:
self.assertEqual(log.read(), expected_log)
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'root': []
}
})
self.assertEqual(request.status_code, 200)
je = JobElement.query.filter_by(job_id=1, id=0).one()
j = Job.query.filter_by(id=1).one()
self.assertIsNotNone(je)
self.assertEqual(je.status, JobStatus.DONE)
self.assertEqual(je.job.status, JobStatus.DONE)
self.assertIsNotNone(j)
self.assertEqual(j.status, JobStatus.DONE)
logfile = os.path.join('/tmp/workers', j.log_uid[:2], j.log_uid,
str(je.id), 'attempt2.log')
self.assertTrue(os.path.isfile(logfile))
expected_log = dedent("""
Job run on host: somehost.domain, returncode: 0, timestamp: timestamp
blah blah
""").strip()
with open(logfile, 'rb') as log:
self.assertEqual(log.read(), expected_log)
self.assertEqual(je.listing, {'root': []})
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/1/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'root': []
}
})
self.assertEqual(request.status_code, 400,
'Exceeding max tries should give 400')
# Test expansion of COPY jobs
db = self.__service.test_db()
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data/somefile',
dst_siteid=16,
dst_filepath='/site2/data/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data/somefile',
dst_siteid=16,
dst_filepath='~/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='~/somefile',
dst_siteid=16,
dst_filepath='~/someotherfile',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
dst_siteid=16,
dst_filepath='/site2/data/somedir',
type=JobType.COPY))
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
dst_siteid=16,
dst_filepath='~',
type=JobType.COPY))
j = Job.query.filter_by(id=4).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=5).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=6).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=7).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
j = Job.query.filter_by(id=8).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
self.__service.fake_auth("TOKEN", "4.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/4/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 200,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=4).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath,
'/site2/data/someotherfile')
self.__service.fake_auth("TOKEN", "5.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/5/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 200,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=5).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath, '~/someotherfile')
self.__service.fake_auth("TOKEN", "6.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/6/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'~': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=6).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 2)
self.assertEqual(j.elements[1].dst_filepath, '~/someotherfile')
self.__service.fake_auth("TOKEN", "7.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/7/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=7).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 3)
self.assertEqual(j.elements[1].dst_filepath,
'/site2/data/somedir/somefile')
self.assertEqual(j.elements[2].dst_filepath,
'/site2/data/somedir/someotherdir/someotherfile')
self.__service.fake_auth("TOKEN", "8.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/8/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=8).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 3)
self.assertEqual(j.elements[1].dst_filepath, '~/somefile')
self.assertEqual(j.elements[2].dst_filepath,
'~/someotherdir/someotherfile')
# Test expansion of REMOVE jobs
db.session.add(
Job(user_id=3,
src_siteid=15,
src_filepath='/site1/data',
type=JobType.REMOVE))
j = Job.query.filter_by(id=9).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 1)
self.__service.fake_auth("TOKEN", "9.0")
request = self.__test.put(
'/workqueue/api/v1.0/worker/jobs/9/elements/0',
data={
'log': 'blah blah',
'returncode': 0,
'host': 'somehost.domain',
'timestamp': 'timestamp',
'listing': {
'/site1/data': [{
'name': 'somefile',
'st_size': 100,
'st_mode': 0o0100655
}, {
'name': 'someotherdir',
'st_size': 200,
'st_mode': 0o040655
}],
'/site1/data/someotherdir': [{
'name': 'someotherfile',
'st_size': 300,
'st_mode': 0o0100655
}, {
'name': 'somedir',
'st_size': 400,
'st_mode': 0o040655
}]
}
})
self.assertEqual(request.status_code, 200)
j = Job.query.filter_by(id=9).one()
self.assertIsNotNone(j)
self.assertEqual(len(j.elements), 6)
self.assertEqual(j.elements[1].src_filepath,
'/site1/data/someotherdir/someotherfile')
self.assertEqual(j.elements[2].src_filepath,
'/site1/data/someotherdir/somedir/')
self.assertEqual(j.elements[3].src_filepath, '/site1/data/somefile')
self.assertEqual(j.elements[4].src_filepath,
'/site1/data/someotherdir/')
self.assertEqual(j.elements[5].src_filepath, '/site1/data/')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_post_job(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={'blah': 12})
self.assertEqual(request.status_code, 400)
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.LIST,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.status, JobStatus.NEW)
self.assertEqual(returned_job['status'], 'NEW')
self.assertEqual(job.type, JobType.LIST)
self.assertEqual(returned_job['type'], 'LIST')
self.assertEqual(job.priority, 5)
self.assertEqual(job.src_credentials, 'somesecret')
self.assertEqual(job.protocol, JobProtocol.GRIDFTP)
self.assertEqual(returned_job['protocol'], 'GRIDFTP')
self.assertIsInstance(job.log_uid, basestring)
self.assertEqual(len(job.elements), 1)
element = job.elements[0]
self.assertEqual(element.type, JobType.LIST)
self.assertEqual(element.src_filepath, '/data/somefile')
self.assertIsNone(element.dst_filepath)
self.assertEqual(element.attempts, 0)
self.assertEqual(element.max_tries, 2)
mock_hrservice.return_value = 12
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15
})
self.assertEqual(request.status_code, 400)
mock_siteclient.site_client.get_cred = mock.MagicMock(
side_effect=["somesecret", "someothersecret"])
# mock_siteclient().get_cred = mock.MagicMock(side_effect=["somesecret", "someothersecret"])
request = self.__test.post('/workqueue/api/v1.0/jobs',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile',
'extra_opts': {},
'attempts': 30,
'max_tries': 3,
'priority': 2,
'protocol': JobProtocol.SSH,
'log_uid': 'my_log_uid'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=12).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.status, JobStatus.NEW)
self.assertEqual(returned_job['status'], 'NEW')
self.assertEqual(job.type, JobType.COPY)
self.assertEqual(returned_job['type'], 'COPY')
self.assertEqual(job.priority, 2)
self.assertEqual(job.src_filepath, '/data/somefile')
self.assertEqual(job.dst_filepath, '/data/someotherfile')
self.assertEqual(job.src_credentials, "somesecret")
self.assertEqual(job.dst_credentials, "someothersecret")
self.assertEqual(job.protocol, JobProtocol.SSH)
self.assertEqual(returned_job['protocol'], 'SSH')
self.assertIsInstance(job.log_uid, basestring)
self.assertNotEqual(job.log_uid, 'my_log_uid')
self.assertEqual(len(job.elements), 1)
element = job.elements[0]
self.assertEqual(element.type, JobType.LIST)
self.assertEqual(element.src_filepath, '/data/somefile')
self.assertEqual(element.dst_filepath, None)
self.assertEqual(element.attempts, 0)
self.assertEqual(element.max_tries, 3)
# @mock.patch.object(HRService.HRService, 'check_token')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_list(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
# mock_siteclient().get_cred = mock.MagicMock(return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/list',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.LIST)
self.assertEqual(returned_job['type'], 'LIST')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_copy(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
side_effect=["somesecret", "someothersecret"])
# mock_siteclient().get_cred = mock.MagicMock(side_effect=["somesecret", "someothersecret"])
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/copy',
data={
'type': JobType.LIST,
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.COPY)
self.assertEqual(returned_job['type'], 'COPY')
# @mock.patch('pdm.workqueue.WorkqueueService.SiteClient')
@mock.patch('pdm.workqueue.WorkqueueService.current_app')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_remove(self, mock_hrservice, mock_siteclient):
Job = self.__service.test_db().tables.Job
mock_siteclient.site_client.get_cred = mock.MagicMock(
return_value="somesecret")
# mock_siteclient().get_cred = mock.MagicMock(return_value="somesecret")
mock_hrservice.return_value = 10
request = self.__test.post('/workqueue/api/v1.0/remove',
data={
'type': JobType.COPY,
'src_siteid': 12,
'src_filepath': '/data/somefile'
})
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
job = Job.query.filter_by(user_id=10).one()
self.assertIsNotNone(job)
self.assertEqual(returned_job, json.loads(job.json()))
self.assertEqual(job.type, JobType.REMOVE)
self.assertEqual(returned_job['type'], 'REMOVE')
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_jobs(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs')
self.assertEqual(request.status_code, 200)
returned_jobs = json.loads(request.data)
self.assertEqual(returned_jobs, [])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs')
self.assertEqual(request.status_code, 200)
returned_jobs = json.loads(request.data)
self.assertEqual(len(returned_jobs), 1)
self.assertDictContainsSubset(
{
'user_id': 1,
'type': 'LIST',
'status': 'NEW'
}, returned_jobs[0])
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_job(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/2')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/1')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2')
self.assertEqual(request.status_code, 200)
returned_job = json.loads(request.data)
self.assertDictContainsSubset(
{
'user_id': 2,
'type': 'REMOVE',
'status': 'NEW'
}, returned_job)
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_elements(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements')
self.assertEqual(request.status_code, 200)
returned_elements = json.loads(request.data)
self.assertEqual(returned_elements, [])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements')
self.assertEqual(request.status_code, 200)
returned_elements = json.loads(request.data)
self.assertEqual(len(returned_elements), 1)
self.assertDictContainsSubset(
{
'job_id': 1,
'type': 'LIST',
'status': 'NEW'
}, returned_elements[0])
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_element(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/0')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/1/elements/0')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/33')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/elements/0')
self.assertEqual(request.status_code, 200)
returned_element = json.loads(request.data)
self.assertDictContainsSubset(
{
'job_id': 2,
'type': 'LIST',
'status': 'NEW'
}, returned_element)
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_job_status(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/3/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get('/workqueue/api/v1.0/jobs/1/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get('/workqueue/api/v1.0/jobs/3/status')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, {'jobid': 3, 'status': 'NEW'})
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_element_status(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/17/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/12/elements/0/status')
self.assertEqual(request.status_code, 404)
mock_hrservice.return_value = 3
request = self.__test.get(
'/workqueue/api/v1.0/jobs/3/elements/0/status')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 3,
'elementid': 0,
'status': 'NEW',
'attempts': 0,
'transferred': 'N/A',
'instant': 'N/A',
'average': 'N/A',
'elapsed': 'N/A'
})
@mock.patch('pdm.userservicedesk.HRService.HRService.check_token')
def test_get_output(self, mock_hrservice):
mock_hrservice.return_value = 10
request = self.__test.get('/workqueue/api/v1.0/jobs/1/output')
self.assertEqual(request.status_code, 404,
"invalid userid should give 404")
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/2/output')
self.assertEqual(request.status_code, 404,
"userid not matching job id should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/10/output')
self.assertEqual(request.status_code, 404,
"Unknown element id should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output')
self.assertEqual(request.status_code, 404,
"Status other than DONE or FAILED gives 404")
db = self.__service.test_db()
session = db.session
list_job = db.tables.Job.query.filter_by(user_id=1).one()
remove_job = db.tables.Job.query.filter_by(user_id=2).one()
self.assertIsNotNone(list_job)
self.assertIsNotNone(remove_job)
list_job.status = JobStatus.DONE
remove_job.status = JobStatus.FAILED
list_job.elements[0].status = JobStatus.DONE
remove_job.elements[0].status = JobStatus.DONE
remove_job.elements[1].status = JobStatus.FAILED
session.merge(list_job)
session.merge(remove_job)
session.expunge(list_job)
session.expunge(remove_job)
session.commit()
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output')
self.assertEqual(
request.status_code, 404,
"Element with 0 attempts is not yet ready, should give 404")
list_job.elements[0].attempts = 1
remove_job.elements[0].attempts = 1
remove_job.elements[1].attempts = 1
list_job.elements[0].listing = {'root': [{'name': 'somefile'}]}
remove_job.elements[0].listing = {'root': [{'name': 'somefile'}]}
session.merge(list_job)
session.merge(remove_job)
session.commit()
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output/9')
self.assertEqual(request.status_code, 404,
"Invalid attempt should give 404")
mock_hrservice.return_value = 1
request = self.__test.get(
'/workqueue/api/v1.0/jobs/1/elements/0/output/1')
self.assertEqual(request.status_code, 500,
"Failure to find logfile should give 500")
list_job_dir = os.path.join('/tmp/workers', list_job.log_uid[:2],
list_job.log_uid, '0')
remove_job_dir = os.path.join('/tmp/workers', remove_job.log_uid[:2],
remove_job.log_uid)
remove_job0_dir = os.path.join(remove_job_dir, '0')
remove_job1_dir = os.path.join(remove_job_dir, '1')
list_job_filename = os.path.join(list_job_dir, "attempt1.log")
remove_job0_filename = os.path.join(remove_job0_dir, "attempt1.log")
remove_job1_filename = os.path.join(remove_job1_dir, "attempt1.log")
if not os.path.exists(list_job_dir):
os.makedirs(list_job_dir)
if not os.path.exists(remove_job0_dir):
os.makedirs(remove_job0_dir)
if not os.path.exists(remove_job1_dir):
os.makedirs(remove_job1_dir)
with open(list_job_filename, 'wb') as listlog,\
open(remove_job0_filename, 'wb') as removelog0,\
open(remove_job1_filename, 'wb') as removelog1:
listlog.write('la la la\n')
removelog0.write('blah blah\n')
removelog1.write('tralala\n')
mock_hrservice.return_value = 2
request = self.__test.get('/workqueue/api/v1.0/jobs/2/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [[{
'status': 'DONE',
'attempt': 1,
'log': 'blah blah\n',
'jobid': 2,
'listing': {
'root': [{
'name': 'somefile'
}]
},
'type': 'LIST',
'elementid': 0
}],
[{
'status': 'FAILED',
'attempt': 1,
'log': 'tralala\n',
'jobid': 2,
'type': 'REMOVE',
'elementid': 1
}], [], [], [], []])
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [{
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
}])
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output/1')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
})
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/0/output/-1')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(
returned_dict, {
'jobid': 2,
'elementid': 0,
'type': 'LIST',
'attempt': 1,
'status': 'DONE',
'log': 'blah blah\n',
'listing': {
'root': [{
'name': 'somefile'
}]
}
})
request = self.__test.get(
'/workqueue/api/v1.0/jobs/2/elements/1/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [{
'jobid': 2,
'elementid': 1,
'type': 'REMOVE',
'attempt': 1,
'status': 'FAILED',
'log': 'tralala\n'
}])
mock_hrservice.return_value = 1
request = self.__test.get('/workqueue/api/v1.0/jobs/1/output')
self.assertEqual(request.status_code, 200)
returned_dict = json.loads(request.data)
self.assertEqual(returned_dict, [[{
'status': 'DONE',
'attempt': 1,
'log': 'la la la\n',
'jobid': 1,
'listing': {
'root': [{
'name': 'somefile'
}]
},
'type': 'LIST',
'elementid': 0
}]])
class TestHRService(unittest.TestCase):
@mock.patch("pdm.userservicedesk.HRService.SiteClient")
def setUp(self, site_mock):
self.__site_mock = site_mock
conf = {
'token_validity': '01:00:00',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
self._conf = copy.deepcopy(conf)
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
self.__service.fake_auth("ALL")
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
self.__past_date = (-datetime.timedelta(0, 60) +
datetime.datetime.utcnow()).isoformat()
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
new_user = db.tables.User(name='John',
surname='Smith',
email='*****@*****.**',
state=HRServiceUserState.VERIFIED,
password=hash_pass('very_secret'))
db.session.add(new_user)
db.session.commit()
self.__service.before_startup(conf) # to continue startup
#
self.__test = self.__service.test_client()
# mail token
time_struct = time.strptime("12:00:00", "%H:%M:%S")
self.token_duration = datetime.timedelta(hours=time_struct.tm_hour,
minutes=time_struct.tm_min,
seconds=time_struct.tm_sec)
self.mail_token_service = TokenService(self._conf['mail_token_secret'])
def test_getUserSelf(self):
"""
GET operation on users/self
:return:
"""
self.__service.fake_auth("TOKEN", {'id': 1})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 500)
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 403) # token expired
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date,
'key': 'unused'
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 200)
user = json.loads(res.data)
assert ('id' not in user)
assert (user['name'] == 'John')
assert (user['surname'] == 'Smith')
assert (user['email'] == '*****@*****.**')
assert (user['state'] == HRServiceUserState.VERIFIED)
assert ('password' not in user)
#
self.__service.fake_auth("TOKEN", {
'id': 2,
'expiry': self.__future_date
})
res = self.__test.get('/users/api/v1.0/users/self')
assert (res.status_code == 404)
@mock.patch("pdm.userservicedesk.HRService.HRService.email_user")
def test_addUser(self, email_user_mock):
"""
Testinf user registration. Ignore emailer at this stage.
:return:
"""
self.__service.fake_auth("ALL")
fred = {
'surname': 'Flintstone',
'name': 'Fred',
'email': '*****@*****.**',
'state': 0,
'password': '******'
}
barney = {
'surname': 'Rubble',
'name': 'Barney',
'email': '*****@*****.**',
'state': 0,
'password': '******'
}
res = self.__test.post('/users/api/v1.0/users', data=fred)
assert (res.status_code == 201)
# db
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(email=fred['email']).first()
assert (dbuser.name == fred['name'])
assert (check_hash(dbuser.password, fred['password']))
assert (dbuser.email == fred['email'])
response = json.loads(res.data)
assert (response['name'] == fred['name'])
assert (response['surname'] == fred['surname'])
assert (response['email'] == fred['email'])
assert (response['state'] == fred['state'])
assert ('password' not in response)
# try to duplicate the user:
res = self.__test.post('/users/api/v1.0/users', data=fred)
assert (res.status_code == 403)
# password too short !
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
# barney OK, but verification email sending fails:
barney['password'] = '******'
email_user_mock.side_effect = RuntimeError
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 500)
#
b_email = barney.pop('email')
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
barney['email'] = b_email
password = barney.pop('password')
res = self.__test.post('/users/api/v1.0/users', data=barney)
assert (res.status_code == 400)
def test_change_password(self):
"""
Test the password changing operation
:return:
"""
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date
})
new_pass_data = {
'passwd': 'very_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 403)
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
new_pass_data = {
'passwd': 'very_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 200)
# check if the password was actually modified:
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert (dbuser.name == "John")
assert (check_hash(dbuser.password, 'even_more_secret'))
#
response = json.loads(res.data)
assert ('password' not in response)
# TODO check last login timestamp later than the time before changing the password.
# wrong password
wrong_pass_data = {
'passwd': 'very_sercet',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=wrong_pass_data)
assert (res.status_code == 403)
# same pass
same_pass_data = {
'passwd': 'even_more_secret',
'newpasswd': 'even_more_secret'
}
res = self.__test.put('/users/api/v1.0/passwd', data=same_pass_data)
assert (res.status_code == 400)
# no pass
no_pass = {'passwd': None, 'newpasswd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
no_pass = {'newpasswd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
no_pass = {'passwd': 'even_more_secret'}
res = self.__test.put('/users/api/v1.0/passwd', data=no_pass)
assert (res.status_code == 400)
#
no_npass = {'passwd': 'even_more_secret', 'newpasswd': None}
res = self.__test.put('/users/api/v1.0/passwd', data=no_npass)
assert (res.status_code == 400)
# weak pass
weak_pass = {'passwd': 'even_more_secret', 'newpasswd': 'test'}
res = self.__test.put('/users/api/v1.0/passwd', data=weak_pass)
assert (res.status_code == 400)
# non existing user
self.__service.fake_auth("TOKEN", {
'id': 7,
'expiry': self.__future_date
})
res = self.__test.put('/users/api/v1.0/passwd', data=new_pass_data)
assert (res.status_code == 403)
# @mock.patch('pdm.userservicedesk.HRService.SiteClient')
def test_delete_user(self):
"""
Test deleting user data
:return:
"""
# not existing user:
self.__service.fake_auth("TOKEN", {
'id': 7,
'expiry': self.__future_date
})
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 404)
assert not self.__site_mock().del_user.called
# attempt to delete Johnny with an expired token
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__past_date,
'key': 'unused'
}) # fake auth John, which is id=1
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 403)
assert not self.__site_mock().del_user.called
# delete poor Johnny ;-(
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date,
'key': 'unused'
}) # fake auth John, which is id=1
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 200)
assert self.__site_mock().del_user.called
def test_deleteUser_SiteService_fail(self):
"""
Test if the user is put back when SiteService fails
:param mock_del_user:
:return:
"""
# delete poor Johnny ;-(
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
self.__site_mock().del_user.side_effect = Exception()
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 500)
assert self.__site_mock().del_user.called
# check if we rolled John back !
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert (dbuser is not None)
@mock.patch('sqlalchemy.orm.scoping.scoped_session.delete')
def test_deleteUser_HR_fail(self, mock_del):
self.__service.fake_auth("TOKEN", {
'id': 1,
'expiry': self.__future_date
})
# fake auth John, which is id=1
mock_del.side_effect = Exception()
res = self.__test.delete('/users/api/v1.0/users/self')
assert (res.status_code == 500)
assert not self.__site_mock().del_user.called
def test_loginUser(self):
"""
Test the user login procedure
:return:
"""
res = self.__test.post('/users/api/v1.0/login') # empty req.
assert (res.status_code == 400)
res = self.__test.post('/users/api/v1.0/login', data=('hulagula'))
assert (res.status_code == 400)
login_creds = {'email': '*****@*****.**', 'passwd': 'very_secret'}
res = self.__test.post('/users/api/v1.0/login', data=login_creds)
assert (res.status_code == 200)
#
token_data = self.__service.token_svc.check(json.loads(res.data))
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
assert token_data['id'] == 1
isoformat = '%Y-%m-%dT%H:%M:%S.%f'
expiry_date = datetime.datetime.strptime(token_data['expiry'],
isoformat)
# conf gives 1h token validity. Check if we are within 10s
assert abs(
(expiry_date - (datetime.datetime.utcnow() +
datetime.timedelta(0, 3600))).total_seconds()) < 10
login_creds_b = {'email': '*****@*****.**'}
res = self.__test.post('/users/api/v1.0/login', data=login_creds_b)
assert (res.status_code == 400)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': 'very_seCret'
})
assert (res.status_code == 403)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': 'very_secret'
})
assert (res.status_code == 403)
res = self.__test.post('/users/api/v1.0/login',
data={
'email': '*****@*****.**',
'passwd': None
})
assert (res.status_code == 400)
# make Johny unverified ;-(
dbuser.state = HRServiceUserState.REGISTERED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/login', data=login_creds)
assert (res.status_code == 401)
@mock.patch('smtplib.SMTP')
@mock.patch.object(HRService, 'compose_and_send')
def test_email_user(self, mcs, smtp_mock):
with self.__service.test_request_context(path="/test"):
with mock.patch.object(pdm.userservicedesk.HRService.current_app,
'mail_token_service') as m_ts:
m_ts.issue = mock.MagicMock(return_value='agfgffsgdf')
HRService.email_user("*****@*****.**")
assert mcs.call_args[0][0] == '*****@*****.**'
assert mcs.call_args[0][1] == 'agfgffsgdf'
#(ignore the timestamp passed in as the third arg.)
def test_verify_user(self):
# isssue a valid mail token
expiry = datetime.datetime.utcnow() + self.token_duration
plain = {'expiry': expiry.isoformat(), 'email': '*****@*****.**'}
token = self.mail_token_service.issue(plain)
#body = os.path.join(self._conf['verification_url'],token)
# verify takes a token only, not the whole email body at the moment
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
#token tempered with:
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token[1:]})
assert res.status_code == 400
# success ?
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 201
#repeat a verification attempt
dbuser.state = HRServiceUserState.VERIFIED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 400
# expired token
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
expired = datetime.datetime.utcnow() - self.token_duration
e_plain = {
'expiry': expired.isoformat(),
'email': '*****@*****.**'
}
e_token = self.mail_token_service.issue(e_plain)
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': e_token})
assert res.status_code == 400
# non existent user:
plain = {'expiry': expiry.isoformat(), 'email': '*****@*****.**'}
token = self.mail_token_service.issue(plain)
res = self.__test.post('/users/api/v1.0/verify',
data={'mailtoken': token})
assert res.status_code == 400
@mock.patch('smtplib.SMTP')
def test_resend_email(self, smtp_mock):
db = self.__service.test_db()
dbuser = db.tables.User.query.filter_by(
email='*****@*****.**').first()
dbuser.state = HRServiceUserState.REGISTERED # unverify !
db.session.add(dbuser)
db.session.commit()
email = '*****@*****.**'
data = {'email': email}
res = self.__test.post('/users/api/v1.0/resend', data=data)
assert res.status_code == 200
dbuser.state = HRServiceUserState.VERIFIED
db.session.add(dbuser)
db.session.commit()
res = self.__test.post('/users/api/v1.0/resend', data=data)
assert res.status_code == 400
res = self.__test.post('/users/api/v1.0/resend',
data={'email': 'hula@gula'})
assert res.status_code == 400
res = self.__test.post('/users/api/v1.0/resend',
data={'Email': 'hula@gula'})
assert res.status_code == 400
@mock.patch('email.MIMEMultipart.MIMEMultipart')
@mock.patch.object(smtplib.SMTP, 'connect')
@mock.patch.object(smtplib.SMTP, 'close')
def test_compose_and_send(self, close_mock, connect_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# force connect to raise the SMTPException derived class. HRService wraps it into
# RuntimeError
connect_mock.return_value = (400, 'cannot connect message'
) # 220 is the success code
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow()
) # timestamp does not matter here
connect_mock.assert_called_with('localhost', None) # from conf{}
# now allow for connect() to raise a socket.error
import socket
connect_mock.side_effect = socket.error
with self.assertRaises(RuntimeError):
HRService.compose_and_send("centos@localhost", 'mytoken_abc',
datetime.datetime.utcnow())
@mock.patch('email.MIMEMultipart.MIMEMultipart')
@mock.patch('smtplib.SMTP')
def test_compose_and_send_sendmail(self, smtp_mock, mail_mock):
with self.__service.test_request_context(path="/test"):
# sendmail errors
mytoken = 'mytoken_abc'
toaddr = "user@remotehost"
body = os.path.join(self._conf['verification_url'], mytoken)
smtp_mock.return_value.sendmail.side_effect = smtplib.SMTPException
with self.assertRaises(RuntimeError):
HRService.compose_and_send(toaddr, mytoken,
datetime.datetime.utcnow())
args = smtp_mock.return_value.sendmail.call_args
assert args[0][0] == self._conf['smtp_server_login']
assert args[0][1] == toaddr
assert body in args[0][2] # check the important part of the email
def test_hello(self):
res = self.__test.get('/users/api/v1.0/hello')
assert (res.status_code == 200)
res_str = json.loads(res.data)
assert (res_str == 'User Service Desk at your service !\n')
class test_Worker(unittest.TestCase):
def setUp(self):
self._service = FlaskServer("pdm.workqueue.WorkqueueService")
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self._service.test_mode(WorkqueueService, {}, with_test=False)
self._service.fake_auth("ALL")
self._test = self._service.test_client()
# Daemon won't setup logger for us so need one in place.
Worker._logger = logging.getLogger("Worker")
with mock.patch('pdm.workqueue.Worker.RESTClient.__init__'),\
mock.patch('pdm.workqueue.Worker.Daemon.__init__'):
self._patcher, self._inst = RESTClientTest.patch_client(
Worker, self._test, '/workqueue/api/v1.0')
# Setup test base class as we don't call super in Worker.__init__ so
# mock away the call to RESTClient.__init__, hence the test base class is
# never initialised. Actually don't need this as __init__ in RESTClientTest
# only sets two variables to none which are set properly above in the patch_client
# call which calls set_test_info so we actually dont want to set them None again.
# RESTClientTest.__init__(self._inst, 'workqueue')
# self._inst._logger = logging.getLogger("Worker")
self._inst._n_shot = 1
def tearDown(self):
self._patcher.stop()
def test_run(self):
workload = [{
'id':
1,
'user_id':
9,
'type':
JobType.LIST,
'status':
JobStatus.SUBMITTED,
'priority':
5,
'protocol':
JobProtocol.DUMMY,
'src_siteid':
12,
'src_filepath':
'/data/somefile',
'src_credentials':
'somesecret',
'dst_credentials':
'someothersecret',
'extra_opts': {},
'elements': [{
"id": 0,
"job_id": 1,
"type": JobType.LIST,
"src_filepath": "/some/file",
"token": 'secret_token'
}]
}]
getjobmock = mock.MagicMock()
outputmock = mock.MagicMock()
getjobmock.return_value = jsonify(workload)
outputmock.return_value = '', 200
with mock.patch.dict(self._service.view_functions, {'WorkqueueService.get_next_job': getjobmock,
'WorkqueueService.return_output': outputmock}),\
mock.patch.object(self._inst._site_client, 'get_endpoints') as mock_get_endpoints,\
mock.patch('pdm.workqueue.Worker.X509Utils.add_ca_to_dir') as mock_ca2dir:
mock_get_endpoints.return_value = {
'endpoints': ['blah1', 'blah2', 'blah3'],
'cas': ['blah blah', 'la la']
}
mock_ca2dir.return_value = '/tmp/somecadir'
self._inst.run()
self.assertTrue(getjobmock.called)
self.assertTrue(outputmock.called)
self.assertTrue(mock_get_endpoints.called)
self.assertEqual(mock_get_endpoints.call_count, 1)
self.assertTrue(mock_ca2dir.called)
class test_WorkqueueClient(unittest.TestCase):
def setUp(self):
self._service = FlaskServer("pdm.workqueue.WorkqueueService")
with mock.patch('pdm.workqueue.WorkqueueService.SiteClient'):
self._service.test_mode(WorkqueueService, {}, with_test=False)
self._service.fake_auth("ALL")
self._test = self._service.test_client()
self._patcher, self._inst = RESTClientTest.patch_client(
WorkqueueClient, self._test, '/workqueue/api/v1.0')
def tearDown(self):
self._patcher.stop()
def test_list(self):
args = {'siteid': 12, 'filepath': '/data/somefile'}
listmock = mock.MagicMock()
listmock.return_value = jsonify(args)
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.list': listmock}):
response = self._inst.list(**args)
self.assertTrue(listmock.called)
self.assertIsInstance(response, dict)
self.assertEqual(response, args)
def test_copy(self):
args = {
'src_siteid': 12,
'src_filepath': '/data/somefile',
'dst_siteid': 15,
'dst_filepath': '/data/someotherfile'
}
copymock = mock.MagicMock()
copymock.return_value = jsonify(args)
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.copy': copymock}):
response = self._inst.copy(**args)
self.assertTrue(copymock.called)
self.assertIsInstance(response, dict)
self.assertEqual(response, args)
def test_remove(self):
args = {'siteid': 12, 'filepath': '/data/somefile'}
removemock = mock.MagicMock()
removemock.return_value = jsonify(args)
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.remove': removemock}):
response = self._inst.remove(**args)
self.assertTrue(removemock.called)
self.assertIsInstance(response, dict)
self.assertEqual(response, args)
def test_jobs(self):
# check the token is passed
methodmock = mock.MagicMock()
methodmock.return_value = jsonify([])
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.get_jobs': methodmock}):
response = self._inst.jobs()
self.assertTrue(methodmock.called)
self.assertEqual(response, [])
def test_job(self):
# check the token is passed
methodmock = mock.MagicMock()
methodmock.return_value = jsonify({})
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.get_job': methodmock}):
response = self._inst.job(1)
self.assertTrue(methodmock.called)
self.assertEqual(response, {})
def test_status(self):
# check the token is passed
methodmock = mock.MagicMock()
methodmock.return_value = jsonify({})
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.get_job_status': methodmock}):
response = self._inst.status(1)
self.assertTrue(methodmock.called)
self.assertEqual(response, {})
methodmock.reset_mock()
with mock.patch.dict(
self._service.view_functions,
{'WorkqueueService.get_element_status': methodmock}):
response = self._inst.status(1, 0)
self.assertTrue(methodmock.called)
self.assertEqual(response, {})
def test_output(self):
# check the token is passed
methodmock = mock.MagicMock()
methodmock.return_value = jsonify({})
with mock.patch.dict(self._service.view_functions,
{'WorkqueueService.get_output': methodmock}):
response = self._inst.output(1)
self.assertTrue(methodmock.called)
self.assertEqual(response, {})
class TestDemoService(unittest.TestCase):
def setUp(self):
conf = { 'test_param': 1111 }
self.__service = FlaskServer("pdm.demo.DemoService")
self.__service.test_mode(DemoService, conf, with_test=True)
self.__service.fake_auth("ALL")
self.__test = self.__service.test_client()
def test_init_existingDB(self):
""" Tests that the config step completes correctly if a DB already
exists.
"""
# Create a clean copy of the service
service = FlaskServer("pdm.demo.DemoService")
service.test_mode(DemoService, None)
service.build_db()
# Add a single turtle to the table
db = service.test_db()
new_turtle = db.tables.Turtle(name='Before')
db.session.add(new_turtle)
db.session.commit()
# Continue service start-up
service.before_startup({}, with_test=True)
service.fake_auth("ALL")
client = service.test_client()
# Now check that we only have one turtle
# Rather than the 3 we get by default
res = client.get('/demo/api/v1.0/turtles')
assert(res.status_code == 200)
assert(len(json.loads(res.data)) == 1)
def test_web(self):
""" Check that the basic website redirect + index page work. """
res = self.__test.get('/')
assert(res.status_code == 302)
assert('/web/turtles' in res.location)
self.__service.testing = True
res = self.__test.get('/web/turtles')
assert(res.status_code == 200)
assert("<html>" in res.data)
def test_hello(self):
res = self.__test.get('/demo/api/v1.0/hello')
assert(res.status_code == 200)
res_str = json.loads(res.data)
assert(res_str == "Hello World!\n")
def test_defTurtles(self):
# We should have 3 turtles by default
res = self.__test.get('/demo/api/v1.0/turtles')
assert(res.status_code == 200)
turtles = json.loads(res.data)
assert(len(turtles) == 3)
def test_addGetDelTurtle(self):
# Try adding a turtle and then get its info
new_turtle = {'name': 'New Turtle'}
res = self.__test.post('/demo/api/v1.0/turtles', data=new_turtle)
assert(res.status_code == 200)
turtle_id = json.loads(res.data)['id']
print "Turtle ID: %s" % turtle_id
# Get info
res = self.__test.get('/demo/api/v1.0/turtles/%u' % turtle_id)
assert(res.status_code == 200)
turtle = json.loads(res.data)
assert(turtle['id'] == turtle_id)
assert(turtle['name'] == 'New Turtle')
# modify the turtle
mod_turtle = {'name': 'New Lovely Turtle'}
res = self.__test.put('/demo/api/v1.0/turtles/%u' % turtle_id,
data=mod_turtle)
assert(res.status_code == 200)
turtle = json.loads(res.data)
assert(turtle['id'] == turtle_id)
assert(turtle['name'] == 'New Lovely Turtle')
# Delete Turtle
res = self.__test.delete('/demo/api/v1.0/turtles/%u' % turtle_id)
assert(res.status_code == 200)
# Check Turtle is gone
res = self.__test.get('/demo/api/v1.0/turtles/%u' % turtle_id)
assert(res.status_code == 404)
def test_delTimmy(self):
# Timmy the Turtle is protected and can't be deleted
# Test that this works
res = self.__test.get('/demo/api/v1.0/turtles')
assert(res.status_code == 200)
turtles = json.loads(res.data)
timmy_id = None
for turtle_id, turtle_name in turtles.iteritems():
if turtle_name == 'Timmy':
timmy_id = int(turtle_id)
break
print "Timmy ID: %s" % timmy_id
assert(timmy_id is not None)
# Found Timmy, now try delete
res = self.__test.delete('/demo/api/v1.0/turtles/%u' % timmy_id)
assert(res.status_code == 401)
def test_getToken(self):
res = self.__test.get('/demo/api/v1.0/get_token')
assert(res.status_code == 200)
assert(len(res.data) > 10)
assert("." in res.data)
# Actually check token content
token_data = self.__service.token_svc.check(json.loads(res.data))
assert(token_data == "Hello")
def test_verifyTokenGood(self):
self.__service.fake_auth("TOKEN", "TTest")
res = self.__test.get('/demo/api/v1.0/verify_token')
assert(res.status_code == 200)
res_str = json.loads(res.data)
assert(res_str == 'Token OK! (TTest)')
def test_verifyTokenBad(self):
res = self.__test.get('/demo/api/v1.0/verify_token')
assert(res.status_code == 200)
res_str = json.loads(res.data)
assert(res_str == 'Token Missing!')
class test_SiteService(unittest.TestCase):
""" Test the SiteService service. """
TEST_SITE = {
'site_name': 'TestSite',
'site_desc': 'A test site.',
'user_ca_cert': 'ABC',
'service_ca_cert': '123',
'auth_type': 0,
'auth_uri': 'localhost:12345',
'public': False,
'def_path': '/root',
'endpoints': ['localhost:12346', 'an.other.host:54321'],
}
@staticmethod
def __db_error(mock_session):
""" Replace a managed_session mock instance
with one that throws an error (simulating
a generic DB failure).
"""
from flask import abort
def run_session(request, message="Error",
logger=None, http_error_code=None):
if http_error_code:
abort(http_error_code, description=message)
raise Exception("DB Error")
mock_session.side_effect = run_session
def set_user_token(self, user_id):
token = {'id': user_id}
self.__service.fake_auth("TOKEN", token)
def setUp(self):
""" Configure the basic service in test mode.
"""
logging.basicConfig(level=logging.DEBUG)
self.__service = FlaskServer("pdm.site.SiteService")
self.__service.test_mode(SiteService, )
self.set_user_token(1000)
self.__client = self.__service.test_client()
# This calls the startup_test functions a second time
# To check it does nothing when the DB already contains entries
self.__service.before_startup({}, with_test=True)
def test_check_uri(self):
""" Test the check_uri function correctly rejects URIs.
"""
# OK
self.assertTrue(SiteService.check_uri("localhost:12345"))
self.assertTrue(SiteService.check_uri("www.google.com:12345"))
self.assertTrue(SiteService.check_uri("127.0.0.1:12345"))
# Missing Port
self.assertFalse(SiteService.check_uri("localhost:"))
# Missing seperator
self.assertFalse(SiteService.check_uri("localhost"))
self.assertFalse(SiteService.check_uri("localhost12345"))
self.assertFalse(SiteService.check_uri("localhost@12345"))
# Starts with invalid char
self.assertFalse(SiteService.check_uri("_localhost:12345"))
self.assertFalse(SiteService.check_uri(".localhost:12345"))
# Non-numeric port
self.assertFalse(SiteService.check_uri("localhost:bah"))
@mock.patch("pdm.site.SiteService.open", create=True)
@mock.patch("pdm.site.SiteService.getConfig")
def test_service_info(self, mock_conf, mock_open):
""" Test the service info endpoint.
"""
# Set-up mock conifg
mock_conf.return_value = {'cafile': 'mytestfile',
'users': 'https://*****:*****@mock.patch("pdm.site.SiteService.managed_session")
def test_add_site_dberror(self, mock_session):
""" Test a general DB is caught correctly in add_site.
"""
self.__db_error(mock_session)
res = self.__client.post('/site/api/v1.0/site', data=self.TEST_SITE)
self.assertEqual(res.status_code, 500)
def test_get_endpoints(self):
""" Test the get endpoint function.
"""
self.__service.fake_auth("CERT", "/CN=Any")
res = self.__client.get('/site/api/v1.0/endpoint/1')
self.assertEqual(res.status_code, 200)
res = json.loads(res.data)
self.assertIsInstance(res, dict)
# Check we got a list of two endpoints
self.assertIn('endpoints', res)
self.assertIsInstance(res['endpoints'], list)
self.assertEqual(len(res['endpoints']), 2)
self.assertIn('cas', res)
self.assertIsInstance(res['cas'], list)
def test_del_user(self):
""" Test deleting all sites belonging to a given user.
"""
# First check we get a 404 if we use the wrong user_id
res = self.__client.delete('/site/api/v1.0/user/1001')
self.assertEqual(res.status_code, 404)
# Now add some sites to delete
test_data = copy.deepcopy(self.TEST_SITE)
res = self.__client.post('/site/api/v1.0/site', data=test_data)
self.assertEqual(res.status_code, 200)
test_data['site_name'] = 'YetAnotherTestSite'
res = self.__client.post('/site/api/v1.0/site', data=test_data)
self.assertEqual(res.status_code, 200)
# Check this user now has two sites
res = self.__client.get('/site/api/v1.0/site')
self.assertEqual(res.status_code, 200)
my_sites = [x for x in json.loads(res.data) if x["is_owner"]]
self.assertEqual(len(my_sites), 2)
# We also need to add a credentials to test
# Add this to the DB directly
db = self.__service.test_db()
Cred = db.tables.Cred
db.session.add(Cred(cred_owner=1000,
site_id=1,
cred_username='******',
cred_expiry=datetime.datetime.utcnow(),
cred_value='secret'))
db.session.commit()
# Call the delete function
res = self.__client.delete('/site/api/v1.0/user/1000')
self.assertEqual(res.status_code, 200)
# Now check user has 0 sites
res = self.__client.get('/site/api/v1.0/site')
self.assertEqual(res.status_code, 200)
my_sites = [x for x in json.loads(res.data) if x["is_owner"]]
self.assertEqual(len(my_sites), 0)
# Check the cred has gone too
cred = Cred.query.filter_by(cred_owner=1000).first()
self.assertIsNone(cred)
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_vomsdir(self, mp_utils):
""" Test that the VO list is correctly loaded from the vomsdir
if one is specified in the config.
"""
TEST_VOS = ["vo1", "vo2.test.vo"]
mp_utils.load_voms_list.return_value = TEST_VOS
# Load in the conf with VOMS
self.__service.before_startup({'vomses': '/myvoms'}, True)
mp_utils.load_voms_list.assert_called_once_with('/myvoms')
# Check that the service endpoint returns the correct list
res = self.__client.get('/site/api/v1.0/service')
self.assertEqual(res.status_code, 200)
service_info = json.loads(res.data)
self.assertIn('vos', service_info)
self.assertItemsEqual(TEST_VOS, service_info['vos'])
def test_session_basics(self):
""" Manually add a credential to the DB and check that all of the
access functions (info, get_cred, delete) behave correctly.
"""
# Create a test site
res = self.__client.post('/site/api/v1.0/site', data=self.TEST_SITE)
self.assertEqual(res.status_code, 200)
site_id = json.loads(res.data)
# Manually register a cred in the DB
db = self.__service.test_db()
Cred = db.tables.Cred
future_time = datetime.datetime.utcnow()
future_time += datetime.timedelta(minutes=5)
db.session.add(Cred(cred_owner=1000,
site_id=site_id,
cred_username='******',
cred_expiry=future_time,
cred_value='secretcred'))
db.session.commit()
# Now check the details
res = self.__client.get('/site/api/v1.0/session/%u' % site_id)
self.assertEqual(res.status_code, 200)
cred_info = json.loads(res.data)
self.assertTrue(cred_info['ok'])
self.assertEqual(cred_info['username'], 'myuser')
# Try getting the cred secret
res = self.__client.get('site/api/v1.0/cred/%u/1000' % site_id)
self.assertEqual(res.status_code, 200)
cred_secret = json.loads(res.data)
self.assertEqual(cred_secret, 'secretcred')
# Now test deletion
res = self.__client.delete('/site/api/v1.0/session/%u' % site_id)
self.assertEqual(res.status_code, 200)
cred_count = Cred.query.filter_by(cred_owner=1000, site_id=site_id).count()
self.assertEqual(cred_count, 0)
@mock.patch("pdm.site.SiteService.X509Utils")
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_logon(self, mp_mock, x509_mock):
""" Check the basic functionality of the logon function. """
mp_mock.logon.return_value = "PROXY"
x509_mock.get_cert_expiry.return_value = datetime.datetime.utcnow()
AUTH_DATA = {'username': "******",
'password': "******",
'lifetime': 36}
# Basic test again public site 2.
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 200)
# Check that the cred was put in the DB
res = self.__client.get('/site/api/v1.0/cred/2/1000')
self.assertEqual(res.status_code, 200)
self.assertEqual(json.loads(res.data), "PROXY")
# Check the myproxy logon parameters
self.assertEqual(mp_mock.logon.call_args[0][0], 'localhost:49998')
self.assertEqual(mp_mock.logon.call_args[0][1], 'testuser')
self.assertEqual(mp_mock.logon.call_args[0][2], 'usersecret')
self.assertIsNone(mp_mock.logon.call_args[0][4])
self.assertEqual(mp_mock.logon.call_args[0][5], 36)
@mock.patch("pdm.site.SiteService.X509Utils")
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_double_logon(self, mp_mock, x509_mock):
""" Check that running logon twice with same params
correctly overwrites the old proxy a second time.
"""
mp_mock.logon.return_value = "PROXY"
# We have to generate a time-zone aware return for the get_cert_expiry
# This is like the real return from X509Utils and was originally triggering
# a comparison problem in the DB driver. So this also serves as the
# regression test for that.
class UTCTZ(datetime.tzinfo):
def utcoffset(self, dt):
return datetime.timedelta(0)
def dst(self, dt):
return datetime.timedelta(0)
def tzname(self, dt):
return "UTC"
tz_info = UTCTZ()
x509_mock.get_cert_expiry.return_value = datetime.datetime.now(tz_info)
# Now store the proxy...
AUTH_DATA = {'username': "******",
'password': "******",
'lifetime': 36}
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 200)
# Try it a second time, different proxy
mp_mock.logon.return_value = "PROXY2"
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 200)
# Check that the cred was overwritten
res = self.__client.get('/site/api/v1.0/cred/2/1000')
self.assertEqual(res.status_code, 200)
self.assertEqual(json.loads(res.data), "PROXY2")
@mock.patch("pdm.site.SiteService.X509Utils")
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_logon_grid(self, mp_mock, x509_mock):
""" Check the logon function for obtaining grid creds. """
TEST_VOS = ["vo1", "vo2.test.vo"]
mp_mock.load_voms_list.return_value = TEST_VOS
mp_mock.logon.return_value = "PROXY"
x509_mock.get_cert_expiry.return_value = datetime.datetime.utcnow()
CA_DIR = "/etc/grid-security/certificates"
CONF_DATA = {
'vomses': '/myvoms',
'cadir': CA_DIR,
}
self.__service.before_startup(CONF_DATA, True)
SITE_ID = 5 # ID of grid site in test data
AUTH_DATA = {'username': "******",
'password': "******",
'vo': "vo2.test.vo",
'lifetime': 36}
# Check basic auth with voms
res = self.__client.post('/site/api/v1.0/session/%u' % SITE_ID,
data=copy.deepcopy(AUTH_DATA))
self.assertEqual(res.status_code, 200)
# Check call params
self.assertEqual(mp_mock.logon.call_args[0][3], CA_DIR)
self.assertEqual(mp_mock.logon.call_args[0][4], "vo2.test.vo")
# Same again with missing VO name
del AUTH_DATA['vo']
res = self.__client.post('/site/api/v1.0/session/%u' % SITE_ID,
data=copy.deepcopy(AUTH_DATA))
self.assertEqual(res.status_code, 400)
# Same again with bad VO name
AUTH_DATA['vo'] = "bad.vo"
res = self.__client.post('/site/api/v1.0/session/%u' % SITE_ID,
data=copy.deepcopy(AUTH_DATA))
self.assertEqual(res.status_code, 400)
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_logon_errors(self, mp_mock):
""" Check the error handling of the logon function. """
# Missing post data
res = self.__client.post('/site/api/v1.0/session/2')
self.assertEqual(res.status_code, 400)
# Missing field
AUTH_DATA = {'username': '******'}
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 400)
# User not allowed at site
AUTH_DATA = {'username': "******",
'password': "******",
'lifetime': 36}
res = self.__client.post('/site/api/v1.0/session/1', data=AUTH_DATA)
self.assertEqual(res.status_code, 404)
# MyProxy error
MSG = "MP failed for some reason"
mp_mock.logon.side_effect = Exception(MSG)
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 400)
self.assertIn(MSG, res.data)
@mock.patch("pdm.site.SiteService.managed_session")
@mock.patch("pdm.site.SiteService.X509Utils")
@mock.patch("pdm.site.SiteService.MyProxyUtils")
def test_logon_dberror(self, mp_mock, x509_mock, mock_session):
""" Checks that a general DB error is caught correctly. """
mp_mock.logon.return_value = "PROXY"
x509_mock.get_cert_expiry.return_value = datetime.datetime.utcnow()
self.__db_error(mock_session)
AUTH_DATA = {'username': "******",
'password': "******",
'lifetime': 36}
res = self.__client.post('/site/api/v1.0/session/2', data=AUTH_DATA)
self.assertEqual(res.status_code, 500)