def log(self, args):
"""
Get job log of a complete finished job (get *log* element of the job output).
Use -v to get complete job info.
:param args: parser arguments, in particualr the job *id*.
:return: None
see: :func:`pdm.userservicedesk.TransferClient.TransferClient.output`
"""
token = UserCommand._get_token(args.token)
if token:
job_id = int(args.job)
client = TransferClientFacade(token)
status = self._status(job_id, client, block=True)
try:
for element in client.output(job_id,
element_id=args.element,
attempt=args.attempt):
for attempt in element:
log_listing = attempt.get('log')
if args.verbosity == logging.DEBUG:
pprint(attempt)
else:
print log_listing
except RESTException as rexc:
print str(rexc)
def copy(self, args): # pylint: disable=no-self-use
"""
Copy files between sites. Not executed if no valid token or not logged in
to either of the sites.
:param args: parser argumens, in parcicular source and destination site paths.
:return: copy response or *None* if site paths were malformed.
see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.copy`
"""
token = UserCommand._get_token(args.token)
if token and self._session_ok(args.src_site, token) \
and self._session_ok(args.dst_site, token):
client = TransferClientFacade(token)
src_site = args.src_site
dst_site = args.dst_site
# remove None values, position args, func and token from the kwargs:
accepted_args = {
key: value
for (key, value) in vars(args).iteritems()
if value is not None and key not in ('func', 'src_site',
'dst_site', 'token',
'block', 'config',
'verbosity')
}
response = client.copy(src_site, dst_site, **accepted_args)
if response:
self._status(response['id'], client, block=args.block)
def jobs(self, args):
"""
Get user jobs' info. See \
:func:`pdm.userservicedesk.TransferClient.TransferClient.jobs`
:param args: parser arguments.
:return: None
"""
token = UserCommand._get_token(args.token)
if token:
client = TransferClientFacade(token)
jobs = client.jobs()
UserCommand._print_formatted_jobs_info(
jobs, token, long_listing=args.long_listing)
def test_split_site_path(self):
site = "localhost:/root/file.txt"
malformed_site = "localhost/root/file.txt" # mind a missing colon
multicolon_site = "localhost:/root/file.txt:1"
a, b = TransferClientFacade.split_site_path(site)
assert a == 'localhost'
assert b == '/root/file.txt'
c, d = TransferClientFacade.split_site_path(malformed_site)
assert d is None
assert c is None
e, f = TransferClientFacade.split_site_path(multicolon_site)
assert e == 'localhost'
assert f == '/root/file.txt:1'
def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock):
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
site_mock().get_sites.return_value = \
[{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'},
{'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}]
site_mock.return_value.set_token = mock.MagicMock()
self.site_id = site_mock().get_sites()[0]['site_id']
self.site2_id = site_mock().get_sites()[1]['site_id']
conf = {
'CS_secret': 'HJGnbfdsV',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
# HR
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
token = {'id': 1, 'expiry': self.__future_date}
self.__service.fake_auth("TOKEN", token)
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
self.__service.before_startup(conf) # to continue startup
# mock_ct.return_value =1
mocked_unpack.return_value = token
self.__htoken = 'whateverhash'
self.__client = TransferClientFacade(self.__htoken)
assert wq_mock.called
mocked_unpack.assert_called_with(self.__htoken)
def sitelist(self, args): # pylint disable-no-self-use
"""
Print list of available sites.
:param args: carry a user token
:return: None
"""
token = UserCommand._get_token(args.token)
if token:
client = TransferClientFacade(token)
sites = client.list_sites()
print '-' + 91 * '-' + '-'
print '|{0:40}|{1:50}|'.format('site:', 'description:')
print '|' + 91 * '-' + '|'
for elem in sites:
print '|{site_name:40s}|{site_desc:50s}|'.format(**elem)
print '-' + 91 * '-' + '-'
def mkdir(self, args):
"""
Create a new directory at a site.
:param args: site - the new directory in a form site:path
:return: None
see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.mkdir`
"""
token = UserCommand._get_token(args.token)
if token and self._session_ok(args.site, token):
client = TransferClientFacade(token)
accepted_args = {
key: value
for (key, value) in vars(args).iteritems()
if value is not None and key not in ('func', 'site', 'token',
'block', 'config',
'verbosity')
}
response = client.mkdir(args.site,
**accepted_args) # max_tries, priority
if response:
self._status(response['id'], client, block=args.block)
def rename(self, args):
"""
Rename a file at a site. It's like a copy, but with the source and destination
site being the same.
:param args: old name: site:path, new name: :path.
:return: None
see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.rename`
"""
token = UserCommand._get_token(args.token)
if token and self._session_ok(args.oldname, token):
client = TransferClientFacade(token)
accepted_args = {
key: value
for (key, value) in vars(args).iteritems()
if value is not None and key not in ('func', 'token', 'block',
'config', 'verbosity',
'oldname', 'newname')
}
response = client.rename(args.oldname, args.newname,
**accepted_args) # max_tries, priority
if response:
self._status(response['id'], client, block=args.block)
def remove(self, args): # pylint: disable=no-self-use
"""
Remove files at remote site.
:param args: parser arguments, in particular *sitename:patname* to remove.
:return: None
see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.remove`
"""
token = UserCommand._get_token(args.token)
if token and self._session_ok(args.site, token):
client = TransferClientFacade(token)
# remove None values, position args, func and token from the kwargs:
accepted_args = {
key: value
for (key, value) in vars(args).iteritems()
if value is not None and key not in ('func', 'site', 'token',
'block', 'config',
'verbosity')
}
response = client.remove(args.site,
**accepted_args) # max_tries, priority)
if response:
self._status(response['id'], client, block=args.block)
def list(self, args): # pylint: disable=no-self-use
"""
List files at remote site.
:param args: parser arguments.
:return: None
"""
nap = 0.2
count = 1
max_iter = max(1, int(args.wait / nap))
#
token = UserCommand._get_token(args.token)
if token and self._session_ok(args.site, token):
client = TransferClientFacade(token)
# remove None values, position args, func and token from the kwargs:
accepted_args = {
key: value
for (key, value) in vars(args).iteritems()
if value is not None and key not in ('func', 'site', 'token',
'config', 'verbosity',
'wait')
}
resp = client.list(args.site,
**accepted_args) # max_tries, priority, depth)
# resp and status both carry job id:
if resp:
status = client.status(resp['id'])
while status['status'] not in ('DONE', 'FAILED'):
sleep(nap) # seconds
status = client.status(resp['id'])
count += 1
if count >= max_iter:
break
if status['status'] == 'DONE':
listing_output = client.output(
resp['id'], 0, -1)[0][0] # listing is 0, last attempt
listing_d_value = listing_output['listing']
root, listing = listing_d_value.items()[0] # top root
self._print_formatted_listing(root, listing_d_value)
elif resp['status'] == 'FAILED':
print " Failed to obtain a listing for job %d " % (
resp['id'], )
else:
print "Timeout. Last status is %s for job id %d" % \
(status['status'], resp['id'])
elif isinstance(resp, list) and not resp:
print "No such site: %s " % (args.site, )
def _session_ok(self, site_path, token):
"""
Check user session at a site.
:param site_name: site to check
:param token: user token
:return:True or False
"""
name, path = TransferClientFacade.split_site_path(site_path)
if name is None:
print "Malformed site path (should be sitename:path)"
return None
site_client, site_id = UserCommand._get_site_id(name, token)
ok = False
if site_id:
ok = site_client.get_session_info(site_id)['ok']
if not ok:
print "Please log to the site %s first" % (name)
else:
print "site %s not found !" % (name)
return ok
def status(self, args):
"""
Get and print status of a job (task).
:param args:
:return: None
see: :func:`pdm.userservicedesk.TransferClient.TransferClient.status`
"""
token = UserCommand._get_token(args.token)
block = args.block
job_id = args.job
element_id = None
if '.' in job_id:
job_id, element_id = args.job.split('.')
job_id = int(job_id)
if element_id is not None:
element_id = int(element_id)
if token:
client = TransferClientFacade(token)
self._status(job_id, client, element_id, block=block)
class TestTransferClient(unittest.TestCase):
# @mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient")
# @mock.patch("pdm.userservicedesk.TransferClient.WorkqueueClient")
##@mock.patch.object(HRService, 'check_token')
@mock.patch("pdm.userservicedesk.HRService.SiteClient")
@mock.patch.object(Tokens.TokenService, 'unpack')
@mock.patch("pdm.userservicedesk.TransferClient.SiteClient")
@mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient.__new__")
def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock):
self.__future_date = (datetime.timedelta(0, 600) +
datetime.datetime.utcnow()).isoformat()
site_mock().get_sites.return_value = \
[{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'},
{'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}]
site_mock.return_value.set_token = mock.MagicMock()
self.site_id = site_mock().get_sites()[0]['site_id']
self.site2_id = site_mock().get_sites()[1]['site_id']
conf = {
'CS_secret': 'HJGnbfdsV',
'smtp_server': 'localhost',
'verification_url':
'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify',
'smtp_server_login': '******',
'smtp_starttls': 'OPTIONAL',
'smtp_login_req': 'OPTIONAL',
'display_from_address': 'PDM mailer <centos@localhost>',
'mail_subject':
'PDM registration - please verify your email address.',
'mail_expiry': '12:00:00',
'mail_token_secret': 'somemailsecretstring'
}
# HR
self.__service = FlaskServer("pdm.userservicedesk.HRService")
self.__service.test_mode(HRService, None) # to skip DB auto build
token = {'id': 1, 'expiry': self.__future_date}
self.__service.fake_auth("TOKEN", token)
# database
self.__service.build_db() # build manually
#
db = self.__service.test_db()
self.__service.before_startup(conf) # to continue startup
# mock_ct.return_value =1
mocked_unpack.return_value = token
self.__htoken = 'whateverhash'
self.__client = TransferClientFacade(self.__htoken)
assert wq_mock.called
mocked_unpack.assert_called_with(self.__htoken)
def test_list(self):
site = "localhost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt'
assert self.__client.list(site, **{'priority':
2}) == 'root/file.txt'
assert mock_list.called
mock_list.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_list.call_args_list
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'list') as mock_list:
mock_list.return_value = 'root/file.txt' # event if ...
assert self.__client.list(wrongurl, **{'priority':
2}) == [] # we return []
assert not mock_list.called
def test_sitelist(self):
sites = self.__client.list_sites()
print sites
assert sites[0]['site_name'] == 'localhost'
assert sites[1]['site_name'] == 'remotehost'
assert 'site_id' not in [dd.keys() for dd in sites]
def test_remove(self):
site = "localhost:/root/file.txt"
# mock_remove.return_value = 'root/file.txt'
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'root/file.txt removed'
assert self.__client.remove(site, **{'priority':
2}) == 'root/file.txt removed'
assert mock_remove.called
mock_remove.assert_called_with(self.site_id,
'/root/file.txt',
priority=2)
print mock_remove.call_args_remove
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'remove') as mock_remove:
mock_remove.return_value = 'whatever..' # event if ...
assert self.__client.remove(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_remove.called
def test_copy(self):
s_site = "localhost:/root/file.txt"
t_site = "remotehost:/root/file.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'root/file.txt copied'
assert self.__client.copy(s_site, t_site,
**{'priority':
2}) == 'root/file.txt copied'
assert mock_copy.called
mock_copy.assert_called_with(self.site_id,
'/root/file.txt',
self.site2_id,
'/root/file.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'copy') as mock_copy:
mock_copy.return_value = 'whatever..' # even if ...
assert self.__client.copy(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_copy.called
def test_split_site_path(self):
site = "localhost:/root/file.txt"
malformed_site = "localhost/root/file.txt" # mind a missing colon
multicolon_site = "localhost:/root/file.txt:1"
a, b = TransferClientFacade.split_site_path(site)
assert a == 'localhost'
assert b == '/root/file.txt'
c, d = TransferClientFacade.split_site_path(malformed_site)
assert d is None
assert c is None
e, f = TransferClientFacade.split_site_path(multicolon_site)
assert e == 'localhost'
assert f == '/root/file.txt:1'
def test_mkdir(self):
site = "localhost:/root/subdir"
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'root/subdir created'
assert self.__client.mkdir(site, **{'priority':
2}) == 'root/subdir created'
assert mock_mkdir.called
mock_mkdir.assert_called_with(self.site_id, '/root/subdir', priority=2)
# now unknown site:
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'mkdir') as mock_mkdir:
mock_mkdir.return_value = 'whatever..' # event if ...
assert self.__client.mkdir(
wrongurl, **{'priority': 2}) == None # we return None
assert not mock_mkdir.called
def test_rename(self):
s_site = "localhost:/root/file.txt"
t_site = ":/root/file2.txt"
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'root/file.txt renamed'
assert self.__client.rename(s_site, t_site,
**{'priority':
2}) == 'root/file.txt renamed'
assert mock_rename.called
mock_rename.assert_called_with(self.site_id,
'/root/file.txt',
'/root/file2.txt',
priority=2)
wrongurl = "localhost2:/root/file.txt" # no such site,
with mock.patch.object(self.__client._TransferClient__wq_client,
'rename') as mock_rename:
mock_rename.return_value = 'whatever..' # even if ...
assert self.__client.rename(
wrongurl, t_site, **{'priority': 2}) == None # we return None
assert not mock_rename.called
def tearDown(self):
pass