def log(self, args): """ Get job log of a complete finished job (get *log* element of the job output). Use -v to get complete job info. :param args: parser arguments, in particualr the job *id*. :return: None see: :func:`pdm.userservicedesk.TransferClient.TransferClient.output` """ token = UserCommand._get_token(args.token) if token: job_id = int(args.job) client = TransferClientFacade(token) status = self._status(job_id, client, block=True) try: for element in client.output(job_id, element_id=args.element, attempt=args.attempt): for attempt in element: log_listing = attempt.get('log') if args.verbosity == logging.DEBUG: pprint(attempt) else: print log_listing except RESTException as rexc: print str(rexc)
def copy(self, args): # pylint: disable=no-self-use """ Copy files between sites. Not executed if no valid token or not logged in to either of the sites. :param args: parser argumens, in parcicular source and destination site paths. :return: copy response or *None* if site paths were malformed. see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.copy` """ token = UserCommand._get_token(args.token) if token and self._session_ok(args.src_site, token) \ and self._session_ok(args.dst_site, token): client = TransferClientFacade(token) src_site = args.src_site dst_site = args.dst_site # remove None values, position args, func and token from the kwargs: accepted_args = { key: value for (key, value) in vars(args).iteritems() if value is not None and key not in ('func', 'src_site', 'dst_site', 'token', 'block', 'config', 'verbosity') } response = client.copy(src_site, dst_site, **accepted_args) if response: self._status(response['id'], client, block=args.block)
def jobs(self, args): """ Get user jobs' info. See \ :func:`pdm.userservicedesk.TransferClient.TransferClient.jobs` :param args: parser arguments. :return: None """ token = UserCommand._get_token(args.token) if token: client = TransferClientFacade(token) jobs = client.jobs() UserCommand._print_formatted_jobs_info( jobs, token, long_listing=args.long_listing)
def test_split_site_path(self): site = "localhost:/root/file.txt" malformed_site = "localhost/root/file.txt" # mind a missing colon multicolon_site = "localhost:/root/file.txt:1" a, b = TransferClientFacade.split_site_path(site) assert a == 'localhost' assert b == '/root/file.txt' c, d = TransferClientFacade.split_site_path(malformed_site) assert d is None assert c is None e, f = TransferClientFacade.split_site_path(multicolon_site) assert e == 'localhost' assert f == '/root/file.txt:1'
def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock): self.__future_date = (datetime.timedelta(0, 600) + datetime.datetime.utcnow()).isoformat() site_mock().get_sites.return_value = \ [{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'}, {'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}] site_mock.return_value.set_token = mock.MagicMock() self.site_id = site_mock().get_sites()[0]['site_id'] self.site2_id = site_mock().get_sites()[1]['site_id'] conf = { 'CS_secret': 'HJGnbfdsV', 'smtp_server': 'localhost', 'verification_url': 'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify', 'smtp_server_login': '******', 'smtp_starttls': 'OPTIONAL', 'smtp_login_req': 'OPTIONAL', 'display_from_address': 'PDM mailer <centos@localhost>', 'mail_subject': 'PDM registration - please verify your email address.', 'mail_expiry': '12:00:00', 'mail_token_secret': 'somemailsecretstring' } # HR self.__service = FlaskServer("pdm.userservicedesk.HRService") self.__service.test_mode(HRService, None) # to skip DB auto build token = {'id': 1, 'expiry': self.__future_date} self.__service.fake_auth("TOKEN", token) # database self.__service.build_db() # build manually # db = self.__service.test_db() self.__service.before_startup(conf) # to continue startup # mock_ct.return_value =1 mocked_unpack.return_value = token self.__htoken = 'whateverhash' self.__client = TransferClientFacade(self.__htoken) assert wq_mock.called mocked_unpack.assert_called_with(self.__htoken)
def sitelist(self, args): # pylint disable-no-self-use """ Print list of available sites. :param args: carry a user token :return: None """ token = UserCommand._get_token(args.token) if token: client = TransferClientFacade(token) sites = client.list_sites() print '-' + 91 * '-' + '-' print '|{0:40}|{1:50}|'.format('site:', 'description:') print '|' + 91 * '-' + '|' for elem in sites: print '|{site_name:40s}|{site_desc:50s}|'.format(**elem) print '-' + 91 * '-' + '-'
def mkdir(self, args): """ Create a new directory at a site. :param args: site - the new directory in a form site:path :return: None see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.mkdir` """ token = UserCommand._get_token(args.token) if token and self._session_ok(args.site, token): client = TransferClientFacade(token) accepted_args = { key: value for (key, value) in vars(args).iteritems() if value is not None and key not in ('func', 'site', 'token', 'block', 'config', 'verbosity') } response = client.mkdir(args.site, **accepted_args) # max_tries, priority if response: self._status(response['id'], client, block=args.block)
def rename(self, args): """ Rename a file at a site. It's like a copy, but with the source and destination site being the same. :param args: old name: site:path, new name: :path. :return: None see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.rename` """ token = UserCommand._get_token(args.token) if token and self._session_ok(args.oldname, token): client = TransferClientFacade(token) accepted_args = { key: value for (key, value) in vars(args).iteritems() if value is not None and key not in ('func', 'token', 'block', 'config', 'verbosity', 'oldname', 'newname') } response = client.rename(args.oldname, args.newname, **accepted_args) # max_tries, priority if response: self._status(response['id'], client, block=args.block)
def remove(self, args): # pylint: disable=no-self-use """ Remove files at remote site. :param args: parser arguments, in particular *sitename:patname* to remove. :return: None see: :func:`pdm.userservicedesk.TransferClientFacade.TransferClientFacade.remove` """ token = UserCommand._get_token(args.token) if token and self._session_ok(args.site, token): client = TransferClientFacade(token) # remove None values, position args, func and token from the kwargs: accepted_args = { key: value for (key, value) in vars(args).iteritems() if value is not None and key not in ('func', 'site', 'token', 'block', 'config', 'verbosity') } response = client.remove(args.site, **accepted_args) # max_tries, priority) if response: self._status(response['id'], client, block=args.block)
def list(self, args): # pylint: disable=no-self-use """ List files at remote site. :param args: parser arguments. :return: None """ nap = 0.2 count = 1 max_iter = max(1, int(args.wait / nap)) # token = UserCommand._get_token(args.token) if token and self._session_ok(args.site, token): client = TransferClientFacade(token) # remove None values, position args, func and token from the kwargs: accepted_args = { key: value for (key, value) in vars(args).iteritems() if value is not None and key not in ('func', 'site', 'token', 'config', 'verbosity', 'wait') } resp = client.list(args.site, **accepted_args) # max_tries, priority, depth) # resp and status both carry job id: if resp: status = client.status(resp['id']) while status['status'] not in ('DONE', 'FAILED'): sleep(nap) # seconds status = client.status(resp['id']) count += 1 if count >= max_iter: break if status['status'] == 'DONE': listing_output = client.output( resp['id'], 0, -1)[0][0] # listing is 0, last attempt listing_d_value = listing_output['listing'] root, listing = listing_d_value.items()[0] # top root self._print_formatted_listing(root, listing_d_value) elif resp['status'] == 'FAILED': print " Failed to obtain a listing for job %d " % ( resp['id'], ) else: print "Timeout. Last status is %s for job id %d" % \ (status['status'], resp['id']) elif isinstance(resp, list) and not resp: print "No such site: %s " % (args.site, )
def _session_ok(self, site_path, token): """ Check user session at a site. :param site_name: site to check :param token: user token :return:True or False """ name, path = TransferClientFacade.split_site_path(site_path) if name is None: print "Malformed site path (should be sitename:path)" return None site_client, site_id = UserCommand._get_site_id(name, token) ok = False if site_id: ok = site_client.get_session_info(site_id)['ok'] if not ok: print "Please log to the site %s first" % (name) else: print "site %s not found !" % (name) return ok
def status(self, args): """ Get and print status of a job (task). :param args: :return: None see: :func:`pdm.userservicedesk.TransferClient.TransferClient.status` """ token = UserCommand._get_token(args.token) block = args.block job_id = args.job element_id = None if '.' in job_id: job_id, element_id = args.job.split('.') job_id = int(job_id) if element_id is not None: element_id = int(element_id) if token: client = TransferClientFacade(token) self._status(job_id, client, element_id, block=block)
class TestTransferClient(unittest.TestCase): # @mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient") # @mock.patch("pdm.userservicedesk.TransferClient.WorkqueueClient") ##@mock.patch.object(HRService, 'check_token') @mock.patch("pdm.userservicedesk.HRService.SiteClient") @mock.patch.object(Tokens.TokenService, 'unpack') @mock.patch("pdm.userservicedesk.TransferClient.SiteClient") @mock.patch("pdm.workqueue.WorkqueueClient.WorkqueueClient.__new__") def setUp(self, wq_mock, site_mock, mocked_unpack, hr_site_client_mock): self.__future_date = (datetime.timedelta(0, 600) + datetime.datetime.utcnow()).isoformat() site_mock().get_sites.return_value = \ [{'site_id': 1, 'site_name': 'localhost', 'site_desc': 'test localhost site'}, {'site_id': 2, 'site_name': 'remotehost', 'site_desc': 'test remotehost site'}] site_mock.return_value.set_token = mock.MagicMock() self.site_id = site_mock().get_sites()[0]['site_id'] self.site2_id = site_mock().get_sites()[1]['site_id'] conf = { 'CS_secret': 'HJGnbfdsV', 'smtp_server': 'localhost', 'verification_url': 'https://pdm.grid.hep.ph.ic.ac.uk:5443/web/verify', 'smtp_server_login': '******', 'smtp_starttls': 'OPTIONAL', 'smtp_login_req': 'OPTIONAL', 'display_from_address': 'PDM mailer <centos@localhost>', 'mail_subject': 'PDM registration - please verify your email address.', 'mail_expiry': '12:00:00', 'mail_token_secret': 'somemailsecretstring' } # HR self.__service = FlaskServer("pdm.userservicedesk.HRService") self.__service.test_mode(HRService, None) # to skip DB auto build token = {'id': 1, 'expiry': self.__future_date} self.__service.fake_auth("TOKEN", token) # database self.__service.build_db() # build manually # db = self.__service.test_db() self.__service.before_startup(conf) # to continue startup # mock_ct.return_value =1 mocked_unpack.return_value = token self.__htoken = 'whateverhash' self.__client = TransferClientFacade(self.__htoken) assert wq_mock.called mocked_unpack.assert_called_with(self.__htoken) def test_list(self): site = "localhost:/root/file.txt" with mock.patch.object(self.__client._TransferClient__wq_client, 'list') as mock_list: mock_list.return_value = 'root/file.txt' assert self.__client.list(site, **{'priority': 2}) == 'root/file.txt' assert mock_list.called mock_list.assert_called_with(self.site_id, '/root/file.txt', priority=2) print mock_list.call_args_list wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'list') as mock_list: mock_list.return_value = 'root/file.txt' # event if ... assert self.__client.list(wrongurl, **{'priority': 2}) == [] # we return [] assert not mock_list.called def test_sitelist(self): sites = self.__client.list_sites() print sites assert sites[0]['site_name'] == 'localhost' assert sites[1]['site_name'] == 'remotehost' assert 'site_id' not in [dd.keys() for dd in sites] def test_remove(self): site = "localhost:/root/file.txt" # mock_remove.return_value = 'root/file.txt' with mock.patch.object(self.__client._TransferClient__wq_client, 'remove') as mock_remove: mock_remove.return_value = 'root/file.txt removed' assert self.__client.remove(site, **{'priority': 2}) == 'root/file.txt removed' assert mock_remove.called mock_remove.assert_called_with(self.site_id, '/root/file.txt', priority=2) print mock_remove.call_args_remove wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'remove') as mock_remove: mock_remove.return_value = 'whatever..' # event if ... assert self.__client.remove( wrongurl, **{'priority': 2}) == None # we return None assert not mock_remove.called def test_copy(self): s_site = "localhost:/root/file.txt" t_site = "remotehost:/root/file.txt" with mock.patch.object(self.__client._TransferClient__wq_client, 'copy') as mock_copy: mock_copy.return_value = 'root/file.txt copied' assert self.__client.copy(s_site, t_site, **{'priority': 2}) == 'root/file.txt copied' assert mock_copy.called mock_copy.assert_called_with(self.site_id, '/root/file.txt', self.site2_id, '/root/file.txt', priority=2) wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'copy') as mock_copy: mock_copy.return_value = 'whatever..' # even if ... assert self.__client.copy( wrongurl, t_site, **{'priority': 2}) == None # we return None assert not mock_copy.called def test_split_site_path(self): site = "localhost:/root/file.txt" malformed_site = "localhost/root/file.txt" # mind a missing colon multicolon_site = "localhost:/root/file.txt:1" a, b = TransferClientFacade.split_site_path(site) assert a == 'localhost' assert b == '/root/file.txt' c, d = TransferClientFacade.split_site_path(malformed_site) assert d is None assert c is None e, f = TransferClientFacade.split_site_path(multicolon_site) assert e == 'localhost' assert f == '/root/file.txt:1' def test_mkdir(self): site = "localhost:/root/subdir" with mock.patch.object(self.__client._TransferClient__wq_client, 'mkdir') as mock_mkdir: mock_mkdir.return_value = 'root/subdir created' assert self.__client.mkdir(site, **{'priority': 2}) == 'root/subdir created' assert mock_mkdir.called mock_mkdir.assert_called_with(self.site_id, '/root/subdir', priority=2) # now unknown site: wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'mkdir') as mock_mkdir: mock_mkdir.return_value = 'whatever..' # event if ... assert self.__client.mkdir( wrongurl, **{'priority': 2}) == None # we return None assert not mock_mkdir.called def test_rename(self): s_site = "localhost:/root/file.txt" t_site = ":/root/file2.txt" with mock.patch.object(self.__client._TransferClient__wq_client, 'rename') as mock_rename: mock_rename.return_value = 'root/file.txt renamed' assert self.__client.rename(s_site, t_site, **{'priority': 2}) == 'root/file.txt renamed' assert mock_rename.called mock_rename.assert_called_with(self.site_id, '/root/file.txt', '/root/file2.txt', priority=2) wrongurl = "localhost2:/root/file.txt" # no such site, with mock.patch.object(self.__client._TransferClient__wq_client, 'rename') as mock_rename: mock_rename.return_value = 'whatever..' # even if ... assert self.__client.rename( wrongurl, t_site, **{'priority': 2}) == None # we return None assert not mock_rename.called def tearDown(self): pass