def upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None, girder_token=None): status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job['_id']) headers = {'Girder-Token': girder_token} assetstore_base_url = get_assetstore_url_base(cluster) assetstore_id = get_assetstore_id(girder_token, cluster) if not job_dir: job_dir = job['dir'] try: with get_connection(girder_token, cluster) as conn: for output in job['output']: if 'folderId' in output and 'path' in output: folder_id = output['folderId'] path = os.path.join(job_dir, output['path']) download_path(conn, girder_token, folder_id, path, assetstore_base_url, assetstore_id) except HttpError as e: job['status'] = JobState.ERROR url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl, job['_id']) logger = get_post_logger('job', girder_token, url) logger.exception(e.responseText) r = requests.patch(status_url, headers=headers, json={'status': JobState.ERROR}) check_status(r) if _get_on_complete(job) == 'terminate': cluster_log_url = '%s/clusters/%s/log' % \ (cumulus.config.girder.baseUrl, cluster['_id']) command.send_task( 'cumulus.tasks.cluster.terminate_cluster', args=(cluster,), kwargs={'log_write_url': cluster_log_url, 'girder_token': girder_token}) # If we where uploading move job to the complete state if job['status'] == JobState.UPLOADING: job_status = from_string(job['status'], task=None, cluster=cluster, job=job, log_write_url=log_write_url, girder_token=girder_token, conn=conn) job_status = Complete(job_status) job_status = job_status.next(JobQueueState.COMPLETE) job_status.run() r = requests.patch(status_url, headers=headers, json={'status': str(job_status)}) check_status(r)
def upload_job_output_to_folder(cluster, job, log_write_url=None, job_dir=None, girder_token=None): status_url = '%s/jobs/%s' % (cumulus.config.girder.baseUrl, job['_id']) headers = {'Girder-Token': girder_token} assetstore_base_url = get_assetstore_url_base(cluster) assetstore_id = get_assetstore_id(girder_token, cluster) if not job_dir: job_dir = job['dir'] try: with get_connection(girder_token, cluster) as conn: for output in job['output']: if 'folderId' in output and 'path' in output: folder_id = output['folderId'] path = os.path.join(job_dir, output['path']) download_path(conn, girder_token, folder_id, path, assetstore_base_url, assetstore_id) except HttpError as e: job['status'] = JobState.ERROR url = '%s/jobs/%s/log' % (cumulus.config.girder.baseUrl, job['_id']) logger = get_post_logger('job', girder_token, url) logger.exception(e.responseText) r = requests.patch(status_url, headers=headers, json={'status': JobState.ERROR}) check_status(r) if _get_on_complete(job) == 'terminate': cluster_log_url = '%s/clusters/%s/log' % \ (cumulus.config.girder.baseUrl, cluster['_id']) command.send_task( 'cumulus.tasks.cluster.terminate_cluster', args=(cluster,), kwargs={'log_write_url': cluster_log_url, 'girder_token': girder_token}) # If we where uploading move job to the complete state if job['status'] == JobState.UPLOADING: job_status = from_string(job['status'], task=None, cluster=cluster, job=job, log_write_url=log_write_url, girder_token=girder_token, conn=conn) job_status = Complete(job_status) job_status = job_status.next(JobQueueState.COMPLETE) job_status.run() r = requests.patch(status_url, headers=headers, json={'status': str(job_status)}) check_status(r)
def test_download(self): cluster_connection = mock.MagicMock() cluster_connection.list.side_effect = self.paths download_path(cluster_connection, self._token['_id'], self._folder['_id'], '/test', 'sftp_assetstores', self._assetstore['_id'], upload=False) files = [] folders = [] for (p, _) in self.model('folder').fileList(self._folder, user=self._user, path='/', data=False): files.append(p) def traverse_folders(folder, path='/Public'): for f in self.model('folder').childFolders(folder, 'folder', user=self._user): print('trav: path: %s' % (path)) current_path = os.path.join(path, f['name']) print('trav: path: %s' % (path)) folders.append(current_path) traverse_folders(f, path=current_path) traverse_folders(self._folder) files.sort() folders.sort() expected_path = os.path.abspath( 'plugins/cumulus/plugin_tests/fixtures/expected_paths.json') with open(expected_path) as fp: expected = json.load(fp) self.assertEqual(files, expected['files']) self.assertEqual(folders, expected['folders'])
def test_import_path(self): file = { 'name': 'test.txt', 'mode': 1, 'size': 123 } folder = { 'name': 'folder', 'mode': stat.S_IFDIR, 'size': 1234 } cluster_connection = mock.MagicMock() cluster_connection.list.side_effect = [[file, folder], [file]] girder_token = 'dummy' parent = { '_id': 'dummy_id' } path = '/my/path' assetstore_id = 'dummy_id' # Mock create item def _create_item(url, request): content = { '_id': 'dummy' } content = json.dumps(content).encode('utf8') headers = { 'content-length': len(content), 'content-type': 'application/json' } self._item_requests.append(request) return httmock.response(200, content, headers, request=request) item_url = '/api/v1/item' create_item = httmock.urlmatch( path=r'^%s$' % item_url, method='POST')(_create_item) # Mock create file def _create_file(url, request): content = { '_id': 'dummy' } content = json.dumps(content).encode('utf8') headers = { 'content-length': len(content), 'content-type': 'application/json' } self._file_requests.append(request) return httmock.response(200, content, headers, request=request) file_url = '/api/v1/sftp_assetstores/%s/files' % assetstore_id create_file = httmock.urlmatch( path=r'^%s$' % file_url, method='POST')(_create_file) # Mock create folder def _create_folder(url, request): content = { '_id': 'dummy' } content = json.dumps(content).encode('utf8') headers = { 'content-length': len(content), 'content-type': 'application/json' } self._folder_requests.append(request) return httmock.response(200, content, headers, request=request) folder_url = '/api/v1/folder' create_folder = httmock.urlmatch( path=r'^%s$' % folder_url, method='POST')(_create_folder) # Mock list folder def _list_folder(url, request): content = [] content = json.dumps(content).encode('utf8') headers = { 'content-length': len(content), 'content-type': 'application/json' } return httmock.response(200, content, headers, request=request) folder_url = '/api/v1/folder' list_folder = httmock.urlmatch( path=r'^%s$' % folder_url, method='GET')(_list_folder) with httmock.HTTMock(create_item, create_file, create_folder, list_folder): download_path(cluster_connection, girder_token, parent, path, 'sftp_assetstores', assetstore_id, upload=False) self.assertEqual(len(self._item_requests), 2) self.assertEqual(len(self._file_requests), 2) self.assertEqual(len(self._folder_requests), 1)