def test_extract_download(data): fileId, filename = data filepath = localDataRoot / str(filename) if not filepath.exists(): gc = GirderClient(apiUrl=source_api_root) gc.authenticate(apiKey=os.environ.get('GIRDER_API_KEY')) gc.downloadFile(fileId, str(filepath)) with zipfile.ZipFile(filepath, 'r') as zipref: zipref.extractall(localDataRoot)
def download_source_media(girder_client: GirderClient, datasetId: str, dest: Path) -> Tuple[List[str], str]: """Download media for dataset to dest path""" media = models.DatasetSourceMedia( **girder_client.get(f'dive_dataset/{datasetId}/media')) dataset = models.GirderMetadataStatic( **girder_client.get(f'dive_dataset/{datasetId}')) if dataset.type == constants.ImageSequenceType: for frameImage in media.imageData: girder_client.downloadItem(frameImage.id, str(dest)) return [str(dest / image.filename) for image in media.imageData], dataset.type elif dataset.type == constants.VideoType and media.video is not None: destination_path = str(dest / media.video.filename) girder_client.downloadFile(media.video.id, destination_path) return [destination_path], dataset.type else: raise Exception(f"unexpected metadata {str(dataset.dict())}")
def transform(self): f = StringIO() cl = GirderClient(apiUrl='https://data.kitware.com/api/v1') cl.downloadFile(self.value, f) return f.getvalue()
class BaseIntegrationTest(unittest.TestCase): def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True): super(BaseIntegrationTest, self).__init__(name) self._job_id = None self._script_id = None self._output_folder_id = None self._input_folder_id = None self._girder_url = girder_url self._girder_user = girder_user self._girder_password = girder_password self._job_timeout = job_timeout self._data = 'Need more input!' self._cleanup = cleanup def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): if not self._cleanup: return if self._job_id: try: url = 'jobs/%s' % self._job_id self._client.delete(url) except Exception as e: traceback.print_exc() if self._script_id: try: url = 'scripts/%s' % self._script_id self._client.delete(url) except Exception: traceback.print_exc() if self._output_folder_id: try: url = 'folder/%s' % self._output_folder_id self._client.delete(url) except Exception: traceback.print_exc() if self._input_folder_id: try: url = 'folder/%s' % self._input_folder_id self._client.delete(url) except Exception: traceback.print_exc() def create_script(self, commands=[ 'sleep 10', 'cat CumulusIntegrationTestInput' ]): body = { 'commands': commands, 'name': 'CumulusIntegrationTestLob' } r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] def create_input(self, folder_name='CumulusInput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._input_folder_id = r['_id'] size = len(self._data) item = self._client.uploadFile(self._input_folder_id, StringIO(self._data), 'CumulusIntegrationTestInput', size, parentType='folder') self._item_id = item['itemId'] def create_output_folder(self, folder_name='CumulusOutput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._output_folder_id = r['_id'] def create_job(self, job_name='CumulusIntegrationTestJob', tail=None): body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [ { 'folderId': self._input_folder_id, 'path': '.' } ] } if tail: body['output'].append({ "path": tail, "tail": True }) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] def submit_job(self, job_params={}, timeout=None): url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(job_params)) start = time.time() while True: time.sleep(1) r = self._client.get('jobs/%s' % self._job_id) if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) self.fail(str(r)) elif r['status'] == 'complete': break if time.time() - start > timeout: self.fail('Job didn\'t complete in timeout') def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) path = os.path.join(tempfile.gettempdir(), self._job_id) try: self._client.downloadFile(r[0]['_id'], path) with open(path, 'rb') as fp: self.assertEqual(fp.read(), self._data) finally: if os.path.exists(path): os.remove(path)
dataset_folder_id = dataset_folder['_id'] parameters = { 'id': fileId, 'type': 'file' } # Get the file resource so we can get the name input_file = client.get('resource/%s' % str(fileId), parameters=parameters) input_file_name = input_file['name'] output_file_name = input_file_name.replace('.nc', '.json') try: # Now download the dataset (fd, filepath) = tempfile.mkstemp() os.close(fd) client.downloadFile(fileId, filepath) # Create temp file and convert to GeoJs contour JSON format output_dir = tempfile.mkdtemp() output_filepath = os.path.join(output_dir, output_file_name) with open(output_filepath, 'w') as fp: fp.write(json_util.dumps(convert(filepath, variable, timestep))) # Create an item for this file output_item = client.createItem(dataset_folder_id, output_file_name, output_file_name) # Now upload the result client.uploadFileToItem(output_item['_id'], output_filepath) output_item_id = output_item['_id']
class BaseIntegrationTest(unittest.TestCase): def __init__(self, name, girder_url, girder_user, girder_password, job_timeout=60, cleanup=True): super(BaseIntegrationTest, self).__init__(name) self._job_id = None self._script_id = None self._output_folder_id = None self._input_folder_id = None self._girder_url = girder_url self._girder_user = girder_user self._girder_password = girder_password self._job_timeout = job_timeout self._data = 'Need more input!' self._cleanup = cleanup def setUp(self): url = '%s/api/v1' % self._girder_url self._client = GirderClient(apiUrl=url) self._client.authenticate(self._girder_user, self._girder_password) user = self._client.get('user/me') self._user_id = user['_id'] r = list(self._client.listFolder(self._user_id, 'user', name='Private')) self.assertEqual(len(r), 1) self._private_folder_id = r[0]['_id'] def tearDown(self): if not self._cleanup: return if self._job_id: try: url = 'jobs/%s' % self._job_id self._client.delete(url) except Exception as e: traceback.print_exc() if self._script_id: try: url = 'scripts/%s' % self._script_id self._client.delete(url) except Exception: traceback.print_exc() if self._output_folder_id: try: url = 'folder/%s' % self._output_folder_id self._client.delete(url) except Exception: traceback.print_exc() if self._input_folder_id: try: url = 'folder/%s' % self._input_folder_id self._client.delete(url) except Exception: traceback.print_exc() def create_script(self, commands=['sleep 10', 'cat CumulusIntegrationTestInput']): body = {'commands': commands, 'name': 'CumulusIntegrationTestLob'} r = self._client.post('scripts', data=json.dumps(body)) self._script_id = r['_id'] def create_input(self, folder_name='CumulusInput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._input_folder_id = r['_id'] size = len(self._data) item = self._client.uploadFile(self._input_folder_id, StringIO(self._data), 'CumulusIntegrationTestInput', size, parentType='folder') self._item_id = item['itemId'] def create_output_folder(self, folder_name='CumulusOutput'): r = self._client.createFolder(self._private_folder_id, folder_name) self._output_folder_id = r['_id'] def create_job(self, job_name='CumulusIntegrationTestJob', tail=None): body = { 'name': job_name, 'scriptId': self._script_id, 'output': [{ 'folderId': self._output_folder_id, 'path': '.' }], 'input': [{ 'folderId': self._input_folder_id, 'path': '.' }] } if tail: body['output'].append({"path": tail, "tail": True}) job = self._client.post('jobs', data=json.dumps(body)) self._job_id = job['_id'] def submit_job(self, job_params={}, timeout=None): url = 'clusters/%s/job/%s/submit' % (self._cluster_id, self._job_id) self._client.put(url, data=json.dumps(job_params)) start = time.time() while True: time.sleep(1) r = self._client.get('jobs/%s' % self._job_id) if r['status'] in ['error', 'unexpectederror']: r = self._client.get('jobs/%s/log' % self._job_id) self.fail(str(r)) elif r['status'] == 'complete': break if time.time() - start > timeout: self.fail('Job didn\'t complete in timeout') def assert_output(self): r = self._client.listItem(self._output_folder_id) self.assertEqual(len(r), 4) stdout_item = None for i in r: if i['name'].startswith('CumulusIntegrationTestJob-%s.o' % self._job_id): stdout_item = i break self.assertIsNotNone(stdout_item) r = self._client.get('item/%s/files' % i['_id']) self.assertEqual(len(r), 1) path = os.path.join(tempfile.gettempdir(), self._job_id) try: self._client.downloadFile(r[0]['_id'], path) with open(path, 'rb') as fp: self.assertEqual(fp.read(), self._data) finally: if os.path.exists(path): os.remove(path)