def test_when_the_url_returns_a_eula_error_it_returns_a_human_readable_message( faker): access_token = faker.password(length=42, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add( responses.GET, url, status=403, body=( '{"status_code":403,"error_description":"EULA Acceptance Failure",' '"resolution_url":"https://example.com/approve_app?client_id=foo"}' )) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(ForbiddenException) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.value.message == ( f'Request could not be completed because you need to agree to the EULA ' f'at https://example.com/approve_app?client_id=foo') assert len(responses.calls) == 2
def test_when_given_unknown_url_it_raises_exception(faker): access_token = faker.password(length=40, special_chars=False) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(Exception): util.download('msdos:choplifter.bas', 'tmp', access_token=access_token, cfg=config)
def test_when_given_an_http_url_it_downloads_the_url(monkeypatch, mocker, faker): access_token = faker.password(length=40, special_chars=False) http_download = mocker.Mock() monkeypatch.setattr(util.http, 'download', http_download) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): util.download('https://example.com/file.txt', 'tmp', access_token=access_token, cfg=config) http_download.assert_called()
def test_when_the_url_returns_a_403_it_throws_a_forbidden_exception(faker): access_token = faker.password(length=41, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add(responses.GET, url, status=403) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(ForbiddenException) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.value.message.startswith('Forbidden') assert len(responses.calls) == 2
def test_when_the_url_returns_a_500_it_does_not_raise_a_forbidden_exception_and_does_not_return_details_to_user( faker): access_token = faker.password(length=43, special_chars=False) url = 'https://example.com/file.txt' config = config_fixture() responses.add(responses.POST, 'https://uat.urs.earthdata.nasa.gov/oauth/tokens/user', status=200, match_querystring=False) responses.add(responses.GET, url, status=500) with mock.patch('builtins.open', mock.mock_open()): with pytest.raises(Exception) as e: util.download(url, '/tmp', access_token=access_token, cfg=config) assert e.type != ForbiddenException and e.type == Exception assert len(responses.calls) == 2
def process_item(self, item, source): """ Converts an input STAC Item's data into Zarr, returning an output STAC item Parameters ---------- item : pystac.Item the item that should be converted source : harmony.message.Source the input source defining the variables, if any, to subset from the item Returns ------- pystac.Item a STAC item containing the Zarr output """ result = item.clone() result.assets = {} # Create a temporary dir for processing we may do workdir = mkdtemp() try: # Get the data file asset = next(v for k, v in item.assets.items() if 'data' in (v.roles or [])) input_filename = download(asset.href, workdir, logger=self.logger, access_token=self.message.accessToken, cfg=self.config) name = generate_output_filename(asset.href, ext='.zarr') root = self.message.stagingLocation + name try: store = self.s3.get_mapper(root=root, check=False, create=True) netcdf_to_zarr(input_filename, store) except Exception as e: # Print the real error and convert to user-facing error that's more digestible self.logger.error(e, exc_info=1) filename = asset.href.split('?')[0].rstrip('/').split('/')[-1] raise ZarrException('Could not convert file to Zarr: %s' % (filename)) # Update the STAC record result.assets['data'] = Asset(root, title=name, media_type='application/x-zarr', roles=['data']) # Return the STAC record return result finally: # Clean up any intermediate resources shutil.rmtree(workdir)
def test_when_given_a_file_path_it_returns_the_file_path( monkeypatch, mocker, faker): access_token = faker.password(length=40, special_chars=False) http_download = mocker.Mock() monkeypatch.setattr(util.http, 'download', http_download) config = config_fixture() with mock.patch('builtins.open', mock.mock_open()): destination_path = util.download('file:///var/logs/example/file.txt', '/put/file/here/', access_token=access_token, cfg=config) assert destination_path.startswith('/var/logs/example/') assert destination_path.endswith('.txt')
def process_item(self, item, source): """ Processes a single input item. Services that are not aggregating multiple input files should prefer to implement this method rather than #invoke This example copies its input to the output, marking "dpi" and "variables" message attributes as having been processed Parameters ---------- item : pystac.Item the item that should be processed source : harmony.message.Source the input source defining the variables, if any, to subset from the item Returns ------- pystac.Item a STAC catalog whose metadata and assets describe the service output """ result = item.clone() result.assets = {} # Create a temporary dir for processing we may do workdir = mkdtemp() try: # Get the data file asset = next(v for k, v in item.assets.items() if 'data' in (v.roles or [])) input_filename = download(asset.href, workdir, logger=self.logger, access_token=self.message.accessToken) # Mark any fields the service processes so later services do not repeat work dpi = self.message.format.process('dpi') # Variable subsetting variables = source.process('variables') # Do the work here! var_names = [v.name for v in variables] print('Processing item %s, DPI=%d, vars=[%s]' % (item.id, dpi, ', '.join(var_names))) working_filename = os.path.join(workdir, 'tmp.txt') shutil.copyfile(input_filename, working_filename) # Stage the output file with a conventional filename output_filename = generate_output_filename(asset.href, ext=None, variable_subset=None, is_regridded=False, is_subsetted=False) url = stage(working_filename, output_filename, 'text/plain', location=self.message.stagingLocation, logger=self.logger) # Update the STAC record result.assets['data'] = Asset(url, title=output_filename, media_type='text/plain', roles=['data']) # Other metadata updates may be appropriate, such as result.bbox and result.geometry # if a spatial subset was performed # Return the STAC record return result finally: # Clean up any intermediate resources shutil.rmtree(workdir)