def run(self):
if 'override_source_projection' in self.layer_cfg:
# But we can still pass in warp_kwargs to set source projection...
raise NotImplementedError(
'override_source_projection not implemented for raster layers.'
)
extent_str = self.layer_cfg.get('extent', 'background')
extent = CONFIG['project']['extents'][extent_str]
warp_kwargs = {
'resampleAlg': 'bilinear',
'outputBounds': list(extent.values()),
'creationOptions': ['COMPRESS=DEFLATE']
}
if 'warp_kwargs' in self.layer_cfg:
warp_kwargs.update(self.layer_cfg['warp_kwargs'])
file_ext = self.input_ext_override or self.layer_cfg['file_type']
inp_path = find_single_file_by_ext(self.input().path, ext=file_ext)
with temporary_path_dir(self.output()) as tmp_dir:
out_path = os.path.join(tmp_dir, self.filename)
warp_raster(inp_path,
out_path,
layer_cfg=self.layer_cfg,
warp_kwargs=warp_kwargs)
def run(self):
with temporary_path_dir(self.output()) as temp_path:
for filename in self.source_cfg['urls']:
source_path = os.path.join(ASSETS_DIR, 'local_data', filename)
out_path = os.path.join(temp_path, os.path.basename(filename))
shutil.copy2(source_path, out_path)
def run(self):
# TODO: Extract this to LayerTask as a property?
# TODO: Find in dir by self.filename instead? Wouldn't work if the
# upstream task was e.g. unzip
ifile = find_single_file_by_ext(self.input().path,
ext=self.layer_cfg['file_type'])
overviews_kwargs = {
'overview_levels': (2, 4, 8, 16),
'resampling_method': 'average'
}
overviews_kwargs.update(self.layer_cfg.get('overviews_kwargs', {}))
overview_levels = overviews_kwargs['overview_levels']
resampling_str = overviews_kwargs['resampling_method']
try:
resampling_method = rio.enums.Resampling[resampling_str]
except KeyError:
raise RuntimeError(
f"'{resampling_str}' is not a valid resampling method.")
with temporary_path_dir(self.output()) as tmp_dir:
tmp_path = os.path.join(tmp_dir, self.filename)
# Copy the existing file into place. Currently, this task creates
# 'internal overviews', which changes the file itself.
shutil.copy2(ifile, tmp_path)
with rio.open(tmp_path, 'r+') as ds:
ds.build_overviews(overview_levels, resampling_method)
def run(self):
if 'cmr' in self.source_cfg:
raise RuntimeError('Use a FetchCmrGranule task!')
with temporary_path_dir(self.output()) as temp_path:
for url in self.source_cfg['urls']:
fetch_and_write_file(url, output_dir=temp_path)
def run(self):
if os.path.isdir(self.input().path):
source_path = self.input().path
else:
source_path = os.path.dirname(self.input().path)
with temporary_path_dir(self.output()) as temp_path:
shutil.copytree(source_path, temp_path, dirs_exist_ok=True)
def run(self):
gzip_paths = find_in_dir_by_ext(self.input().path, ext='.gz')
with temporary_path_dir(self.output()) as temp_path:
for gzip_path in gzip_paths:
with gzip.open(gzip_path, 'rb') as gf:
decompress_filename = os.path.basename(gzip_path).replace('.gz', '')
decompress_filepath = os.path.join(temp_path, decompress_filename)
with open(decompress_filepath, 'wb') as f:
f.write(gf.read())
def run(self):
with temporary_path_dir(self.output()) as tmp_dir:
out_path = os.path.join(tmp_dir, self.filename)
inp_path = find_single_file_by_ext(self.input().path,
ext=self.layer_cfg['file_type'])
gdal_calc_kwargs = self.layer_cfg['gdal_calc_kwargs']
gdal_calc_raster(inp_path,
out_path,
layer_cfg=self.layer_cfg,
gdal_calc_kwargs=gdal_calc_kwargs)
def run(self):
logger.info(f"Filtering {self.layer_cfg['id']}...")
shapefile = find_single_file_by_ext(self.input().path, ext='.shp')
gdf = filter_vector(
shapefile,
filter_func=self.filter_func
)
with temporary_path_dir(self.output()) as temp_path:
fn = os.path.join(temp_path, self.filename)
gdf.to_file(fn, driver='ESRI Shapefile')
def run(self):
rar_path = find_single_file_by_ext(self.input().path, ext='.rar')
rf = rarfile.RarFile(rar_path)
with temporary_path_dir(self.output()) as temp_path:
if 'extract_file' in self.decompress_kwargs:
rf.extract(self.decompress_kwargs['extract_file'],
path=temp_path)
else:
rf.extractall(path=temp_path)
rf.close()
def run(self):
with temporary_path_dir(self.output()) as temp_dir:
input_fp = find_single_file_by_ext(self.input().path, ext='.nc')
output_filename = f"{self.dataset_name}{self.layer_cfg['file_type']}"
output_fp = os.path.join(temp_dir, output_filename)
from_dataset_path = f'NETCDF:{input_fp}:{self.dataset_name}'
logger.debug(
f'Using gdal.Translate to convert from {from_dataset_path} to {output_fp}'
)
gdal.Translate(output_fp, from_dataset_path)
def run(self):
granule = get_cmr_granule(
granule_ur=self.source_cfg['granule_ur'],
collection_concept_id=self.source_cfg['collection_concept_id'])
with temporary_path_dir(self.output()) as temp_path:
for url in granule.urls:
if not self.session:
self.session = make_session(hosts=[url])
fetch_and_write_file(url,
output_dir=temp_path,
session=self.session)
def run(self):
zf_path = find_single_file_by_ext(self.input().path, ext='.zip')
zf = zipfile.ZipFile(zf_path)
with temporary_path_dir(self.output()) as temp_path:
if 'extract_files' in self.decompress_kwargs:
for extract_file in self.decompress_kwargs['extract_files']:
zf.extract(extract_file, path=temp_path)
else:
# zf.extractall instead???
for fn in zf.namelist():
zf.extract(fn, temp_path)
zf.close()
def run(self):
input_ogr2ogr_kwargs = self.layer_cfg.get('ogr2ogr_kwargs', {})
# Extract the extent from the config, defaulting to 'background'.
layer_extent_str = self.layer_cfg.get('extent', 'background')
extent = CONFIG['project']['extents'][layer_extent_str]
clipdst = ('"{xmin}" "{ymin}" '
'"{xmax}" "{ymax}"').format(**extent) # noqa: FS002
ogr2ogr_kwargs = {
# Output an UTF-8 encoded shapefile instead of default ISO-8859-1
'lco': 'ENCODING=UTF-8',
't_srs': CONFIG['project']['crs'],
# As opposed to `clipsrc`, `clipdst` uses the destination SRS
# (`t_srs`) to clip the input after reprojection.
'clipdst': clipdst,
}
ogr2ogr_kwargs.update(input_ogr2ogr_kwargs)
if 'input_filename' in ogr2ogr_kwargs:
input_filename = os.path.join(
self.input().path,
ogr2ogr_kwargs.pop('input_filename')
)
else:
shapefile = find_single_file_by_ext(self.input().path, ext='.shp')
input_filename = shapefile
with temporary_path_dir(self.output()) as temp_path:
# Before doing the requested transformation, make the vector data
# valid. We have to do the SQL step here because this command will
# change the internal table name of the output shapefile.
infile = os.path.join(self.input().path, input_filename)
# TODO probably need to cleanup this output? Put it in another dir?
valid_outfile = os.path.join(
temp_path,
'valid.shp'
)
valid_kwargs = {'makevalid': ''}
if 'sql' in ogr2ogr_kwargs:
valid_kwargs['sql'] = ogr2ogr_kwargs.pop('sql')
ogr2ogr(infile, valid_outfile, **valid_kwargs)
infile = valid_outfile
# Do the requested transformation
outfile = os.path.join(
temp_path,
self.filename
)
ogr2ogr(infile, outfile, **ogr2ogr_kwargs)
