def convert_to_1b(in_filename, out_filename,
time_slot_start, time_slot_end,
shortname, orbit):
"""Convert hrpt file to level 1b.
"""
(handle, tempname) = tempfile.mkstemp(prefix="hrpt_decommuted",
dir=WORKING_DIR)
os.close(handle)
del handle
LOG.debug("Decommuting...")
decommutation(in_filename, tempname,
time_slot_start, time_slot_end,
shortname)
LOG.debug("Calibrating, navigating...")
calibration_navigation(tempname, time_slot_start, shortname)
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, "regional" + shortname + ".cfg"))
new_dir = conf.get("avhrr-level2", "dir", raw=True)
new_name = conf.get("avhrr-level2", "filename", raw=True)
options = {"satellite": shortname, "orbit": orbit}
pathname = time_slot_start.strftime(os.path.join(new_dir, new_name))%options
LOG.debug("Saving to "+pathname)
ensure_dir(pathname)
shutil.move(tempname, pathname)
def convert_to_1b(in_filename, out_filename, time_slot_start, time_slot_end,
shortname, orbit):
"""Convert hrpt file to level 1b.
"""
(handle, tempname) = tempfile.mkstemp(prefix="hrpt_decommuted",
dir=WORKING_DIR)
os.close(handle)
del handle
LOG.debug("Decommuting...")
decommutation(in_filename, tempname, time_slot_start, time_slot_end,
shortname)
LOG.debug("Calibrating, navigating...")
calibration_navigation(tempname, time_slot_start, shortname)
conf = ConfigParser()
conf.read(os.path.join(CONFIG_PATH, "regional" + shortname + ".cfg"))
new_dir = conf.get("avhrr-level2", "dir", raw=True)
new_name = conf.get("avhrr-level2", "filename", raw=True)
options = {"satellite": shortname, "orbit": orbit}
pathname = time_slot_start.strftime(os.path.join(new_dir,
new_name)) % options
LOG.debug("Saving to " + pathname)
ensure_dir(pathname)
shutil.move(tempname, pathname)
def pil_save(self, filename, compression = 6, fformat = None):
"""Save the image to the given *filename* using PIL. For now, the
compression level [0-9] is ignored, due to PIL's lack of support. See
also :meth:`save`.
"""
# PIL does not support compression option.
del compression
if self.is_empty():
raise IOError("Cannot save an empty image")
ensure_dir(filename)
fformat = fformat or os.path.splitext(filename)[1][1:4]
fformat = check_image_format(fformat)
self.pil_image().save(filename, fformat)
def pil_save(self, filename, compression=6, fformat=None):
"""Save the image to the given *filename* using PIL. For now, the
compression level [0-9] is ignored, due to PIL's lack of support. See
also :meth:`save`.
"""
# PIL does not support compression option.
del compression
if self.is_empty():
raise IOError("Cannot save an empty image")
ensure_dir(filename)
fformat = fformat or os.path.splitext(filename)[1][1:4]
fformat = check_image_format(fformat)
params = {}
if fformat == 'png':
# Take care of GeoImage.tags.
params['pnginfo'] = self._pngmeta()
self.pil_image().save(filename, fformat, **params)
def geotiff_save(self, filename, compression=6,
tags=None, gdal_options=None,
blocksize=0, geotransform=None,
spatialref=None, floating_point=False,
writer_options=None):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata). By
default it uses the 'area' instance to generate geotransform and
spatialref information, this can be overwritten by the arguments
*geotransform* and *spatialref*. *floating_point* allows the saving of
'L' mode images in floating point format if set to True.
When argument *writer_options* is not none and entry 'fill_value_subst'
is included, its numeric value will be used to substitute image data
that would be equal to the fill_value (used to replace masked data).
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
tags = tags or {}
writer_options = writer_options or {}
if floating_point:
if self.mode != "L":
raise ValueError("Image must be in 'L' mode for floating point"
" geotif saving")
if self.fill_value is None:
logger.warning("Image with floats cannot be transparent, "
"so setting fill_value to 0")
self.fill_value = 0
channels = [self.channels[0].astype(np.float64)]
fill_value = self.fill_value or [0]
gformat = gdal.GDT_Float64
opacity = 0
else:
nbits = int(tags.get("NBITS", "8"))
if nbits > 16:
dtype = np.uint32
gformat = gdal.GDT_UInt32
elif nbits > 8:
dtype = np.uint16
gformat = gdal.GDT_UInt16
else:
dtype = np.uint8
gformat = gdal.GDT_Byte
opacity = np.iinfo(dtype).max
channels, fill_value = self._finalize(dtype)
fill_value_subst = writer_options.get(
write_opts.WR_OPT_FILL_VALUE_SUBST, None)
if fill_value is not None and fill_value_subst is not None:
for i, chan in enumerate(channels):
np.place(chan, chan == fill_value[i], int(fill_value_subst))
logger.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if(self.mode == "L"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
1,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels,
opacity, fill_value)
elif(self.mode == "LA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[1],
fill_value)
elif(self.mode == "RGB"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
3,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels,
opacity, fill_value)
elif(self.mode == "RGBA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[3],
fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode"
" %s is not implemented." % self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotransform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
from pyresample import utils
from mpop.projector import get_area_def
try:
area = get_area_def(self.area)
except (utils.AreaNotFound, AttributeError):
area = self.area
try:
adfgeotransform = [area.area_extent[0], area.pixel_size_x, 0,
area.area_extent[3], 0, -area.pixel_size_y]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.ImportFromProj4(area.proj4_string)
srs.SetProjCS(area.proj_id)
try:
srs.SetWellKnownGeogCS(area.proj_dict['ellps'])
except KeyError:
pass
try:
# Check for epsg code.
srs.ImportFromEPSG(int(area.proj_dict['init'].
lower().split('epsg:')[1]))
except (KeyError, IndexError):
pass
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
logger.exception("Could not load geographic data, invalid area")
self.tags.update({'TIFFTAG_DATETIME':
self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, '')
# Close the dataset
dst_ds = None
def save_object(self, obj, hook=None):
"""save *obj* to the filelist.
"""
files_by_ext = self._get_by_ext()
for extkey in files_by_ext:
path, filename = os.path.split(files_by_ext[extkey][0])
origname = filename
try:
ensure_dir(files_by_ext[extkey][0])
handle, tmpfilename = tempfile.mkstemp(extkey,
"mpop_tmp",
path)
os.fsync(handle)
obj.save(tmpfilename)
os.fsync(handle)
os.chmod(tmpfilename, 0644)
os.fsync(handle)
except Exception:
logger.exception("Something went wrong in saving file... "
"Dumping trace.")
logger.warning("Job skipped, going on with the next.")
continue
for filename in files_by_ext[extkey][1:]:
path2, trash = os.path.split(filename)
del trash
ensure_dir(filename)
handle2, tmpfilename2 = tempfile.mkstemp(extkey,
"mpop_tmp",
path2)
os.fsync(handle2)
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
except (IOError, OSError):
logger.exception("Copying file %s to %s failed"
%(tmpfilename,tmpfilename2))
logger.info("Retrying...")
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
logger.info("Went OK this time...")
except (IOError, OSError):
logger.exception("No way...")
try:
os.rename(tmpfilename2, filename)
if hook:
hook(obj, filename=origname, uri=filename)
except (IOError, OSError):
logger.exception("Renaming file %s to %s failed"
%(tmpfilename2,filename))
logger.info("Retrying...")
try:
os.rename(tmpfilename2, filename)
if hook:
hook(obj, filename=origname, uri=filename)
except (IOError, OSError):
logger.exception("No way...")
logger.debug("Done saving "+filename)
os.rename(tmpfilename, files_by_ext[extkey][0])
os.fsync(handle)
os.close(handle)
if hook:
hook(obj, filename=origname, uri=files_by_ext[extkey][0])
logger.debug("Done saving "+files_by_ext[extkey][0])
def save_object(self, obj, hook=None):
"""save *obj* to the filelist.
"""
files_by_ext = self._get_by_ext()
for extkey in files_by_ext:
path, filename = os.path.split(files_by_ext[extkey][0])
origname = filename
try:
ensure_dir(files_by_ext[extkey][0])
handle, tmpfilename = tempfile.mkstemp(extkey, "mpop_tmp",
path)
os.fsync(handle)
obj.save(tmpfilename)
os.fsync(handle)
os.chmod(tmpfilename, 0644)
os.fsync(handle)
except Exception:
logger.exception("Something went wrong in saving file... "
"Dumping trace.")
logger.warning("Job skipped, going on with the next.")
continue
for filename in files_by_ext[extkey][1:]:
path2, trash = os.path.split(filename)
del trash
ensure_dir(filename)
handle2, tmpfilename2 = tempfile.mkstemp(
extkey, "mpop_tmp", path2)
os.fsync(handle2)
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
except (IOError, OSError):
logger.exception("Copying file %s to %s failed" %
(tmpfilename, tmpfilename2))
logger.info("Retrying...")
try:
shutil.copy(tmpfilename, tmpfilename2)
os.fsync(handle2)
os.close(handle2)
logger.info("Went OK this time...")
except (IOError, OSError):
logger.exception("No way...")
try:
os.rename(tmpfilename2, filename)
if hook:
hook(obj, filename=origname, uri=filename)
except (IOError, OSError):
logger.exception("Renaming file %s to %s failed" %
(tmpfilename2, filename))
logger.info("Retrying...")
try:
os.rename(tmpfilename2, filename)
if hook:
hook(obj, filename=origname, uri=filename)
except (IOError, OSError):
logger.exception("No way...")
logger.debug("Done saving " + filename)
os.rename(tmpfilename, files_by_ext[extkey][0])
os.fsync(handle)
os.close(handle)
if hook:
hook(obj, filename=origname, uri=files_by_ext[extkey][0])
logger.debug("Done saving " + files_by_ext[extkey][0])
def geotiff_save(self, filename, compression=6,
tags=None, gdal_options=None,
blocksize=0, geotransform=None,
spatialref=None, floating_point=False):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata). By
default it uses the 'area' instance to generate geotransform and
spatialref information, this can be overwritten by the arguments
*geotransform* and *spatialref*. *floating_point* allows the saving of
'L' mode images in floating point format if set to True.
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
if floating_point:
if self.mode != "L":
raise ValueError("Image must be in 'L' mode for floating point"
" geotif saving")
channels = [self.channels[0].astype(np.float64)]
fill_value = self.fill_value or 0
gformat = gdal.GDT_Float64
else:
channels, fill_value = self._finalize()
gformat = gdal.GDT_Byte
LOG.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if(self.mode == "L"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
1,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif(self.mode == "LA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
2,
gformat,
g_opts)
self._gdal_write_channels(dst_ds,
channels[:-1], channels[1],
fill_value)
elif(self.mode == "RGB"):
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename,
self.width,
self.height,
3,
gformat,
g_opts)
else:
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif(self.mode == "RGBA"):
ensure_dir(filename)
g_opts.append("ALPHA=YES")
dst_ds = raster.Create(filename,
self.width,
self.height,
4,
gformat,
g_opts)
self._gdal_write_channels(dst_ds, channels[:-1], channels[3], fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode"
" %s is not implemented."%self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotranform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
try:
from pyresample import utils
from mpop.projector import get_area_def
area = get_area_def(self.area)
except (utils.AreaNotFound, AttributeError):
area = self.area
try:
adfgeotransform = [area.area_extent[0], area.pixel_size_x, 0,
area.area_extent[3], 0, -area.pixel_size_y]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.ImportFromProj4(area.proj4_string)
srs.SetProjCS(area.proj_id)
try:
srs.SetWellKnownGeogCS(area.proj_dict['ellps'])
except KeyError:
pass
try:
# Check for epsg code.
srs.SetAuthority('PROJCS', 'EPSG',
int(area.proj_dict['init'].
split('epsg:')[1]))
except (KeyError, IndexError):
pass
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
LOG.exception("Could not load geographic data, invalid area")
self.tags.update({'TIFFTAG_DATETIME':
self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, '')
# Close the dataset
dst_ds = None
def geotiff_save(
self, filename, compression=6, tags=None, gdal_options=None, blocksize=0, geotransform=None, spatialref=None
):
"""Save the image to the given *filename* in geotiff_ format, with the
*compression* level in [0, 9]. 0 means not compressed. The *tags*
argument is a dict of tags to include in the image (as metadata).
By default it uses the 'area' instance to generate geotransform and spatialref
information, this can be overwritte by the arguments *geotransform* and
*spatialref*.
.. _geotiff: http://trac.osgeo.org/geotiff/
"""
from osgeo import gdal, osr
raster = gdal.GetDriverByName("GTiff")
channels, fill_value = self._finalize()
LOG.debug("Saving to GeoTiff.")
if tags is not None:
self.tags.update(tags)
if gdal_options is not None:
self.gdal_options.update(gdal_options)
g_opts = ["=".join(i) for i in self.gdal_options.items()]
if compression != 0:
g_opts.append("COMPRESS=DEFLATE")
g_opts.append("ZLEVEL=" + str(compression))
if blocksize != 0:
g_opts.append("TILED=YES")
g_opts.append("BLOCKXSIZE=" + str(blocksize))
g_opts.append("BLOCKYSIZE=" + str(blocksize))
if self.mode == "L":
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename, self.width, self.height, 1, gdal.GDT_Byte, g_opts)
else:
dst_ds = raster.Create(filename, self.width, self.height, 2, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif self.mode == "RGB":
ensure_dir(filename)
if fill_value is not None:
dst_ds = raster.Create(filename, self.width, self.height, 3, gdal.GDT_Byte, g_opts)
else:
dst_ds = raster.Create(filename, self.width, self.height, 4, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, 255, fill_value)
elif self.mode == "RGBA":
ensure_dir(filename)
dst_ds = raster.Create(filename, self.width, self.height, 4, gdal.GDT_Byte, g_opts)
self._gdal_write_channels(dst_ds, channels, channels[3], fill_value)
else:
raise NotImplementedError("Saving to GeoTIFF using image mode" " %s is not implemented." % self.mode)
# Create raster GeoTransform based on upper left corner and pixel
# resolution ... if not overwritten by argument geotranform.
if geotransform:
dst_ds.SetGeoTransform(geotransform)
if spatialref:
if not isinstance(spatialref, str):
spatialref = spatialref.ExportToWkt()
dst_ds.SetProjection(spatialref)
else:
try:
from pyresample import utils
from mpop.projector import get_area_def
area = get_area_def(self.area_id)
except (utils.AreaNotFound, AttributeError):
area = self.area_id
try:
adfgeotransform = [
area.area_extent[0],
area.pixel_size_x,
0,
area.area_extent[3],
0,
-area.pixel_size_y,
]
dst_ds.SetGeoTransform(adfgeotransform)
srs = osr.SpatialReference()
srs.SetProjCS(area.proj_id)
srs.ImportFromProj4(area.proj4_string)
srs = srs.ExportToWkt()
dst_ds.SetProjection(srs)
except AttributeError:
LOG.exception("Could not load geographic data, invalid area")
self.tags.update({"TIFFTAG_DATETIME": self.time_slot.strftime("%Y:%m:%d %H:%M:%S")})
dst_ds.SetMetadata(self.tags, "")
# Close the dataset
dst_ds = None