def __init__(self,dir, recurse=True, archive=False):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
'''
format_regex = formats.format_regex
dir=utilities.uncpath(utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats={}
for f in utilities.rglob(dir,'|'.join(format_regex), True, re.IGNORECASE, recurse=recurse, archive=archive):
#Don't return existing overviews
if f[-7:] in ('qlk.jpg','thm.jpg'):continue
#Use utf-8 encoding to fix Issue 20
if f[:4]=='/vsi':f=utilities.encode(f)
else:f=utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r,os.path.basename(f),re.IGNORECASE):
if fileformats.has_key(r):fileformats[r].append(f)
else:fileformats[r]=[f]
break
files=[]
for r in format_regex:
if fileformats.has_key(r):files.extend(fileformats[r])
#Class vars
self.errors=[] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
self.files=files
self.file=''
self.filecount=len(self.files)
def __init__(self, dir, recurse=True, archive=False, excludes=[]):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
@type recurse: C{bool}
@param recurse: Recurse into subdirectories?
@type archive: C{bool}
@param archive: Look in zip/gzip archives
@type excludes: C{list}
@param excludes: List of glob style file/directory exclusion pattern/s
'''
#Class vars
self.errors = [
] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
format_regex = formats.format_regex
dir = utilities.uncpath(
utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats = {}
for f in utilities.rglob(dir,
'|'.join(format_regex),
True,
re.IGNORECASE,
recurse=recurse,
archive=archive,
excludes=excludes,
onerror=self.onerror,
followlinks=False):
#Don't return existing overviews
if f[-7:] in ('qlk.jpg', 'thm.jpg'): continue
#Use utf-8 encoding to fix Issue 20
if f[:4] == '/vsi': f = utilities.encode(f)
else:
f = utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r, os.path.basename(f), re.IGNORECASE):
if fileformats.has_key(r): fileformats[r].append(f)
else: fileformats[r] = [f]
break
files = []
for r in format_regex:
if fileformats.has_key(r): files.extend(fileformats[r])
#Class vars
self.files = files
self.file = ''
self.filecount = len(self.files)
def v1(self,f=None):
dom = self._dom
self.metadata['sceneid'] = dom.xpath('string(/Dimap_Document/Dataset_Id/DATASET_NAME)')
bands=dom.xpath('/Dimap_Document/Spectral_Band_Info/BAND_DESCRIPTION')
self.metadata['bands']=','.join([band.xpath('string(.)') for band in bands])
try:__default__.Dataset.__getmetadata__(self, f) #autopopulate basic metadata
except geometry.GDALError,err: #Work around reading images with lowercase filenames when
#the DATA_FILE_PATH is uppercase
# - eg samba filesystems which get converted to lowercase
dfp=dom.xpath('/Dimap_Document/Data_Access/Data_File/DATA_FILE_PATH')[0]
fn=utilities.encode(dfp.xpath('string(@href)')) #XML is unicode, gdal.Open doesn't like unicode
if not os.path.dirname(fn):fn=os.path.join(os.path.dirname(f),fn)
exists,img=utilities.exists(fn,True)
if exists and not os.path.exists(fn):
import tempfile
tmpfd,tmpfn=tempfile.mkstemp(suffix='.dim',prefix='metadata')
dfp.set('href',img)
tmpfo=os.fdopen(tmpfd,'w')
tmpfo.write(etree.tostring(dom))
tmpfo.flush()
tmpfo.close()
cwd=os.path.abspath(os.curdir)
tmp=os.path.split(tmpfn)
os.chdir(tmp[0]) #CD to the tmp dir so __default__.Dataset.__getmetadata__ doesn't
__default__.Dataset.__getmetadata__(self, tmp[1])
gdalmd=self._gdaldataset.GetMetadata()
self._gdaldataset=geometry.OpenDataset(img)
self._gdaldataset.SetMetadata(gdalmd)
os.unlink(tmpfn)
os.chdir(cwd)
else:raise
def UpdateRecord(self,extent,attributes,where_clause):
'''Update record/s
@type where_clause: C{str}
@param where_clause: Shapefile supported SQL where clause
@type attributes: C{dict}
@param attributes: Must match field names passed to __init__()
'''
try:
geom=GeomFromExtent(extent,self._srs)
if self._srs.IsGeographic(): #basic coordinate bounds test. Can't do for projected though
srs=osr.SpatialReference()
srs.ImportFromEPSG(4283)#4326)
valid = GeomFromExtent([-180,-90,180,90], srs=srs)
if not valid.Contains(geom):
#raise ValueError, 'Invalid extent coordinates'
warnings.warn('Invalid extent coordinates')
lyr=self._shape.GetLayer()
lyr.SetAttributeFilter(where_clause)
feat=lyr.GetNextFeature()
try:feat.SetGeometryDirectly(geom)
except:return self.WriteRecord(extent,attributes)
while feat:
for a in attributes:
if a in self.fields:feat.SetField(self.fields[a], utilities.encode(attributes[a]))
lyr.SetFeature(feat)
feat=lyr.GetNextFeature()
lyr.SyncToDisk()
except Exception, err:
self.__error__(err)
def getoverview(self,outfile=None,width=800,format='JPG'):
'''
Generate overviews for DIMAP imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
@todo: Should we do something with the band display order metadata?
<Band_Display_Order>
<RED_CHANNEL>1</RED_CHANNEL>
<GREEN_CHANNEL>2</GREEN_CHANNEL>
<BLUE_CHANNEL>3</BLUE_CHANNEL>
</Band_Display_Order>
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
f=self.fileinfo['filepath']
#browse=os.path.join(os.path.dirname(f),'PREVIEW.JPG')
fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
fn=utilities.encode(fp.xpath('string(@href)')) #XML is unicode, gdal.Open doesn't like unicode
browse=os.path.join(os.path.dirname(f),fn)
if os.path.exists(browse) and gdal.Open(browse).RasterXSize >= width:
try:return overviews.resize(browse,outfile,width)
except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
def WriteRecord(self,extent,attributes):
'''Write record
@type extent: C{list}
@param extent: [[ulx,uly],[urx,ury],[lrx,lry],[llx,lly]]
@type attributes: C{dict}
@param attributes: Must match field names passed to __init__()
'''
try:
geom=GeomFromExtent(extent,self._srs)
if self._srs.IsGeographic(): #basic coordinate bounds test. Can't do for projected though
srs=osr.SpatialReference()
srs.ImportFromEPSG(4283)#4326)
valid = GeomFromExtent([-180,-90,180,90], srs=srs)
if not valid.Contains(geom):
#raise ValueError, 'Invalid extent coordinates'
warnings.warn('Invalid extent coordinates')
lyr=self._shape.GetLayer(0)
feat = ogr.Feature(lyr.GetLayerDefn())
for a in attributes:
#if a in self.fields:feat.SetField(a, attributes[a])
if a in self.fields:feat.SetField(self.fields[a], utilities.encode(attributes[a]))
feat.SetGeometryDirectly(geom)
lyr.CreateFeature(feat)
lyr.SyncToDisk()
except Exception, err:
self.__error__(err)
def Open(f):
''' Open an image with the appropriate driver.
@type f: C{str}
@param f: a filepath to the dataset to open.
@rtype: C{formats.Dataset}
@return: L{formats.Dataset<formats.__dataset__.Dataset>} object
@todo: perhaps log the entire error stack if a file couldn't be opened?
'''
errors=[] #error stack
f=utilities.encode(f) #Issue 20
#f=utilities.normcase(utilities.uncpath(utilities.realpath(f)))
#Try custom formats
for lib in __formats__:
fr='|'.join(__formats__[lib].format_regex)
rx=_re.compile(fr, _re.IGNORECASE)
if rx.search(f):
try:
ds=__formats__[lib].Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception,err:
if debug:
errinfo=utilities.FormatTraceback(_sys.exc_info()[2],10)
errargs=[arg for arg in err.args]
errargs.append(errinfo)
err.args=tuple(errargs)
errors.append(err)
def UpdateRecord(self, extent, attributes, where_clause):
'''Update record/s
@type where_clause: C{str}
@param where_clause: Shapefile supported SQL where clause
@type attributes: C{dict}
@param attributes: Must match field names passed to __init__()
'''
try:
geom = GeomFromExtent(extent, self._srs)
if self._srs.IsGeographic(
): #basic coordinate bounds test. Can't do for projected though
srs = osr.SpatialReference()
srs.ImportFromEPSG(4283) #4326)
valid = GeomFromExtent([-180, -90, 180, 90], srs=srs)
if not valid.Contains(geom):
#raise ValueError, 'Invalid extent coordinates'
warnings.warn('Invalid extent coordinates')
lyr = self._shape.GetLayer()
lyr.SetAttributeFilter(where_clause)
feat = lyr.GetNextFeature()
try:
feat.SetGeometryDirectly(geom)
except:
return self.WriteRecord(extent, attributes)
while feat:
for a in attributes:
if a in self.fields:
feat.SetField(self.fields[a],
utilities.encode(attributes[a]))
lyr.SetFeature(feat)
feat = lyr.GetNextFeature()
lyr.SyncToDisk()
except Exception, err:
self.__error__(err)
def Open(f):
''' Open an image with the appropriate driver.
@type f: C{str}
@param f: a filepath to the dataset to open.
@rtype: C{formats.Dataset}
@return: L{formats.Dataset<formats.__dataset__.Dataset>} object
@todo: perhaps log the entire error stack if a file couldn't be opened?
'''
errors = [] #error stack
f = utilities.encode(f) #Issue 20
#f=utilities.normcase(utilities.uncpath(utilities.realpath(f)))
#Try custom formats
for lib in __formats__:
fr = '|'.join(__formats__[lib].format_regex)
rx = _re.compile(fr, _re.IGNORECASE)
if rx.search(f):
try:
ds = __formats__[lib].Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception, err:
if debug:
errinfo = utilities.FormatTraceback(_sys.exc_info()[2], 10)
errargs = [arg for arg in err.args]
errargs.append(errinfo)
err.args = tuple(errargs)
errors.append(err)
def WriteRecord(self, extent, attributes):
'''Write record
@type extent: C{list}
@param extent: [[ulx,uly],[urx,ury],[lrx,lry],[llx,lly]]
@type attributes: C{dict}
@param attributes: Must match field names passed to __init__()
'''
try:
geom = GeomFromExtent(extent, self._srs)
if self._srs.IsGeographic(
): #basic coordinate bounds test. Can't do for projected though
srs = osr.SpatialReference()
srs.ImportFromEPSG(4283) #4326)
valid = GeomFromExtent([-180, -90, 180, 90], srs=srs)
if not valid.Contains(geom):
#raise ValueError, 'Invalid extent coordinates'
warnings.warn('Invalid extent coordinates')
lyr = self._shape.GetLayer(0)
feat = ogr.Feature(lyr.GetLayerDefn())
for a in attributes:
#if a in self.fields:feat.SetField(a, attributes[a])
if a in self.fields:
feat.SetField(self.fields[a],
utilities.encode(attributes[a]))
feat.SetGeometryDirectly(geom)
lyr.CreateFeature(feat)
lyr.SyncToDisk()
except Exception, err:
self.__error__(err)
def cmd(self, root, label, dir, var):
ad = tkFileDialog.askdirectory(parent=root,
initialdir=dir.get(),
title=label)
if ad:
ad = utilities.encode(os.path.normpath(ad))
dir.set(ad)
var.set(ad)
def Open(f):
''' Open an image with the appropriate driver.
@type f: C{str}
@param f: a filepath to the dataset to open.
@rtype: C{formats.Dataset}
@return: L{formats.Dataset<formats.__dataset__.Dataset>} object
@todo: perhaps log the entire error stack if a file couldn't be opened?
'''
errors=[] #error stack
f=utilities.encode(f) #Issue 20
#f=utilities.normcase(utilities.uncpath(utilities.realpath(f)))
#Try custom formats
for lib in __formats__:
fr='|'.join(__formats__[lib].format_regex)
rx=_re.compile(fr, _re.IGNORECASE)
if rx.search(f):
try:
ds=__formats__[lib].Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception as err:
if debug:
errinfo=utilities.FormatTraceback(_sys.exc_info()[2],10)
errargs=[arg for arg in err.args]
errargs.append(errinfo)
err.args=tuple(errargs)
errors.append(err)
#Try default formats
try:
fr='|'.join(__default__.format_regex)
rx=_re.compile(fr, _re.IGNORECASE)
if rx.search(f):
ds=__default__.Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception as err:
if debug:
errinfo=utilities.FormatTraceback(_sys.exc_info()[2],10)
errargs=[arg for arg in err.args]
errargs.append(errinfo)
err.args=tuple(errargs)
errors.append(err)
#Couldn't open file, raise the last error in the stack
#if len(errors) > 0: raise errors[-1].__class__,'\n'.join(errors[-1].args) #Updated for Python 2.6, not all args are strings...
if len(errors) > 0: raise errors[-1].__class__(*errors[-1].args)
else:raise Exception('Unable to open %s' % f)
'''@todo: perhaps log the entire error stack if a file couldn't be opened?'''
def Open(f):
''' Open an image with the appropriate driver.
@type f: C{str}
@param f: a filepath to the dataset to open.
@rtype: C{formats.Dataset}
@return: L{formats.Dataset<formats.__dataset__.Dataset>} object
@todo: perhaps log the entire error stack if a file couldn't be opened?
'''
errors = [] #error stack
f = utilities.encode(f) #Issue 20
#f=utilities.normcase(utilities.uncpath(utilities.realpath(f)))
#Try custom formats
for lib in __formats__:
fr = '|'.join(__formats__[lib].format_regex)
rx = _re.compile(fr, _re.IGNORECASE)
if rx.search(f):
try:
ds = __formats__[lib].Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception as err:
if debug:
errinfo = utilities.FormatTraceback(_sys.exc_info()[2], 10)
errargs = [arg for arg in err.args]
errargs.append(errinfo)
err.args = tuple(errargs)
errors.append(err)
#Try default formats
try:
fr = '|'.join(__default__.format_regex)
rx = _re.compile(fr, _re.IGNORECASE)
if rx.search(f):
ds = __default__.Dataset(f)
return ds
except NotImplementedError:
pass #Used when a format driver can't open a file, but doesn't want to raise an error
except Exception as err:
if debug:
errinfo = utilities.FormatTraceback(_sys.exc_info()[2], 10)
errargs = [arg for arg in err.args]
errargs.append(errinfo)
err.args = tuple(errargs)
errors.append(err)
#Couldn't open file, raise the last error in the stack
#if len(errors) > 0: raise errors[-1].__class__,'\n'.join(errors[-1].args) #Updated for Python 2.6, not all args are strings...
if len(errors) > 0: raise errors[-1].__class__(*errors[-1].args)
else: raise Exception('Unable to open %s' % f)
'''@todo: perhaps log the entire error stack if a file couldn't be opened?'''
def __getmetadata__(self, f=None):
'''Read Metadata for a DIMAP image as GDAL doesn't quite get it all...'''
if not f: f = self.fileinfo['filepath']
#dom=etree.parse(f) #Takes tooo long to parse the whole file, so just read as far as we need...
strxml = ''
for line in open(f, 'r'):
if line.upper().strip() == '<DATA_STRIP>': break
else: strxml += line
if not '</Dimap_Document>' in strxml: strxml += '</Dimap_Document>'
dom = etree.fromstring(strxml)
self.metadata['sceneid'] = dom.xpath(
'string(/Dimap_Document/Dataset_Id/DATASET_NAME)')
bands = dom.xpath(
'/Dimap_Document/Spectral_Band_Info/BAND_DESCRIPTION')
self.metadata['bands'] = ','.join(
[band.xpath('string(.)') for band in bands])
try:
__default__.Dataset.__getmetadata__(
self, f) #autopopulate basic metadata
except geometry.GDALError, err: #Work around reading images with lowercase filenames when
#the DATA_FILE_PATH is uppercase
# - eg samba filesystems which get converted to lowercase
dfp = dom.xpath(
'/Dimap_Document/Data_Access/Data_File/DATA_FILE_PATH')[0]
fn = utilities.encode(
dfp.xpath('string(@href)'
)) #XML is unicode, gdal.Open doesn't like unicode
if not os.path.dirname(fn):
fn = os.path.join(os.path.dirname(f), fn)
exists, img = utilities.exists(fn, True)
if exists and not os.path.exists(fn):
import tempfile
tmpfd, tmpfn = tempfile.mkstemp(suffix='.dim',
prefix='metadata')
dfp.set('href', img)
tmpfo = os.fdopen(tmpfd, 'w')
tmpfo.write(etree.tostring(dom))
tmpfo.flush()
tmpfo.close()
cwd = os.path.abspath(os.curdir)
tmp = os.path.split(tmpfn)
os.chdir(
tmp[0]
) #CD to the tmp dir so __default__.Dataset.__getmetadata__ doesn't
__default__.Dataset.__getmetadata__(self, tmp[1])
gdalmd = self._gdaldataset.GetMetadata()
self._gdaldataset = geometry.OpenDataset(img)
self._gdaldataset.SetMetadata(gdalmd)
os.unlink(tmpfn)
os.chdir(cwd)
else:
raise
def __init__(self, dir, recurse=True, archive=False):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
'''
format_regex = formats.format_regex
dir = utilities.uncpath(
utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats = {}
for f in utilities.rglob(dir,
'|'.join(format_regex),
True,
re.IGNORECASE,
recurse=recurse,
archive=archive):
#Use utf-8 encoding to fix Issue 20
if f[:4] == '/vsi': f = utilities.encode(f)
else:
f = utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r, os.path.basename(f), re.IGNORECASE):
if fileformats.has_key(r): fileformats[r].append(f)
else: fileformats[r] = [f]
break
files = []
for r in format_regex:
if fileformats.has_key(r): files.extend(fileformats[r])
#Class vars
self.errors = [
] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
self.files = files
self.file = ''
self.filecount = len(self.files)
def v1(self, f=None):
dom = self._dom
self.metadata['sceneid'] = dom.xpath(
'string(/Dimap_Document/Dataset_Id/DATASET_NAME)')
bands = dom.xpath(
'/Dimap_Document/Spectral_Band_Info/BAND_DESCRIPTION')
self.metadata['bands'] = ','.join(
[band.xpath('string(.)') for band in bands])
try:
__default__.Dataset.__getmetadata__(
self, f) #autopopulate basic metadata
except geometry.GDALError, err: #Work around reading images with lowercase filenames when
#the DATA_FILE_PATH is uppercase
# - eg samba filesystems which get converted to lowercase
dfp = dom.xpath(
'/Dimap_Document/Data_Access/Data_File/DATA_FILE_PATH')[0]
fn = utilities.encode(
dfp.xpath('string(@href)'
)) #XML is unicode, gdal.Open doesn't like unicode
if not os.path.dirname(fn):
fn = os.path.join(os.path.dirname(f), fn)
exists, img = utilities.exists(fn, True)
if exists and not os.path.exists(fn):
import tempfile
tmpfd, tmpfn = tempfile.mkstemp(suffix='.dim',
prefix='metadata')
dfp.set('href', img)
tmpfo = os.fdopen(tmpfd, 'w')
tmpfo.write(etree.tostring(dom))
tmpfo.flush()
tmpfo.close()
cwd = os.path.abspath(os.curdir)
tmp = os.path.split(tmpfn)
os.chdir(
tmp[0]
) #CD to the tmp dir so __default__.Dataset.__getmetadata__ doesn't
__default__.Dataset.__getmetadata__(self, tmp[1])
gdalmd = self._gdaldataset.GetMetadata()
self._gdaldataset = geometry.OpenDataset(img)
self._gdaldataset.SetMetadata(gdalmd)
os.unlink(tmpfn)
os.chdir(cwd)
else:
raise
def getoverview(self, outfile=None, width=800, format='JPG'):
'''
Generate overviews for DIMAP imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
@todo:
- Should we do something with the band display order metadata?
<Band_Display_Order>
<RED_CHANNEL>1</RED_CHANNEL>
<GREEN_CHANNEL>2</GREEN_CHANNEL>
<BLUE_CHANNEL>3</BLUE_CHANNEL>
</Band_Display_Order>
'''
from metageta import overviews
try:
#First check for a browse graphic, no point re-inventing the wheel...
f = self.fileinfo['filepath']
#if self.dimap_version[0]==1:
# fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
#else:
# fp=self._dom.xpath('/Dimap_Document/Dataset_Identification/DATASET_QL_PATH')[0]
fp = self._dom.xpath('string(//*/DATASET_QL_PATH/@href)')
fn = utilities.encode(
fp) #XML is unicode, gdal.Open doesn't like unicode
browse = os.path.join(os.path.dirname(f), fn)
if os.path.exists(
browse) and gdal.Open(browse).RasterXSize >= width:
return overviews.resize(browse, outfile, width)
except:
pass
return __default__.Dataset.getoverview(self, outfile, width,
format) #Do it the slow way...
def __getmetadata__(self,f=None):
'''Read Metadata for a DIMAP image as GDAL doesn't quite get it all...'''
if not f:f=self.fileinfo['filepath']
#dom=etree.parse(f) #Takes tooo long to parse the whole file, so just read as far as we need...
strxml=''
for line in open(f, 'r'):
if line.upper().strip()=='<DATA_STRIP>':break
else: strxml+=line
if not '</Dimap_Document>' in strxml:strxml+='</Dimap_Document>'
dom=etree.fromstring(strxml)
self.metadata['sceneid'] = dom.xpath('string(/Dimap_Document/Dataset_Id/DATASET_NAME)')
bands=dom.xpath('/Dimap_Document/Spectral_Band_Info/BAND_DESCRIPTION')
self.metadata['bands']=','.join([band.xpath('string(.)') for band in bands])
try:__default__.Dataset.__getmetadata__(self, f) #autopopulate basic metadata
except geometry.GDALError,err: #Work around reading images with lowercase filenames when
#the DATA_FILE_PATH is uppercase
# - eg samba filesystems which get converted to lowercase
dfp=dom.xpath('/Dimap_Document/Data_Access/Data_File/DATA_FILE_PATH')[0]
fn=utilities.encode(dfp.xpath('string(@href)')) #XML is unicode, gdal.Open doesn't like unicode
if not os.path.dirname(fn):fn=os.path.join(os.path.dirname(f),fn)
exists,img=utilities.exists(fn,True)
if exists and not os.path.exists(fn):
import tempfile
tmpfd,tmpfn=tempfile.mkstemp(suffix='.dim',prefix='metadata')
dfp.set('href',img)
tmpfo=os.fdopen(tmpfd,'w')
tmpfo.write(etree.tostring(dom))
tmpfo.flush()
tmpfo.close()
cwd=os.path.abspath(os.curdir)
tmp=os.path.split(tmpfn)
os.chdir(tmp[0]) #CD to the tmp dir so __default__.Dataset.__getmetadata__ doesn't
__default__.Dataset.__getmetadata__(self, tmp[1])
gdalmd=self._gdaldataset.GetMetadata()
self._gdaldataset=geometry.OpenDataset(img)
self._gdaldataset.SetMetadata(gdalmd)
os.unlink(tmpfn)
os.chdir(cwd)
else:raise
def getoverview(ds,outfile,width,format,bands,stretch_type,*stretch_args):
'''
Generate overviews for imagery
@type ds: C{GDALDataset}
@param ds: a GDALDataset object
@type outfile: C{str}
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: C{int}
@param width: output image width
@type format: C{str}
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@type bands: C{list}
@param bands: list of band numbers (base 1) in processing order - e.g [3,2,1]
@type stretch_type: C{str}
@param stretch_type: stretch to apply to overview image,
one of [L{NONE<_stretch_NONE>},L{PERCENT<_stretch_PERCENT>},L{MINMAX<_stretch_MINMAX>},L{STDDEV<_stretch_STDDEV>},L{COLOURTABLE<_stretch_COLOURTABLE>},L{COLOURTABLELUT<_stretch_COLOURTABLELUT>},L{RANDOM<_stretch_RANDOM>},L{UNIQUE<_stretch_UNIQUE>}].
@type stretch_args: C{list}
@param stretch_args: args to pass to the stretch algorithms
@rtype: C{str}
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
#mapping table for file extension -> GDAL format code
imageformats={'JPG':'JPEG', #JPEG JFIF (.jpg)
'PNG':'PNG', #Portable Network Graphics (.png)
'GIF':'GIF', #Graphics Interchange Format (.gif)
'BMP':'BMP', #Microsoft Windows Device Independent Bitmap (.bmp)
'TIF':'GTiff' #Tagged Image File Format/GeoTIFF (.tif)
}
if outfile:
outfile=utilities.encode(outfile)
format=os.path.splitext(outfile)[1].replace('.','') #overrides "format" arg if supplied
ovdriver=gdal.GetDriverByName(imageformats.get(format.upper(), 'JPEG')) #Get format code, default to 'JPEG' if supplied format doesn't match the predefined ones...
cols=ds.RasterXSize
rows=ds.RasterYSize
vrtcols=width
vrtrows=int(math.ceil(width*float(rows)/cols))
vrtdrv=gdal.GetDriverByName('VRT')
tempvrts={} #dict with keys=file descriptor and values=[filename,GDALDataset]
#Below is a bit of a kludge to handle creating a VRT that is based on an in-memory vrt file
drv=ds.GetDriver().ShortName
desc=ds.GetDescription()
if drv == 'VRT':
if not desc or desc[0] == '<':# input path is an in-memory vrt file
vrtfd,vrtfn=tempfile.mkstemp('.vrt')
ds=vrtdrv.CreateCopy(vrtfn,ds)
tempvrts[vrtfd]=[vrtfn,ds]
#if stretch_type != 'NONE':
# tmpxml=stretch('NONE',vrtcols,vrtrows,ds,bands)
# vrtfd,vrtfn=tempfile.mkstemp('.vrt')
# ds=vrtdrv.CreateCopy(vrtfn,geometry.OpenDataset(tmpxml))
# tempvrts[vrtfd]=[vrtfn,ds]
vrtxml=stretch(stretch_type,vrtcols,vrtrows,ds,bands,*stretch_args)
vrtds=geometry.OpenDataset(vrtxml)
if outfile:
cpds=ovdriver.CreateCopy(outfile, vrtds)
if not cpds:raise geometry.GDALError, 'Unable to generate overview image.'
else:
fd,fn=tempfile.mkstemp(suffix='.'+format.lower(), prefix='getoverviewtempimage')
cpds=ovdriver.CreateCopy(fn, vrtds)
if not cpds:raise geometry.GDALError, 'Unable to generate overview image.'
outfile=os.fdopen(fd).read()
os.unlink(fn)
for vrt in tempvrts:
tempvrts[vrt][1]=None
os.close(vrt)
del tempvrts[vrt][1]
os.unlink(tempvrts[vrt][0])
return outfile
optvals,argvals = parser.parse_args()
if optvals.dir and optvals.dir[-1]=='"': #Fix C:" issue when called from Explorer context menu
optvals.dir=optvals.dir[:-1]+'\\'
logger=None
forceexit=True
#Do we need to pop up the GUI?
if not optvals.dir or not optvals.xlsx:
#Add existing command line args values to opt default values so they show in the gui
for opt in parser.option_list:
opt.default=vars(optvals).get(opt.dest,None)
keepalive=True
hasrun=False
validate=getargs.Command(writablecallback,xlsxarg)
if optvals.xlsx:
optvals.xlsx=utilities.checkExt(utilities.encode(optvals.xlsx), ['.xlsx'])
xlsxarg.value=optvals.xlsx
xlsxarg.callback()
while keepalive:
#Pop up the GUI
args=getargs.GetArgs(dirarg,medarg,xlsxarg,updatearg,recursearg,archivearg,ovarg,kaarg,callback=validate,title=APP,icon=ICON)
if args:#GetArgs returns None if user cancels the GUI/closes the dialog (or Tkinter can not be imported)
keepalive=args.keepalive
args.xlsx = utilities.checkExt(utilities.encode(args.xlsx), ['.xlsx'])
log=args.xlsx.replace('.xlsx','.log')
if logger and logger.logging:
logger.logfile=log
else:
logger=getlogger(log,name=APP,debug=optvals.debug)
keepalive=args.keepalive
def getoverview(ds, outfile, width, format, bands, stretch_type,
*stretch_args):
'''
Generate overviews for imagery
@type ds: C{GDALDataset}
@param ds: a GDALDataset object
@type outfile: C{str}
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: C{int}
@param width: output image width
@type format: C{str}
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@type bands: C{list}
@param bands: list of band numbers (base 1) in processing order - e.g [3,2,1]
@type stretch_type: C{str}
@param stretch_type: stretch to apply to overview image,
one of [L{NONE<_stretch_NONE>},L{PERCENT<_stretch_PERCENT>},L{MINMAX<_stretch_MINMAX>},L{STDDEV<_stretch_STDDEV>},L{COLOURTABLE<_stretch_COLOURTABLE>},L{COLOURTABLELUT<_stretch_COLOURTABLELUT>},L{RANDOM<_stretch_RANDOM>},L{UNIQUE<_stretch_UNIQUE>}].
@type stretch_args: C{list}
@param stretch_args: args to pass to the stretch algorithms
@rtype: C{str}
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
#mapping table for file extension -> GDAL format code
imageformats = {
'JPG': 'JPEG', #JPEG JFIF (.jpg)
'PNG': 'PNG', #Portable Network Graphics (.png)
'GIF': 'GIF', #Graphics Interchange Format (.gif)
'BMP': 'BMP', #Microsoft Windows Device Independent Bitmap (.bmp)
'TIF': 'GTiff' #Tagged Image File Format/GeoTIFF (.tif)
}
worldfileexts = {
'JPG': '.jgw', #JPEG JFIF (.jpg)
'PNG': '.pgw', #Portable Network Graphics (.png)
'GIF': '.gfw', #Graphics Interchange Format (.gif)
'BMP': '.bpw', #Microsoft Windows Device Independent Bitmap (.bmp)
'TIF': '.tfw' #Tagged Image File Format/GeoTIFF (.tif)
}
if outfile:
outfile = utilities.encode(outfile)
format = os.path.splitext(outfile)[1].replace(
'.', '') #overrides "format" arg if supplied
ovdriver = gdal.GetDriverByName(
imageformats.get(format.upper(), 'JPEG')
) #Get format code, default to 'JPEG' if supplied format doesn't match the predefined ones...
cols = ds.RasterXSize
rows = ds.RasterYSize
vrtcols = width
vrtrows = int(math.ceil(width * float(rows) / cols))
gt = ds.GetGeoTransform()
vrtpx = cols / float(vrtcols) * gt[1]
vrtpy = rows / float(vrtrows) * gt[5]
vrtgt = (gt[0], vrtpx, gt[2], gt[3], gt[4], vrtpy)
vrtfn = stretch(stretch_type, vrtcols, vrtrows, ds, bands, *stretch_args)
gdal.UseExceptions()
vrtds = gdal.Open(vrtfn, gdal.GA_ReadOnly)
vrtds.SetGeoTransform(vrtgt)
if outfile:
cpds = ovdriver.CreateCopy(outfile, vrtds)
wf_ext = worldfileexts.get(format.upper(), '.jgw')
open(outfile[:-4] + wf_ext, 'w').write('\n'.join([
str(vrtgt[1]),
str(vrtgt[4]),
str(vrtgt[2]),
str(vrtgt[5]),
str(vrtgt[0] + 0.5 * vrtgt[1] + 0.5 * vrtgt[2]),
str(vrtgt[3] + 0.5 * vrtgt[4] + 0.5 * vrtgt[5])
]))
if not cpds:
raise geometry.GDALError, 'Unable to generate overview image.'
else:
fn = '/vsimem/%s.%s' % (tempfile._RandomNameSequence().next(),
format.lower())
cpds = ovdriver.CreateCopy(fn, vrtds)
if not cpds:
raise geometry.GDALError, 'Unable to generate overview image.'
outfile = read_vsimem(fn)
gdal.Unlink(fn)
return outfile
def cmd(self,root,label,dir,var):
ad = tkFileDialog.askdirectory(parent=root,initialdir=dir.get(),title=label)
if ad:
ad=utilities.encode(os.path.normpath(ad))
dir.set(ad)
var.set(ad)
def getoverview(ds, outfile, width, format, bands, stretch_type, *stretch_args):
"""
Generate overviews for imagery
@type ds: C{GDALDataset}
@param ds: a GDALDataset object
@type outfile: C{str}
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: C{int}
@param width: output image width
@type format: C{str}
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@type bands: C{list}
@param bands: list of band numbers (base 1) in processing order - e.g [3,2,1]
@type stretch_type: C{str}
@param stretch_type: stretch to apply to overview image,
one of [L{NONE<_stretch_NONE>},L{PERCENT<_stretch_PERCENT>},L{MINMAX<_stretch_MINMAX>},L{STDDEV<_stretch_STDDEV>},L{COLOURTABLE<_stretch_COLOURTABLE>},L{COLOURTABLELUT<_stretch_COLOURTABLELUT>},L{RANDOM<_stretch_RANDOM>},L{UNIQUE<_stretch_UNIQUE>}].
@type stretch_args: C{list}
@param stretch_args: args to pass to the stretch algorithms
@rtype: C{str}
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
"""
# mapping table for file extension -> GDAL format code
imageformats = {
"JPG": "JPEG", # JPEG JFIF (.jpg)
"PNG": "PNG", # Portable Network Graphics (.png)
"GIF": "GIF", # Graphics Interchange Format (.gif)
"BMP": "BMP", # Microsoft Windows Device Independent Bitmap (.bmp)
"TIF": "GTiff", # Tagged Image File Format/GeoTIFF (.tif)
}
worldfileexts = {
"JPG": ".jgw", # JPEG JFIF (.jpg)
"PNG": ".pgw", # Portable Network Graphics (.png)
"GIF": ".gfw", # Graphics Interchange Format (.gif)
"BMP": ".bpw", # Microsoft Windows Device Independent Bitmap (.bmp)
"TIF": ".tfw", # Tagged Image File Format/GeoTIFF (.tif)
}
if outfile:
outfile = utilities.encode(outfile)
format = os.path.splitext(outfile)[1].replace(".", "") # overrides "format" arg if supplied
ovdriver = gdal.GetDriverByName(
imageformats.get(format.upper(), "JPEG")
) # Get format code, default to 'JPEG' if supplied format doesn't match the predefined ones...
cols = ds.RasterXSize
rows = ds.RasterYSize
vrtcols = width
vrtrows = int(math.ceil(width * float(rows) / cols))
gt = ds.GetGeoTransform()
vrtpx = cols / float(vrtcols) * gt[1]
vrtpy = rows / float(vrtrows) * gt[5]
vrtgt = (gt[0], vrtpx, gt[2], gt[3], gt[4], vrtpy)
vrtfn = stretch(stretch_type, vrtcols, vrtrows, ds, bands, *stretch_args)
gdal.UseExceptions()
vrtds = gdal.Open(vrtfn, gdal.GA_ReadOnly)
vrtds.SetGeoTransform(vrtgt)
if outfile:
cpds = ovdriver.CreateCopy(outfile, vrtds)
wf_ext = worldfileexts.get(format.upper(), ".jgw")
open(outfile[:-4] + wf_ext, "w").write(
"\n".join(
[
str(vrtgt[1]),
str(vrtgt[4]),
str(vrtgt[2]),
str(vrtgt[5]),
str(vrtgt[0] + 0.5 * vrtgt[1] + 0.5 * vrtgt[2]),
str(vrtgt[3] + 0.5 * vrtgt[4] + 0.5 * vrtgt[5]),
]
)
)
if not cpds:
raise geometry.GDALError, "Unable to generate overview image."
else:
fn = "/vsimem/%s.%s" % (tempfile._RandomNameSequence().next(), format.lower())
cpds = ovdriver.CreateCopy(fn, vrtds)
if not cpds:
raise geometry.GDALError, "Unable to generate overview image."
outfile = read_vsimem(fn)
gdal.Unlink(fn)
return outfile
def main():
APP = 'MetaGETA Crawler'
ICON = icons.app_img
description = 'Run the metadata crawler'
parser = optparse.OptionParser(description=description)
opt = parser.add_option('-d',
dest="dir",
metavar="dir",
help='The directory to crawl')
dirarg = getargs.DirArg(opt,
initialdir='',
enabled=True,
icon=icons.dir_img)
dirarg.tooltip = 'The directory to start recursively searching for raster imagery.'
opt = parser.add_option("-r",
"--recurse",
action="store_true",
dest="recurse",
default=False,
help="Search directory recursively")
recursearg = getargs.BoolArg(
opt, tooltip='Do you want to search in sub-directories?')
opt = parser.add_option("-a",
"--archive",
action="store_true",
dest="archive",
default=False,
help="Search compressed archives (tar/zip)?")
archivearg = getargs.BoolArg(
opt,
tooltip=
'Do you want to search compressed archives (tar/zip)?\nNote that this will slow the crawler down considerably.'
)
opt = parser.add_option('-m',
dest="med",
metavar="media",
help='CD/DVD ID')
medarg = getargs.StringArg(opt, enabled=False, required=False)
medarg.tooltip = 'You can enter an ID for a CD/DVD, this defaults to the disc volume label.'
opt = parser.add_option("-x",
dest="xlsx",
metavar="xlsx",
help="Output metadata spreadsheet")
xlsxarg = getargs.FileArg(opt,
filter=[('Excel 2007 Spreadsheet', '*.xlsx')],
icon=icons.xls_img,
saveas=True)
xlsxarg.tooltip = 'The Excel Spreadsheet to write the metadata to. A shapefile of extents, a logfile and overview images are also output to the same directory.'
opt = parser.add_option("-u",
"--update",
action="store_true",
dest="update",
default=False,
help="Update existing crawl results")
updatearg = getargs.BoolArg(
opt,
tooltip='Do you want to update existing crawl results?',
enabled=False)
opt = parser.add_option("-o",
"--overviews",
action="store_true",
dest="ovs",
default=False,
help="Generate overview images")
ovarg = getargs.BoolArg(opt)
ovarg.tooltip = 'Do you want to generate overview (quicklook and thumbnail) images?'
opt = parser.add_option(
'-e',
"--exclude",
dest="excludes",
metavar="excludes",
help=
'Space delimited glob style file/directory basename exclusion pattern/s i.e *.png somedir? img_[0-9][0-9].tif',
default='')
excarg = getargs.StringArg(opt, enabled=True, required=False)
excarg.tooltip = opt.help
excarg.label = 'File/directory exclusion pattern'
opt = parser.add_option("--debug",
action="store_true",
dest="debug",
default=False,
help="Turn debug output on")
opt = parser.add_option("--keep-alive",
action="store_true",
dest="keepalive",
default=False,
help="Keep this dialog box open")
kaarg = getargs.BoolArg(opt)
kaarg.tooltip = 'Do you want to keep this dialog box open after running the metadata crawl so you can run another?'
#Add a callback to the directory arg, use the Command class so we can has arguments
dirarg.callback = getargs.Command(mediacallback, dirarg, medarg)
#Add a callback to the xlsx arg, use the Command class so we can has arguments
xlsxarg.callback = getargs.Command(xlsxcallback, xlsxarg, updatearg)
#Parse existing command line args
optvals, argvals = parser.parse_args()
if optvals.dir and optvals.dir[
-1] == '"': #Fix C:" issue when called from Explorer context menu
optvals.dir = optvals.dir[:-1] + '\\'
logger = None
forceexit = True
#Do we need to pop up the GUI?
if not optvals.dir or not optvals.xlsx:
#Add existing command line args values to opt default values so they show in the gui
for opt in parser.option_list:
opt.default = vars(optvals).get(opt.dest, None)
keepalive = True
hasrun = False
validate = getargs.Command(writablecallback, xlsxarg)
if optvals.xlsx:
optvals.xlsx = utilities.checkExt(utilities.encode(optvals.xlsx),
['.xlsx'])
xlsxarg.value = optvals.xlsx
xlsxarg.callback()
while keepalive:
#Pop up the GUI
args = getargs.GetArgs(dirarg,
medarg,
xlsxarg,
updatearg,
recursearg,
archivearg,
ovarg,
excarg,
kaarg,
callback=validate,
title=APP,
icon=ICON)
if args: #GetArgs returns None if user cancels the GUI/closes the dialog (or Tkinter can not be imported)
keepalive = args.keepalive
args.xlsx = utilities.checkExt(utilities.encode(args.xlsx),
['.xlsx'])
log = args.xlsx.replace('.xlsx', '.log')
if logger and logger.logging:
logger.logfile = log
else:
logger = getlogger(log, name=APP, debug=optvals.debug)
keepalive = args.keepalive
forceexit = True
execute(args.dir, args.xlsx, logger, args.med, args.update,
args.ovs, args.recurse, args.archive, args.excludes)
forceexit = False
hasrun = True
else:
if not hasrun: parser.print_help()
keepalive = False
else: #No need for the GUI
xlsx = utilities.checkExt(utilities.encode(optvals.xlsx), ['.xlsx'])
log = xlsx.replace('.xlsx', '.log')
logger = getlogger(log, name=APP, debug=optvals.debug)
execute(optvals.dir, xlsx, logger, optvals.med, optvals.update,
optvals.ovs, optvals.recurse, optvals.archive,
optvals.excludes)
if logger:
logger.debug('Shutting down')
logger.shutdown()
del logger