def getoverview(self, outfile=None, width=800, format='JPG'):
'''
Generate overviews for Digital Globe imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
f = self.fileinfo['filepath']
browse = os.path.splitext(f)[0] + '-browse.jpg'
if os.path.exists(browse):
try:
return overviews.resize(browse, outfile, width)
except:
return __default__.Dataset.getoverview(
self, outfile, width, format) #Try it the slow way...
else:
return __default__.Dataset.getoverview(
self, outfile, width, format) #Do it the slow way...
def getoverview(self,outfile=None,width=800,format='JPG'):
'''
Generate overviews for ALOS imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
for f in self.filelist:
browse=False
if '.jpg' in f.lower():
browse=f
break
if browse:
try:return overviews.resize(browse,outfile,width)
except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
def getoverview(self,outfile=None,width=800,format='JPG'):
'''
Generate overviews for ALOS imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
for f in self.filelist:
browse=False
if '.jpg' in f.lower():
browse=f
break
if browse:
try:return overviews.resize(browse,outfile,width)
except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
def getoverview(self,outfile=None,width=800,format='JPG'):
'''
Generate overviews for DIMAP imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
@todo: Should we do something with the band display order metadata?
<Band_Display_Order>
<RED_CHANNEL>1</RED_CHANNEL>
<GREEN_CHANNEL>2</GREEN_CHANNEL>
<BLUE_CHANNEL>3</BLUE_CHANNEL>
</Band_Display_Order>
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
f=self.fileinfo['filepath']
#browse=os.path.join(os.path.dirname(f),'PREVIEW.JPG')
fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
fn=utilities.encode(fp.xpath('string(@href)')) #XML is unicode, gdal.Open doesn't like unicode
browse=os.path.join(os.path.dirname(f),fn)
if os.path.exists(browse) and gdal.Open(browse).RasterXSize >= width:
try:return overviews.resize(browse,outfile,width)
except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
def getoverview(self, outfile=None, width=800, format='JPG'):
'''
Generate overviews for DIMAP imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
@todo:
- Should we do something with the band display order metadata?
<Band_Display_Order>
<RED_CHANNEL>1</RED_CHANNEL>
<GREEN_CHANNEL>2</GREEN_CHANNEL>
<BLUE_CHANNEL>3</BLUE_CHANNEL>
</Band_Display_Order>
'''
from metageta import overviews
try:
#First check for a browse graphic, no point re-inventing the wheel...
f = self.fileinfo['filepath']
#if self.dimap_version[0]==1:
# fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
#else:
# fp=self._dom.xpath('/Dimap_Document/Dataset_Identification/DATASET_QL_PATH')[0]
fp = self._dom.xpath('string(//*/DATASET_QL_PATH/@href)')
fn = utilities.encode(
fp) #XML is unicode, gdal.Open doesn't like unicode
browse = os.path.join(os.path.dirname(f), fn)
if os.path.exists(
browse) and gdal.Open(browse).RasterXSize >= width:
return overviews.resize(browse, outfile, width)
except:
pass
return __default__.Dataset.getoverview(self, outfile, width,
format) #Do it the slow way...
def getoverview(self,outfile=None,width=800,format='JPG'):
'''
Generate overviews for Digital Globe imagery
@type outfile: str
@param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
@type width: int
@param width: image width
@type format: str
@param format: format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
@rtype: str
@return: filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
'''
from metageta import overviews
#First check for a browse graphic, no point re-inventing the wheel...
f=self.fileinfo['filepath']
browse=os.path.splitext(f)[0]+'-browse.jpg'
if os.path.exists(browse):
try:return overviews.resize(browse,outfile,width)
except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
def main(dir, xlsx, logger, mediaid=None, update=False, getovs=False, recurse=False, archive=False):
""" Run the Metadata Crawler
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
@type xlsx: C{str}
@param xlsx: Excel spreadsheet to write metadata to
@type logger: C{progresslogger.ProgressLogger}
@param logger: Use an already instantiated logger
@type mediaid:C{str}
@param mediaid:CD/DVD media ID
@type getovs: C{boolean}
@param getovs: Generate overview (quicklook/thumbnail) images
@type recurse: C{boolean}
@param recurse: Search directory recursively?
@type archive: C{boolean}
@param archive: Search compressed archives (tar/zip)?
@return: C{progresslogger.ProgressLogger}
"""
shp=xlsx.replace('.xlsx','.shp')
format_regex = formats.format_regex
format_fields = formats.fields
logger.debug(' '.join(sys.argv))
#raise Exception
#ExcelWriter=utilities.ExcelWriter(xlsx,format_fields.keys(),update=update)
with utilities.ExcelWriter(xlsx,format_fields.keys(),update=update) as ExcelWriter:
try:
#Are we updating an existing crawl?
records={}
if update and os.path.exists(xlsx):
#Do we need to recreate the shapefile?
if os.path.exists(shp):
ShapeWriter=False
else:
logger.info('%s does not exist, it will be recreated...'%shp)
ShapeWriter=geometry.ShapeWriter(shp,format_fields,update=False)
#Build a dict of existing records
row=-1
#with utilities.ExcelReader(xlsx) as ExcelReader: #Using a context manager ensures closure before writing
for row,rec in enumerate(utilities.ExcelReader(xlsx)):
#Check if the dataset still exists, mark it DELETED if it doesn't
if os.path.exists(rec['filepath']) or rec['mediaid'] !='' or \
(rec['filepath'][0:4]=='/vsi' and utilities.compressed_file_exists(rec['filepath'],False)):
if ShapeWriter:
ext=[rec['UL'].split(','),rec['UR'].split(','),rec['LR'].split(','),rec['LL'].split(',')]
ShapeWriter.WriteRecord(ext,rec)
#Kludge to ensure backwards compatibility with previously generated guids
#records[rec['guid']]=rec
records[utilities.uuid(rec['filepath'])]=(row,rec)
else:
if rec.get('DELETED',0)not in [1,'1']:
rec['DELETED']=1
ExcelWriter.UpdateRecord(rec,row)
logger.info('Marked %s as deleted' % (rec['filepath']))
if row==-1:logger.info('Output spreadsheet is empty, no records to update')
ExcelWriter.save()
del ShapeWriter
ShapeWriter=geometry.ShapeWriter(shp,format_fields,update=update)
except Exception,err:
logger.error('%s' % utilities.ExceptionInfo())
logger.debug(utilities.ExceptionInfo(10))
#sys.exit(1)
return
logger.info('Searching for files...')
now=time.time()
Crawler=crawler.Crawler(dir,recurse=recurse,archive=archive)
logger.info('Found %s files...'%Crawler.filecount)
#Loop thru dataset objects returned by Crawler
for ds in Crawler:
try:
logger.debug('Attempting to open %s'%Crawler.file)
fi=ds.fileinfo
fi['filepath']=utilities.uncpath(fi['filepath'])
fi['filelist']='|'.join(utilities.uncpath(ds.filelist))
#qlk=utilities.uncpath(os.path.join(os.path.dirname(xlsx),'%s.%s.qlk.jpg'%(fi['filename'],fi['guid'])))
#thm=utilities.uncpath(os.path.join(os.path.dirname(xlsx),'%s.%s.thm.jpg'%(fi['filename'],fi['guid'])))
qlk=os.path.join(os.path.dirname(xlsx),'%s.%s.qlk.jpg'%(fi['filename'],fi['guid']))
thm=os.path.join(os.path.dirname(xlsx),'%s.%s.thm.jpg'%(fi['filename'],fi['guid']))
if update and ds.guid in records:
row,rec=records[ds.guid]
#Issue 35: if it's not modified, but we've asked for overview images and it doesn't already have them....
if ismodified(rec,fi,os.path.dirname(xlsx)) or (not rec['quicklook'] and getovs):
md=ds.metadata
geom=ds.extent
md.update(fi)
logger.info('Updated metadata for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
try:
if rec['quicklook'] and os.path.exists(rec['quicklook']):getovs=False #Don't update overview
if getovs:
qlk=ds.getoverview(qlk, width=800)
#We don't need to regenerate it, just resize it
#thm=ds.getoverview(thm, width=150)
thm=overviews.resize(qlk,thm,width=150)
md['quicklook']=os.path.basename(qlk)
md['thumbnail']=os.path.basename(thm)
#md['quicklook']=utilities.uncpath(qlk)
#md['thumbnail']=utilities.uncpath(thm)
logger.info('Updated overviews for %s' % Crawler.file)
except Exception,err:
logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
logger.debug(utilities.ExceptionInfo(10))
try:
ExcelWriter.UpdateRecord(md,row)
except Exception,err:
logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
logger.debug(utilities.ExceptionInfo(10))
try:
ShapeWriter.UpdateRecord(geom,md,'guid="%s"'%rec['guid'])
except Exception,err:
logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
logger.debug(utilities.ExceptionInfo(10))
else:
logger.info('Metadata did not need updating for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
continue
logger.debug(utilities.ExceptionInfo(10))
else:
logger.info('Metadata did not need updating for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
continue
else:
md=ds.metadata
geom=ds.extent
md.update(fi)
if mediaid:md.update({'mediaid':mediaid})
logger.info('Extracted metadata from %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
try:
if getovs:
qlk=ds.getoverview(qlk, width=800)
#We don't need to regenerate it, just resize it
#thm=ds.getoverview(thm, width=150)
thm=overviews.resize(qlk,thm,width=150)
md['quicklook']=os.path.basename(qlk)
md['thumbnail']=os.path.basename(thm)
#md['quicklook']=utilities.uncpath(qlk)
#md['thumbnail']=utilities.uncpath(thm)
logger.info('Generated overviews from %s' % Crawler.file)
except Exception as err:
logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
logger.debug(utilities.ExceptionInfo(10))
try:
ExcelWriter.WriteRecord(md)
except Exception as err:
logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
logger.debug(utilities.ExceptionInfo(10))
try:
ShapeWriter.WriteRecord(geom,md)
