Exemple #1
0
    def getoverview(self, outfile=None, width=800, format='JPG'):
        '''
        Generate overviews for Digital Globe imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
        '''
        from metageta import overviews

        #First check for a browse graphic, no point re-inventing the wheel...
        f = self.fileinfo['filepath']
        browse = os.path.splitext(f)[0] + '-browse.jpg'
        if os.path.exists(browse):

            try:
                return overviews.resize(browse, outfile, width)
            except:
                return __default__.Dataset.getoverview(
                    self, outfile, width, format)  #Try it the slow way...

        else:
            return __default__.Dataset.getoverview(
                self, outfile, width, format)  #Do it the slow way...
Exemple #2
0
    def getoverview(self,outfile=None,width=800,format='JPG'): 
        '''
        Generate overviews for ALOS imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
        '''
        from metageta import overviews

        #First check for a browse graphic, no point re-inventing the wheel...
        for f in self.filelist:
            browse=False
            if '.jpg' in f.lower():
                browse=f
                break
        if browse:
            try:return overviews.resize(browse,outfile,width)
            except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
        else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
Exemple #3
0
    def getoverview(self,outfile=None,width=800,format='JPG'): 
        '''
        Generate overviews for ALOS imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
        '''
        from metageta import overviews

        #First check for a browse graphic, no point re-inventing the wheel...
        for f in self.filelist:
            browse=False
            if '.jpg' in f.lower():
                browse=f
                break
        if browse:
            try:return overviews.resize(browse,outfile,width)
            except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...
        else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
    def getoverview(self,outfile=None,width=800,format='JPG'):
        '''
        Generate overviews for DIMAP imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)

        @todo: Should we do something with the band display order metadata?
            <Band_Display_Order>
                <RED_CHANNEL>1</RED_CHANNEL>
                <GREEN_CHANNEL>2</GREEN_CHANNEL>
                <BLUE_CHANNEL>3</BLUE_CHANNEL>
            </Band_Display_Order>
        '''
        from metageta import overviews

        #First check for a browse graphic, no point re-inventing the wheel...
        f=self.fileinfo['filepath']
        #browse=os.path.join(os.path.dirname(f),'PREVIEW.JPG')
        fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
        fn=utilities.encode(fp.xpath('string(@href)')) #XML is unicode, gdal.Open doesn't like unicode
        browse=os.path.join(os.path.dirname(f),fn)

        if os.path.exists(browse) and gdal.Open(browse).RasterXSize >= width:

            try:return overviews.resize(browse,outfile,width)
            except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...

        else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
Exemple #5
0
    def getoverview(self, outfile=None, width=800, format='JPG'):
        '''
        Generate overviews for DIMAP imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)

        @todo:
             - Should we do something with the band display order metadata?

                 <Band_Display_Order>
                 <RED_CHANNEL>1</RED_CHANNEL>
                 <GREEN_CHANNEL>2</GREEN_CHANNEL>
                 <BLUE_CHANNEL>3</BLUE_CHANNEL>
                 </Band_Display_Order>
        '''
        from metageta import overviews

        try:
            #First check for a browse graphic, no point re-inventing the wheel...
            f = self.fileinfo['filepath']
            #if self.dimap_version[0]==1:
            #    fp=self._dom.xpath('/Dimap_Document/Dataset_Id/DATASET_QL_PATH')[0]
            #else:
            #    fp=self._dom.xpath('/Dimap_Document/Dataset_Identification/DATASET_QL_PATH')[0]
            fp = self._dom.xpath('string(//*/DATASET_QL_PATH/@href)')
            fn = utilities.encode(
                fp)  #XML is unicode, gdal.Open doesn't like unicode
            browse = os.path.join(os.path.dirname(f), fn)

            if os.path.exists(
                    browse) and gdal.Open(browse).RasterXSize >= width:
                return overviews.resize(browse, outfile, width)

        except:
            pass
        return __default__.Dataset.getoverview(self, outfile, width,
                                               format)  #Do it the slow way...
    def getoverview(self,outfile=None,width=800,format='JPG'):
        '''
        Generate overviews for Digital Globe imagery

        @type  outfile: str
        @param outfile: a filepath to the output overview image. If supplied, format is determined from the file extension
        @type  width:   int
        @param width:   image width
        @type  format:  str
        @param format:  format to generate overview image, one of ['JPG','PNG','GIF','BMP','TIF']. Not required if outfile is supplied.
        @rtype:         str
        @return:        filepath (if outfile is supplied)/binary image data (if outfile is not supplied)
        '''
        from metageta import overviews

        #First check for a browse graphic, no point re-inventing the wheel...
        f=self.fileinfo['filepath']
        browse=os.path.splitext(f)[0]+'-browse.jpg'
        if os.path.exists(browse):

            try:return overviews.resize(browse,outfile,width)
            except:return __default__.Dataset.getoverview(self,outfile,width,format) #Try it the slow way...

        else: return __default__.Dataset.getoverview(self,outfile,width,format)#Do it the slow way...
Exemple #7
0
def main(dir, xlsx, logger, mediaid=None, update=False, getovs=False, recurse=False, archive=False):

    """ Run the Metadata Crawler

        @type  dir:    C{str}
        @param dir:    The directory to start the metadata crawl.
        @type  xlsx:    C{str}
        @param xlsx:    Excel spreadsheet to write metadata to
        @type  logger: C{progresslogger.ProgressLogger}
        @param logger: Use an already instantiated logger
        @type  mediaid:C{str}
        @param mediaid:CD/DVD media ID
        @type  getovs: C{boolean}
        @param getovs: Generate overview (quicklook/thumbnail) images
        @type  recurse: C{boolean}
        @param recurse: Search directory recursively?
        @type  archive: C{boolean}
        @param archive: Search compressed archives (tar/zip)?
        @return:  C{progresslogger.ProgressLogger}
    """

    shp=xlsx.replace('.xlsx','.shp')

    format_regex  = formats.format_regex
    format_fields = formats.fields

    logger.debug(' '.join(sys.argv))

    #raise Exception
    #ExcelWriter=utilities.ExcelWriter(xlsx,format_fields.keys(),update=update)
    with utilities.ExcelWriter(xlsx,format_fields.keys(),update=update) as ExcelWriter:
        try:
            #Are we updating an existing crawl?
            records={}
            if update and os.path.exists(xlsx):

                #Do we need to recreate the shapefile?
                if os.path.exists(shp):
                    ShapeWriter=False
                else:
                    logger.info('%s does not exist, it will be recreated...'%shp)
                    ShapeWriter=geometry.ShapeWriter(shp,format_fields,update=False)

                #Build a dict of existing records
                row=-1
                #with utilities.ExcelReader(xlsx) as ExcelReader: #Using a context manager ensures closure before writing
                for row,rec in enumerate(utilities.ExcelReader(xlsx)):
                    #Check if the dataset still exists, mark it DELETED if it doesn't
                    if os.path.exists(rec['filepath']) or rec['mediaid'] !='' or \
                       (rec['filepath'][0:4]=='/vsi' and utilities.compressed_file_exists(rec['filepath'],False)):
                        if ShapeWriter:
                            ext=[rec['UL'].split(','),rec['UR'].split(','),rec['LR'].split(','),rec['LL'].split(',')]
                            ShapeWriter.WriteRecord(ext,rec)
                        #Kludge to ensure backwards compatibility with previously generated guids
                        #records[rec['guid']]=rec
                        records[utilities.uuid(rec['filepath'])]=(row,rec)
                    else:
                        if rec.get('DELETED',0)not in [1,'1']:
                            rec['DELETED']=1
                            ExcelWriter.UpdateRecord(rec,row)
                            logger.info('Marked %s as deleted' % (rec['filepath']))
                if row==-1:logger.info('Output spreadsheet is empty, no records to update')
                ExcelWriter.save()
                del ShapeWriter
            ShapeWriter=geometry.ShapeWriter(shp,format_fields,update=update)

        except Exception,err:
            logger.error('%s' % utilities.ExceptionInfo())
            logger.debug(utilities.ExceptionInfo(10))
            #sys.exit(1)
            return

        logger.info('Searching for files...')
        now=time.time()
        Crawler=crawler.Crawler(dir,recurse=recurse,archive=archive)
        logger.info('Found %s files...'%Crawler.filecount)

        #Loop thru dataset objects returned by Crawler
        for ds in Crawler:
            try:
                logger.debug('Attempting to open %s'%Crawler.file)
                fi=ds.fileinfo
                fi['filepath']=utilities.uncpath(fi['filepath'])
                fi['filelist']='|'.join(utilities.uncpath(ds.filelist))
                #qlk=utilities.uncpath(os.path.join(os.path.dirname(xlsx),'%s.%s.qlk.jpg'%(fi['filename'],fi['guid'])))
                #thm=utilities.uncpath(os.path.join(os.path.dirname(xlsx),'%s.%s.thm.jpg'%(fi['filename'],fi['guid'])))
                qlk=os.path.join(os.path.dirname(xlsx),'%s.%s.qlk.jpg'%(fi['filename'],fi['guid']))
                thm=os.path.join(os.path.dirname(xlsx),'%s.%s.thm.jpg'%(fi['filename'],fi['guid']))

                if update and ds.guid in records:
                    row,rec=records[ds.guid]
                    #Issue 35: if it's not modified, but we've asked for overview images and it doesn't already have them....
                    if ismodified(rec,fi,os.path.dirname(xlsx)) or (not rec['quicklook'] and getovs):
                        md=ds.metadata
                        geom=ds.extent
                        md.update(fi)
                        logger.info('Updated metadata for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
                        try:
                            if rec['quicklook'] and os.path.exists(rec['quicklook']):getovs=False #Don't update overview
                            if getovs:
                                qlk=ds.getoverview(qlk, width=800)
                                #We don't need to regenerate it, just resize it
                                #thm=ds.getoverview(thm, width=150)
                                thm=overviews.resize(qlk,thm,width=150)
                                md['quicklook']=os.path.basename(qlk)
                                md['thumbnail']=os.path.basename(thm)
                                #md['quicklook']=utilities.uncpath(qlk)
                                #md['thumbnail']=utilities.uncpath(thm)
                                logger.info('Updated overviews for %s' % Crawler.file)
                        except Exception,err:
                            logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
                            logger.debug(utilities.ExceptionInfo(10))
                        try:
                            ExcelWriter.UpdateRecord(md,row)
                        except Exception,err:
                            logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
                            logger.debug(utilities.ExceptionInfo(10))
                        try:
                            ShapeWriter.UpdateRecord(geom,md,'guid="%s"'%rec['guid'])
                        except Exception,err:
                            logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
                            logger.debug(utilities.ExceptionInfo(10))
                    else:
                        logger.info('Metadata did not need updating for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
                        continue
Exemple #8
0
             logger.debug(utilities.ExceptionInfo(10))
     else:
         logger.info('Metadata did not need updating for %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
         continue
 else:
     md=ds.metadata
     geom=ds.extent
     md.update(fi)
     if mediaid:md.update({'mediaid':mediaid})
     logger.info('Extracted metadata from %s, %s files remaining' % (Crawler.file,len(Crawler.files)))
     try:
         if getovs:
             qlk=ds.getoverview(qlk, width=800)
             #We don't need to regenerate it, just resize it
             #thm=ds.getoverview(thm, width=150)
             thm=overviews.resize(qlk,thm,width=150)
             md['quicklook']=os.path.basename(qlk)
             md['thumbnail']=os.path.basename(thm)
             #md['quicklook']=utilities.uncpath(qlk)
             #md['thumbnail']=utilities.uncpath(thm)
             logger.info('Generated overviews from %s' % Crawler.file)
     except Exception as err:
         logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
         logger.debug(utilities.ExceptionInfo(10))
     try:
         ExcelWriter.WriteRecord(md)
     except Exception as err:
         logger.error('%s\n%s' % (Crawler.file, utilities.ExceptionInfo()))
         logger.debug(utilities.ExceptionInfo(10))
     try:
         ShapeWriter.WriteRecord(geom,md)