def __init__(self, f=None): '''Open the dataset''' if not f: f = self.fileinfo['filepath'] self.filelist = [r for r in utilities.rglob(os.path.dirname(f)) ] #everything in this dir and below. led = glob.glob(os.path.dirname(f) + '/[Ll][Ee][Aa][Dd]*')[0] #volume file meta = open(led, 'rb').read() ''' metadata has 4 records, each is 4320 (LS) or 6120 (SPOT) bytes long: File descriptor record; Scene header record; Map projection (scene-related) ancillary record; Radiometric transformation ancillary record. ''' #Record 2 - Scene header record record = 2 recordlength = 4320 #LS 5 satellite = utilities.readbinary(meta, (record - 1) * recordlength, 309, 324) if not satellite[0:7] == 'LANDSAT': raise NotImplementedError #This error gets ignored in __init__.Open()
def __init__(self, f): if not f: f = self.fileinfo['filepath'] if f[:4] == '/vsi': raise NotImplementedError d = os.path.dirname(f) if open(f).read(1024).strip()[0] == '<': #HTML file, ignore it. raise NotImplementedError if 'HRF' in f.upper(): self._filetype = 'HRF' #rex='BAND[1-57]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' #not picking up the ACRES .ers files rex = 'BAND[1-57].*|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' elif 'HTM' in f.upper(): self._filetype = 'HTM' #rex='BAND6[LH]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' #not picking up the ACRES .ers files rex = 'BAND6[LH].*|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' elif 'HPN' in f.upper(): self._filetype = 'HPN' #rex='BAND8\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' #not picking up the ACRES .ers files rex = 'BAND8.*|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' filelist = [f] #header filelist.extend([ f for f in utilities.rglob( d, rex, regex=True, regex_flags=re.I, recurse=False) ]) #bands self.filelist = filelist #"self.filelist" is a property, not a list, we can only 'get' or 'set' it.
def __init__(self, f): if f[:4] == '/vsi': raise NotImplementedError dirname, basename = os.path.split(f) rootname, ext = os.path.splitext(basename) self.filelist = [ f for f in utilities.rglob(dirname, pattern=rootname + ".*", regex=True, regex_flags=re.I, recurse=False) ] self.exts = { '.tif': 'GTiff/GeoTIFF', '.img': 'HFA/Erdas Imagine Images (.img)', '.ntf': 'NITF/National Imagery Transmission Format (.ntf)', '.pix': 'PCI Geomatics Database File (.pix)' } self.img = False for ext in self.exts: imgs = [i for i in self.filelist if i[-4:] in [ext, ext.upper()]] if imgs: self.img = imgs[0] break if not self.img: raise NotImplementedError, 'Matching DigitalGlobe imagery file not found:\n'
def __init__(self,dir, recurse=True, archive=False): ''' Iterator for metadata crawling @type dir: C{str} @param dir: The directory to start the metadata crawl. ''' format_regex = formats.format_regex dir=utilities.uncpath(utilities.realpath(utilities.normcase(utilities.encode(dir)))) #Build a dict of matching files and regexes then sort according to the priority of the regex formats fileformats={} for f in utilities.rglob(dir,'|'.join(format_regex), True, re.IGNORECASE, recurse=recurse, archive=archive): #Don't return existing overviews if f[-7:] in ('qlk.jpg','thm.jpg'):continue #Use utf-8 encoding to fix Issue 20 if f[:4]=='/vsi':f=utilities.encode(f) else:f=utilities.realpath(utilities.normcase(utilities.encode(f))) for r in format_regex: #This is so we always return _default_ format datasets last. if re.search(r,os.path.basename(f),re.IGNORECASE): if fileformats.has_key(r):fileformats[r].append(f) else:fileformats[r]=[f] break files=[] for r in format_regex: if fileformats.has_key(r):files.extend(fileformats[r]) #Class vars self.errors=[] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info self.files=files self.file='' self.filecount=len(self.files)
def CreateMEF(outdir, xmlfile, uid, overviews=[], cat=categories['default'], ops=operations['default']): '''Generate Geonetwork "Metadata Exchange Format" from an ISO19139 XML record @see: U{http://www.fao.org/geonetwork/docs/ch17s02.html} U{http://trac.osgeo.org/geonetwork/wiki/MEF} @param outdir: Directory to write MEF file to. @param xmlfile: XML file to create MEF from. @param uid: ID of metadata record (UUID/GUID string). @keyword overviews: List of overview image file (e.g quicklooks & thumbnails) OPTIONAL. @keyword cat: List of GeoNetwork categories to include in the MEF OPTIONAL. @todo: Assumes metadata is ISO19139, need to make generic somehow... ''' xmldir = _path.dirname(xmlfile) curdir = _path.abspath(_os.curdir) #mefdir=_path.join(_os.environ['TEMP'],_path.basename(_path.splitext(xmlfile)[0])) mefdir = mefdir = _tmp.mkdtemp() mefpath = '%s.mef' % (_path.join( outdir, _path.basename(_path.splitext(xmlfile)[0]))) try: # if _path.exists(mefpath): _os.remove(mefpath) #if _path.exists(mefdir):_sh.rmtree(mefdir) mef = _zip.ZipFile(mefpath, 'w', _zip.ZIP_DEFLATED) #_os.mkdir(mefdir) _os.chdir(mefdir) _sh.copy(xmlfile, 'metadata.xml') if overviews: _os.mkdir('public') for f in overviews: _sh.copy(f, _path.join('public', _path.basename(f))) _CreateInfo(uid, overviews, cat, ops) #_sh.copy(xmlfile,'metadata.xml') for f in _utilities.rglob('.'): if not _path.isdir(f): mef.write(f) finally: try: _os.chdir(curdir) except: pass try: mef.close() del mef except: pass try: _sh.rmtree(mefdir) except: pass
def __init__(self,f=None): if not f:f=self.fileinfo['filepath'] self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] self._led=f try:self._vol=glob.glob(os.path.dirname(f) + '/[Vv][Oo][Ll]*')[0] #volume file except:self._vol=False img_regex=[ r'IMG-0[1-4]-ALAV.*_U$', #ALOS AVNIR-2 img file r'.*\.tif$', #ALOS AVNIR-2 ACRES orthocorrected tif file r'IMG-[HV][HV]-ALPSR.*UD$', #ALOS PALSAR r'IMG-ALPSM.*\_U[BFNW]$' #ALOS PRISM ] self._imgs=[i for i in utilities.rglob(os.path.dirname(f),'|'.join(img_regex),True,re.I, False)] self.fileinfo['filepath']=self._led #change filename to leader file self.fileinfo['filename']=os.path.basename(self._led)
def __init__(self,f=None): if not f:f=self.fileinfo['filepath'] if f[:4]=='/vsi':raise NotImplementedError self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] self._led=f try:self._vol=glob.glob(os.path.dirname(f) + '/[Vv][Oo][Ll]*')[0] #volume file except:self._vol=False img_regex=[ r'IMG-0[1-4]-ALAV.*_U$', #ALOS AVNIR-2 img file r'.*\.tif$', #ALOS AVNIR-2 ACRES orthocorrected tif file r'IMG-[HV][HV]-ALPSR.*UD$', #ALOS PALSAR r'IMG-ALPSM.*\_U[BFNW]$' #ALOS PRISM ] self._imgs=[i for i in utilities.rglob(os.path.dirname(f),'|'.join(img_regex),True,re.I, False)] self.fileinfo['filepath']=self._led #change filename to leader file self.fileinfo['filename']=os.path.basename(self._led)
def CreateMEF(outdir, xmlfile, uid, overviews=[], cat=categories["default"], ops=operations["default"]): """Generate Geonetwork "Metadata Exchange Format" from an ISO19139 XML record @see: U{http://www.fao.org/geonetwork/docs/ch17s02.html} U{http://trac.osgeo.org/geonetwork/wiki/MEF} @param outdir: Directory to write MEF file to. @param xmlfile: XML file to create MEF from. @param uid: ID of metadata record (UUID/GUID string). @keyword overviews: List of overview image file (e.g quicklooks & thumbnails) OPTIONAL. @keyword cat: List of GeoNetwork categories to include in the MEF OPTIONAL. @todo: Assumes metadata is ISO19139, need to make generic somehow... """ xmldir = _path.dirname(xmlfile) curdir = _path.abspath(_os.curdir) # mefdir=_path.join(_os.environ['TEMP'],_path.basename(_path.splitext(xmlfile)[0])) mefdir = mefdir = _tmp.mkdtemp() mefpath = "%s.mef" % (_path.join(outdir, _path.basename(_path.splitext(xmlfile)[0]))) try: # if _path.exists(mefpath): _os.remove(mefpath) # if _path.exists(mefdir):_sh.rmtree(mefdir) mef = _zip.ZipFile(mefpath, "w", _zip.ZIP_DEFLATED) # _os.mkdir(mefdir) _os.chdir(mefdir) _sh.copy(xmlfile, "metadata.xml") if overviews: _os.mkdir("public") for f in overviews: _sh.copy(f, _path.join("public", _path.basename(f))) _CreateInfo(uid, overviews, cat, ops) # _sh.copy(xmlfile,'metadata.xml') for f in _utilities.rglob("."): if not _path.isdir(f): mef.write(f) finally: try: _os.chdir(curdir) except: pass try: mef.close() del mef except: pass try: _sh.rmtree(mefdir) except: pass
def __init__(self, f=None): if not f: f = self.fileinfo["filepath"] if f[:4] == "/vsi": raise NotImplementedError self.filelist = [r for r in utilities.rglob(os.path.dirname(f))] self._led = f try: self._vol = glob.glob(os.path.dirname(f) + "/[Vv][Oo][Ll]*")[0] # volume file except: self._vol = False img_regex = [ r"IMG-0[1-4]-ALAV.*_U$", # ALOS AVNIR-2 img file r".*\.tif$", # ALOS AVNIR-2 ACRES orthocorrected tif file r"IMG-[HV][HV]-ALPSR.*UD$", # ALOS PALSAR r"IMG-ALPSM.*\_U[BFNW]$", # ALOS PRISM ] self._imgs = [i for i in utilities.rglob(os.path.dirname(f), "|".join(img_regex), True, re.I, False)] self.fileinfo["filepath"] = self._led # change filename to leader file self.fileinfo["filename"] = os.path.basename(self._led)
def __init__(self,f=None): '''Open the dataset''' if not f:f=self.fileinfo['filepath'] led=glob.glob(os.path.dirname(f) + '/[Ll][Ee][Aa][Dd]*')[0] #volume file meta = open(led,'rb').read() #Record 2 - Scene header record record=2 recordlength=3960 #SPOT recordlength=3960 satellite=utilities.readbinary(meta,(record-1)*recordlength,613,628) if not satellite[0:4] == 'SPOT': raise NotImplementedError #This error gets ignored in __init__.Open() self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] #everything in this dir and below.
def __init__(self, dir, recurse=True, archive=False, excludes=[]): ''' Iterator for metadata crawling @type dir: C{str} @param dir: The directory to start the metadata crawl. @type recurse: C{bool} @param recurse: Recurse into subdirectories? @type archive: C{bool} @param archive: Look in zip/gzip archives @type excludes: C{list} @param excludes: List of glob style file/directory exclusion pattern/s ''' #Class vars self.errors = [ ] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info format_regex = formats.format_regex dir = utilities.uncpath( utilities.realpath(utilities.normcase(utilities.encode(dir)))) #Build a dict of matching files and regexes then sort according to the priority of the regex formats fileformats = {} for f in utilities.rglob(dir, '|'.join(format_regex), True, re.IGNORECASE, recurse=recurse, archive=archive, excludes=excludes, onerror=self.onerror, followlinks=False): #Don't return existing overviews if f[-7:] in ('qlk.jpg', 'thm.jpg'): continue #Use utf-8 encoding to fix Issue 20 if f[:4] == '/vsi': f = utilities.encode(f) else: f = utilities.realpath(utilities.normcase(utilities.encode(f))) for r in format_regex: #This is so we always return _default_ format datasets last. if re.search(r, os.path.basename(f), re.IGNORECASE): if fileformats.has_key(r): fileformats[r].append(f) else: fileformats[r] = [f] break files = [] for r in format_regex: if fileformats.has_key(r): files.extend(fileformats[r]) #Class vars self.files = files self.file = '' self.filecount = len(self.files)
def __init__(self,f): if f[:4]=='/vsi':raise NotImplementedError dirname,basename=os.path.split(f) rootname,ext=os.path.splitext(basename) self.filelist=[f for f in utilities.rglob(dirname, pattern=rootname+".*", regex=True, regex_flags=re.I, recurse=False)] self.exts={'.tif':'GTiff/GeoTIFF', '.img':'HFA/Erdas Imagine Images (.img)', '.ntf':'NITF/National Imagery Transmission Format (.ntf)', '.pix':'PCI Geomatics Database File (.pix)'} self.img=False for ext in self.exts: imgs=[i for i in self.filelist if i[-4:] in [ext,ext.upper()]] if imgs: self.img=imgs[0] break if not self.img:raise NotImplementedError, 'Matching DigitalGlobe imagery file not found:\n'
def __init__(self,f): if not f:f=self.fileinfo['filepath'] d=os.path.dirname(f) if open(f).read(1024).strip()[0]=='<':#HTML file, ignore it. raise NotImplementedError if 'HRF' in f.upper(): self._filetype='HRF' #rex='BAND[1-57]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' #not picking up the ACRES .ers files rex='BAND[1-57].*|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' elif 'HTM' in f.upper(): self._filetype='HTM' #rex='BAND6[LH]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' #not picking up the ACRES .ers files rex='BAND6[LH].*|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' elif 'HPN' in f.upper(): self._filetype='HPN' #rex='BAND8\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' #not picking up the ACRES .ers files rex='BAND8.*|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' filelist=[f] #header filelist.extend([f for f in utilities.rglob(d,rex,regex=True, regex_flags=re.I, recurse=False)]) #bands self.filelist=filelist #"self.filelist" is a property, not a list, we can only 'get' or 'set' it.
def __init__(self,f=None): '''Open the dataset''' if not f:f=self.fileinfo['filepath'] self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] #everything in this dir and below. led=glob.glob(os.path.dirname(f) + '/[Ll][Ee][Aa][Dd]*')[0] #volume file meta = open(led,'rb').read() ''' metadata has 4 records, each is 4320 (LS) or 6120 (SPOT) bytes long: File descriptor record; Scene header record; Map projection (scene-related) ancillary record; Radiometric transformation ancillary record. ''' #Record 2 - Scene header record record=2 recordlength=4320 #LS 5 satellite=utilities.readbinary(meta,(record-1)*recordlength,309,324) if not satellite[0:7] == 'LANDSAT': raise NotImplementedError #This error gets ignored in __init__.Open()
def __init__(self, dir, recurse=True, archive=False): ''' Iterator for metadata crawling @type dir: C{str} @param dir: The directory to start the metadata crawl. ''' format_regex = formats.format_regex dir = utilities.uncpath( utilities.realpath(utilities.normcase(utilities.encode(dir)))) #Build a dict of matching files and regexes then sort according to the priority of the regex formats fileformats = {} for f in utilities.rglob(dir, '|'.join(format_regex), True, re.IGNORECASE, recurse=recurse, archive=archive): #Use utf-8 encoding to fix Issue 20 if f[:4] == '/vsi': f = utilities.encode(f) else: f = utilities.realpath(utilities.normcase(utilities.encode(f))) for r in format_regex: #This is so we always return _default_ format datasets last. if re.search(r, os.path.basename(f), re.IGNORECASE): if fileformats.has_key(r): fileformats[r].append(f) else: fileformats[r] = [f] break files = [] for r in format_regex: if fileformats.has_key(r): files.extend(fileformats[r]) #Class vars self.errors = [ ] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info self.files = files self.file = '' self.filecount = len(self.files)