def __init__(self, f=None):
'''Open the dataset'''
if not f: f = self.fileinfo['filepath']
self.filelist = [r for r in utilities.rglob(os.path.dirname(f))
] #everything in this dir and below.
led = glob.glob(os.path.dirname(f) +
'/[Ll][Ee][Aa][Dd]*')[0] #volume file
meta = open(led, 'rb').read()
'''
metadata has 4 records, each is 4320 (LS) or 6120 (SPOT) bytes long:
File descriptor record;
Scene header record;
Map projection (scene-related) ancillary record;
Radiometric transformation ancillary record.
'''
#Record 2 - Scene header record
record = 2
recordlength = 4320 #LS 5
satellite = utilities.readbinary(meta, (record - 1) * recordlength,
309, 324)
if not satellite[0:7] == 'LANDSAT':
raise NotImplementedError #This error gets ignored in __init__.Open()
def __init__(self, f):
if not f: f = self.fileinfo['filepath']
if f[:4] == '/vsi': raise NotImplementedError
d = os.path.dirname(f)
if open(f).read(1024).strip()[0] == '<': #HTML file, ignore it.
raise NotImplementedError
if 'HRF' in f.upper():
self._filetype = 'HRF'
#rex='BAND[1-57]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' #not picking up the ACRES .ers files
rex = 'BAND[1-57].*|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST'
elif 'HTM' in f.upper():
self._filetype = 'HTM'
#rex='BAND6[LH]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' #not picking up the ACRES .ers files
rex = 'BAND6[LH].*|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST'
elif 'HPN' in f.upper():
self._filetype = 'HPN'
#rex='BAND8\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' #not picking up the ACRES .ers files
rex = 'BAND8.*|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST'
filelist = [f] #header
filelist.extend([
f for f in utilities.rglob(
d, rex, regex=True, regex_flags=re.I, recurse=False)
]) #bands
self.filelist = filelist #"self.filelist" is a property, not a list, we can only 'get' or 'set' it.
def __init__(self, f):
if f[:4] == '/vsi': raise NotImplementedError
dirname, basename = os.path.split(f)
rootname, ext = os.path.splitext(basename)
self.filelist = [
f for f in utilities.rglob(dirname,
pattern=rootname + ".*",
regex=True,
regex_flags=re.I,
recurse=False)
]
self.exts = {
'.tif': 'GTiff/GeoTIFF',
'.img': 'HFA/Erdas Imagine Images (.img)',
'.ntf': 'NITF/National Imagery Transmission Format (.ntf)',
'.pix': 'PCI Geomatics Database File (.pix)'
}
self.img = False
for ext in self.exts:
imgs = [i for i in self.filelist if i[-4:] in [ext, ext.upper()]]
if imgs:
self.img = imgs[0]
break
if not self.img:
raise NotImplementedError, 'Matching DigitalGlobe imagery file not found:\n'
def __init__(self,dir, recurse=True, archive=False):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
'''
format_regex = formats.format_regex
dir=utilities.uncpath(utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats={}
for f in utilities.rglob(dir,'|'.join(format_regex), True, re.IGNORECASE, recurse=recurse, archive=archive):
#Don't return existing overviews
if f[-7:] in ('qlk.jpg','thm.jpg'):continue
#Use utf-8 encoding to fix Issue 20
if f[:4]=='/vsi':f=utilities.encode(f)
else:f=utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r,os.path.basename(f),re.IGNORECASE):
if fileformats.has_key(r):fileformats[r].append(f)
else:fileformats[r]=[f]
break
files=[]
for r in format_regex:
if fileformats.has_key(r):files.extend(fileformats[r])
#Class vars
self.errors=[] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
self.files=files
self.file=''
self.filecount=len(self.files)
def CreateMEF(outdir,
xmlfile,
uid,
overviews=[],
cat=categories['default'],
ops=operations['default']):
'''Generate Geonetwork "Metadata Exchange Format" from an ISO19139 XML record
@see:
U{http://www.fao.org/geonetwork/docs/ch17s02.html}
U{http://trac.osgeo.org/geonetwork/wiki/MEF}
@param outdir: Directory to write MEF file to.
@param xmlfile: XML file to create MEF from.
@param uid: ID of metadata record (UUID/GUID string).
@keyword overviews: List of overview image file (e.g quicklooks & thumbnails) OPTIONAL.
@keyword cat: List of GeoNetwork categories to include in the MEF OPTIONAL.
@todo: Assumes metadata is ISO19139, need to make generic somehow...
'''
xmldir = _path.dirname(xmlfile)
curdir = _path.abspath(_os.curdir)
#mefdir=_path.join(_os.environ['TEMP'],_path.basename(_path.splitext(xmlfile)[0]))
mefdir = mefdir = _tmp.mkdtemp()
mefpath = '%s.mef' % (_path.join(
outdir, _path.basename(_path.splitext(xmlfile)[0])))
try: #
if _path.exists(mefpath): _os.remove(mefpath)
#if _path.exists(mefdir):_sh.rmtree(mefdir)
mef = _zip.ZipFile(mefpath, 'w', _zip.ZIP_DEFLATED)
#_os.mkdir(mefdir)
_os.chdir(mefdir)
_sh.copy(xmlfile, 'metadata.xml')
if overviews:
_os.mkdir('public')
for f in overviews:
_sh.copy(f, _path.join('public', _path.basename(f)))
_CreateInfo(uid, overviews, cat, ops)
#_sh.copy(xmlfile,'metadata.xml')
for f in _utilities.rglob('.'):
if not _path.isdir(f): mef.write(f)
finally:
try:
_os.chdir(curdir)
except:
pass
try:
mef.close()
del mef
except:
pass
try:
_sh.rmtree(mefdir)
except:
pass
def __init__(self,f=None):
if not f:f=self.fileinfo['filepath']
self.filelist=[r for r in utilities.rglob(os.path.dirname(f))]
self._led=f
try:self._vol=glob.glob(os.path.dirname(f) + '/[Vv][Oo][Ll]*')[0] #volume file
except:self._vol=False
img_regex=[
r'IMG-0[1-4]-ALAV.*_U$', #ALOS AVNIR-2 img file
r'.*\.tif$', #ALOS AVNIR-2 ACRES orthocorrected tif file
r'IMG-[HV][HV]-ALPSR.*UD$', #ALOS PALSAR
r'IMG-ALPSM.*\_U[BFNW]$' #ALOS PRISM
]
self._imgs=[i for i in utilities.rglob(os.path.dirname(f),'|'.join(img_regex),True,re.I, False)]
self.fileinfo['filepath']=self._led #change filename to leader file
self.fileinfo['filename']=os.path.basename(self._led)
def __init__(self,f=None):
if not f:f=self.fileinfo['filepath']
if f[:4]=='/vsi':raise NotImplementedError
self.filelist=[r for r in utilities.rglob(os.path.dirname(f))]
self._led=f
try:self._vol=glob.glob(os.path.dirname(f) + '/[Vv][Oo][Ll]*')[0] #volume file
except:self._vol=False
img_regex=[
r'IMG-0[1-4]-ALAV.*_U$', #ALOS AVNIR-2 img file
r'.*\.tif$', #ALOS AVNIR-2 ACRES orthocorrected tif file
r'IMG-[HV][HV]-ALPSR.*UD$', #ALOS PALSAR
r'IMG-ALPSM.*\_U[BFNW]$' #ALOS PRISM
]
self._imgs=[i for i in utilities.rglob(os.path.dirname(f),'|'.join(img_regex),True,re.I, False)]
self.fileinfo['filepath']=self._led #change filename to leader file
self.fileinfo['filename']=os.path.basename(self._led)
def CreateMEF(outdir, xmlfile, uid, overviews=[], cat=categories["default"], ops=operations["default"]):
"""Generate Geonetwork "Metadata Exchange Format" from an ISO19139 XML record
@see:
U{http://www.fao.org/geonetwork/docs/ch17s02.html}
U{http://trac.osgeo.org/geonetwork/wiki/MEF}
@param outdir: Directory to write MEF file to.
@param xmlfile: XML file to create MEF from.
@param uid: ID of metadata record (UUID/GUID string).
@keyword overviews: List of overview image file (e.g quicklooks & thumbnails) OPTIONAL.
@keyword cat: List of GeoNetwork categories to include in the MEF OPTIONAL.
@todo: Assumes metadata is ISO19139, need to make generic somehow...
"""
xmldir = _path.dirname(xmlfile)
curdir = _path.abspath(_os.curdir)
# mefdir=_path.join(_os.environ['TEMP'],_path.basename(_path.splitext(xmlfile)[0]))
mefdir = mefdir = _tmp.mkdtemp()
mefpath = "%s.mef" % (_path.join(outdir, _path.basename(_path.splitext(xmlfile)[0])))
try: #
if _path.exists(mefpath):
_os.remove(mefpath)
# if _path.exists(mefdir):_sh.rmtree(mefdir)
mef = _zip.ZipFile(mefpath, "w", _zip.ZIP_DEFLATED)
# _os.mkdir(mefdir)
_os.chdir(mefdir)
_sh.copy(xmlfile, "metadata.xml")
if overviews:
_os.mkdir("public")
for f in overviews:
_sh.copy(f, _path.join("public", _path.basename(f)))
_CreateInfo(uid, overviews, cat, ops)
# _sh.copy(xmlfile,'metadata.xml')
for f in _utilities.rglob("."):
if not _path.isdir(f):
mef.write(f)
finally:
try:
_os.chdir(curdir)
except:
pass
try:
mef.close()
del mef
except:
pass
try:
_sh.rmtree(mefdir)
except:
pass
def __init__(self, f=None):
if not f:
f = self.fileinfo["filepath"]
if f[:4] == "/vsi":
raise NotImplementedError
self.filelist = [r for r in utilities.rglob(os.path.dirname(f))]
self._led = f
try:
self._vol = glob.glob(os.path.dirname(f) + "/[Vv][Oo][Ll]*")[0] # volume file
except:
self._vol = False
img_regex = [
r"IMG-0[1-4]-ALAV.*_U$", # ALOS AVNIR-2 img file
r".*\.tif$", # ALOS AVNIR-2 ACRES orthocorrected tif file
r"IMG-[HV][HV]-ALPSR.*UD$", # ALOS PALSAR
r"IMG-ALPSM.*\_U[BFNW]$", # ALOS PRISM
]
self._imgs = [i for i in utilities.rglob(os.path.dirname(f), "|".join(img_regex), True, re.I, False)]
self.fileinfo["filepath"] = self._led # change filename to leader file
self.fileinfo["filename"] = os.path.basename(self._led)
def __init__(self,f=None):
'''Open the dataset'''
if not f:f=self.fileinfo['filepath']
led=glob.glob(os.path.dirname(f) + '/[Ll][Ee][Aa][Dd]*')[0] #volume file
meta = open(led,'rb').read()
#Record 2 - Scene header record
record=2
recordlength=3960 #SPOT recordlength=3960
satellite=utilities.readbinary(meta,(record-1)*recordlength,613,628)
if not satellite[0:4] == 'SPOT':
raise NotImplementedError #This error gets ignored in __init__.Open()
self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] #everything in this dir and below.
def __init__(self, dir, recurse=True, archive=False, excludes=[]):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
@type recurse: C{bool}
@param recurse: Recurse into subdirectories?
@type archive: C{bool}
@param archive: Look in zip/gzip archives
@type excludes: C{list}
@param excludes: List of glob style file/directory exclusion pattern/s
'''
#Class vars
self.errors = [
] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
format_regex = formats.format_regex
dir = utilities.uncpath(
utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats = {}
for f in utilities.rglob(dir,
'|'.join(format_regex),
True,
re.IGNORECASE,
recurse=recurse,
archive=archive,
excludes=excludes,
onerror=self.onerror,
followlinks=False):
#Don't return existing overviews
if f[-7:] in ('qlk.jpg', 'thm.jpg'): continue
#Use utf-8 encoding to fix Issue 20
if f[:4] == '/vsi': f = utilities.encode(f)
else:
f = utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r, os.path.basename(f), re.IGNORECASE):
if fileformats.has_key(r): fileformats[r].append(f)
else: fileformats[r] = [f]
break
files = []
for r in format_regex:
if fileformats.has_key(r): files.extend(fileformats[r])
#Class vars
self.files = files
self.file = ''
self.filecount = len(self.files)
def __init__(self,f):
if f[:4]=='/vsi':raise NotImplementedError
dirname,basename=os.path.split(f)
rootname,ext=os.path.splitext(basename)
self.filelist=[f for f in utilities.rglob(dirname, pattern=rootname+".*", regex=True, regex_flags=re.I, recurse=False)]
self.exts={'.tif':'GTiff/GeoTIFF',
'.img':'HFA/Erdas Imagine Images (.img)',
'.ntf':'NITF/National Imagery Transmission Format (.ntf)',
'.pix':'PCI Geomatics Database File (.pix)'}
self.img=False
for ext in self.exts:
imgs=[i for i in self.filelist if i[-4:] in [ext,ext.upper()]]
if imgs:
self.img=imgs[0]
break
if not self.img:raise NotImplementedError, 'Matching DigitalGlobe imagery file not found:\n'
def __init__(self,f):
if not f:f=self.fileinfo['filepath']
d=os.path.dirname(f)
if open(f).read(1024).strip()[0]=='<':#HTML file, ignore it.
raise NotImplementedError
if 'HRF' in f.upper():
self._filetype='HRF'
#rex='BAND[1-57]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST' #not picking up the ACRES .ers files
rex='BAND[1-57].*|L7[0-9]{7,7}_[0-9]{11,11}_B[1-57]0\.FST'
elif 'HTM' in f.upper():
self._filetype='HTM'
#rex='BAND6[LH]\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST' #not picking up the ACRES .ers files
rex='BAND6[LH].*|L7[0-9]{7,7}_[0-9]{11,11}_B6[1-2]\.FST'
elif 'HPN' in f.upper():
self._filetype='HPN'
#rex='BAND8\.dat|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST' #not picking up the ACRES .ers files
rex='BAND8.*|L7[0-9]{7,7}_[0-9]{11,11}_B80\.FST'
filelist=[f] #header
filelist.extend([f for f in utilities.rglob(d,rex,regex=True, regex_flags=re.I, recurse=False)]) #bands
self.filelist=filelist #"self.filelist" is a property, not a list, we can only 'get' or 'set' it.
def __init__(self,f=None):
'''Open the dataset'''
if not f:f=self.fileinfo['filepath']
self.filelist=[r for r in utilities.rglob(os.path.dirname(f))] #everything in this dir and below.
led=glob.glob(os.path.dirname(f) + '/[Ll][Ee][Aa][Dd]*')[0] #volume file
meta = open(led,'rb').read()
'''
metadata has 4 records, each is 4320 (LS) or 6120 (SPOT) bytes long:
File descriptor record;
Scene header record;
Map projection (scene-related) ancillary record;
Radiometric transformation ancillary record.
'''
#Record 2 - Scene header record
record=2
recordlength=4320 #LS 5
satellite=utilities.readbinary(meta,(record-1)*recordlength,309,324)
if not satellite[0:7] == 'LANDSAT':
raise NotImplementedError #This error gets ignored in __init__.Open()
def __init__(self, dir, recurse=True, archive=False):
''' Iterator for metadata crawling
@type dir: C{str}
@param dir: The directory to start the metadata crawl.
'''
format_regex = formats.format_regex
dir = utilities.uncpath(
utilities.realpath(utilities.normcase(utilities.encode(dir))))
#Build a dict of matching files and regexes then sort according to the priority of the regex formats
fileformats = {}
for f in utilities.rglob(dir,
'|'.join(format_regex),
True,
re.IGNORECASE,
recurse=recurse,
archive=archive):
#Use utf-8 encoding to fix Issue 20
if f[:4] == '/vsi': f = utilities.encode(f)
else:
f = utilities.realpath(utilities.normcase(utilities.encode(f)))
for r in format_regex: #This is so we always return _default_ format datasets last.
if re.search(r, os.path.basename(f), re.IGNORECASE):
if fileformats.has_key(r): fileformats[r].append(f)
else: fileformats[r] = [f]
break
files = []
for r in format_regex:
if fileformats.has_key(r): files.extend(fileformats[r])
#Class vars
self.errors = [
] #A list of files that couldn't be opened. Contains a tuple with file name, error info, debug info
self.files = files
self.file = ''
self.filecount = len(self.files)
