def register(self,cycle=None,since=None):
if since:
since=datetime.strptime(since,"%Y-%m-%d")
print(since)
else:
since=self._dbinvent.lastupdate
#create a list of files which need to be (re)registered
if self.updated:
files=self.updated
else:
slurplogger().info("Listing files to process (this can take a while)...")
if cycle:
files=[UriFile(file) for file in findFiles(os.path.join(self._dbinvent.datadir,self.sat,self.phase,"c%03d"%(cycle)),'.*\.nc$',since=since)]
else:
files=[UriFile(file) for file in findFiles(os.path.join(self._dbinvent.datadir,self.sat,self.phase),'.*\.nc$',since=since)]
if not files:
slurplogger().info("No updated files found")
return
newfiles=self.retainnewUris(files)
if not newfiles:
slurplogger().info("Nothing to update")
return
for uri in newfiles:
meta=radsMetaDataExtractor(uri)
if not meta:
#don't register empty entries
continue
self.addEntry(meta)
self.updateInvent()
def register(self, pattern=None):
"""Register static gravity fields donwloaded in the data director
:param pattern: only register files whose filename obeys this regular expression
"""
if not pattern:
pattern = '.*\.gz'
#create a list of files which need to be (re)registered
if self.updated:
files = self.updated
else:
files = [
UriFile(file) for file in findFiles(self.dataDir(), pattern)
]
#loop over files
for uri in files:
urilike = os.path.basename(uri.url)
if not self.uriNeedsUpdate(urilike, uri.lastmod):
continue
meta = icgemMetaExtractor(uri)
self.addEntry(meta)
self.updateInvent()
def register(self):
#create a list of files which need to be (re)registered
if self.updated:
files=self.updated
else:
files=[UriFile(file) for file in findFiles(self._dbinvent.datadir,'.*gfc.gz',since=self._dbinvent.lastupdate)]
newfiles=self.retainnewUris(files)
#loop over files
for uri in newfiles:
slurplogger().info("extracting meta info from %s"%(uri.url))
meta=icgemMetaExtractor(uri)
meta=enhanceMeta(meta)
self.addEntry(meta)
self.updateInvent()
def register(self,rundir=None,pattern='.*\.nc$'):
"""register netcdf output files
@param rundir: directory where the netcdf files reside
@param pattern: regular expression which the netcdfiles must obey defaults tkakes all files ending with nc"""
if not rundir:
raise RuntimeError("A directory/regex with output data needs to be supplied when registering this dataset")
newfiles=self.retainnewUris([UriFile(file) for file in findFiles(rundir,pattern)])
for uri in newfiles:
meta=orasMetaExtractor(uri)
if not meta:
#don't register empty entries
continue
self.addEntry(meta)
self._dbinvent.data["Description"]="ORAS5 output data table"
self.setDataDir(os.path.abspath(rundir))
self._dbinvent.data["grid"]="025"
self.updateInvent()
def register(self, pattern=None):
if not pattern:
pattern = '.*\.gz'
#create a list of files which need to be (re)registered
if self.updated:
files = self.updated
else:
files = [
UriFile(file) for file in findFiles(self.dataDir(), pattern)
]
#loop over files
for uri in files:
urilike = os.path.basename(uri.url)
if not self.uriNeedsUpdate(urilike, uri.lastmod):
continue
meta = icgemMetaExtractor(uri)
self.addEntry(meta)
self.updateInvent()
def register(self):
"""Checks the directory for updated raster files and updates them in the database"""
#find all relevant files
newfiles = [
UriFile(file) for file in findFiles(self.srcdir, self.rastregex)
]
self.dropTable()
if self.tiles:
#expand a single raster in tiles
if len(newfiles) != 1:
raise RuntimeError(
"Don't know how to tile multiple input rasters")
#possibly create a temporary table for storing the complete raster
# we need to get a session which is bound to a single transaction unit so we can make use of temporay tables
trans, ses = self.db.transsession()
tmptable = self.db.createTable("tmp_raster", [
Column("id", Integer, primary_key=True),
Column("rast", Raster(spatial_index=False))
],
temporary=True,
bind=ses.get_bind())
meta = self.rastExtract(newfiles[0])
entry = tmptable(rast=meta["rast"])
ses.add(entry)
# create the table but bind it to this session
self.createTable(self.columns(), session=ses)
# fill the new table from the tiles
qry = select([
func.ST_Tile(tmptable.rast, self.tiles[0],
self.tiles[1]).label('rast')
])
ses.execute(self.table.__table__.insert().from_select(['rast'],
qry))
del meta["rast"]
#also set the other column data
if meta:
ses.execute(self.table.__table__.update().values(**meta))
#submit transaction
trans.commit()
else:
for uri in newfiles:
meta = self.rastExtract(uri)
if not meta:
#don't register empty entries
continue
if self.table == None:
#create the table when it does not exist
self.createTable(self.columns())
self.addEntry(meta)
self._ses.commit()
#fix the srid
if self.srid:
self._ses.execute(
text(
"select UpdateRasterSRID('%s'::name,'%s'::name,'rast'::name,%d)"
% (self.scheme, self.name, self.srid)))
#add/compute raster constraints
self._ses.execute(
text(
"select AddRasterConstraints('%s'::name,'%s'::name,'rast'::name)"
% (self.scheme, self.name)))
if self.regularblocking:
self._ses.execute(
text(
"select AddRasterConstraints('%s'::name,'%s'::name,'rast'::name,'regular_blocking')"
% (self.scheme, self.name)))
#create overviews
if self.overviews:
for factor in self.overviews:
self._ses.execute(
text(
"select ST_CreateOverview('%s.%s'::regclass, 'rast', %d, 'Lanczos')"
% (self.scheme, self.name, factor)))
self.updateInvent()
