def __init__(self, comp, rawComp, region, acqdate, process):
self.comp = comp
self.layertype = 'ancillary'
self.comp = comp
self.rawComp = rawComp
self.acqdate = acqdate
self.ancilid = '%(p)s.%(d)s' % {
'p': process.dsid,
'd': rawComp.datalayer
}
self.datadir = rawComp.datadir
self.metadir = rawComp.metadir
if len(rawComp.accessdate) >= 4:
self.accessdate = mj_dt.yyyymmddDate(rawComp.accessdate)
else:
self.accessdate = mj_dt.Today()
self.createdate = mj_dt.Today()
self.FPN = False
if self.comp.hdr == 'ers':
FN = '%(f)s.%(d)s.%(e)s' % {
'f': self.rawComp.datafile,
'd': self.comp.dat,
'e': self.comp.hdr
}
else:
if len(self.comp.hdr) > 0:
FN = '%(f)s.%(e)s' % {
'f': self.rawComp.datafile,
'e': self.comp.hdr
}
elif len(self.dat) > 0:
FN = '%(f)s.%(e)s' % {
'f': self.rawComp.datafile,
'e': self.comp.dat
}
elif len(self.comp.hdr) == 0 and len(self.comp.dat) == 0:
#The data is in a folder, e.g. ArcVew raster format
FN = self.datafile
else:
print(self.datafile, self.dat, self.hdr)
exit('FN error in ancillary')
#For some reason os.path.join does not work here
FP = '%s/%s' % (self.comp.mainpath, self.rawComp.datadir)
#FP = os.path.join(ancilC.mainpath,self.datadir)
FPN = os.path.join(FP, FN)
self.FPN = FPN
#add the path except for volume and mai folder + to add to db
dbFP = os.path.join(self.rawComp.datadir, FN)
dbFP = '../%(d)s' % {'d': dbFP}
self.dbFP = dbFP
if self.comp.hdr in ['zip']:
self.zip = 'zip'
elif self.comp.hdr in ['tar.gz', 'tar']:
self.zip = 'tar.gz'
else:
self.zip = False
def _SearchToPostgres(self):
'''Load search holdings to local db
Does not utilize the layer class but take parameters directly from xml
'''
#Set todays date
today = mj_dt.Today()
#set the paths
prodPath ='%s.%s' %(self.process.params.product, self.process.params.version)
localPath = os.path.join('/Volumes',self.process.srcpath.volume,'DAAC-SMAP',prodPath)
#Loop over the dates
for datum in self.process.srcperiod.datumD:
if self.process.srcperiod.datumD[datum]['acqdate'] > today:
#skip all dates later than today (there can be no images from the future)
continue
#convert date to pointed string used on the server
dateStr = mj_dt.DateToStrPointDate(self.process.srcperiod.datumD[datum]['acqdate'])
localFPN = os.path.join(localPath,dateStr)
#Create a sub-folder called done, when the search results are transferred to the db the html will be moved into the done folder
tarFPN = os.path.join(localPath,'done',dateStr)
if not os.path.exists(os.path.split(tarFPN)[0]):
os.makedirs(os.path.split(tarFPN)[0])
if os.path.exists(localFPN):
self._ReadSMAPhtml(self.session,localFPN,tarFPN,self.process.srcperiod.datumD[datum]['acqdate'])
else:
print ('SMAP file missing', localFPN)
def _CheckSmap(self):
#Set the expected layers and parameters for filling the db
queryD = {}
queryD['product'] = {'val':self.process.params.product, 'op':'=' }
queryD['retrieve'] = {'val':'Y', 'op':'=' }
self.paramL = ['source', 'product', 'folder', 'band', 'prefix', 'suffix', 'celltype', 'dataunit', 'scalefac', 'offsetadd', 'cellnull', 'measure', 'retrieve', 'hdffolder', 'hdfgrid']
self.compL = ['source', 'product', 'folder', 'band', 'prefix', 'suffix', 'celltype', 'dataunit', 'scalefac', 'offsetadd', 'cellnull', 'measure']
self.extractL = self.session._SelectSMAPTemplate( queryD, self.paramL )
#from geoimagine.support.modis import DisentangleModisTileName as convFN
#First loop over the src folder structure to find all tiles at this position
#Construct a dummy tile, to get the FP
smapid = 'smapid'
hdfFN = '*.%(hdr)s' % {'hdr':self.process.srcpath.hdrfiletype}
product = self.process.params.product
version = self.process.params.version
source = '%(p)s.%(v)s' %{'p':product,'v':version}
acqdate = mj_dt.Today()
tile = (hdfFN, smapid, source, product, version, 'original', acqdate)
smapTile = self._ConstructDaacTile(tile,self.process.srcpath)
datepath = os.path.split(smapTile.FPN)[0]
locuspath = os.path.split(datepath)[0]
for root, directories, filenames in os.walk(locuspath):
for filename in filenames:
if filename.endswith(self.process.srcpath.hdrfiletype):
queryD = {'smapfilename':filename}
paramL = ['smapid', 'smapfilename', 'source', 'product', 'version', 'acqdate']
tile = self.session._SelectSingleSMAPDaacTile(queryD,paramL)
smapid, smapfilename, source, product, version, acqdate = tile
tile = (smapfilename, smapid, source, product, version, 'original', acqdate)
smapTile = self._ConstructDaacTile(tile,self.process.srcpath)
#Replace is needed for adjusting between SMAP and Karttur default naming conventions
source = source.replace('-E','_E')
source = source.replace('-S','_S')
if os.path.exists(smapTile.FPN):
self.session._InsertSmapData(smapTile.query)
statusD = {'smapid': smapid,'column':'downloaded', 'status': 'Y'}
self.session._UpdateSmapStatus(statusD)
#Only tiles found on file are checked, should it be updated
self._SearchExtractLayers(acqdate)
if self.nrExploded == len(self.extractL):
statusD = {'smapid': smapid,'column':'organized', 'status': 'Y'}
self.session._UpdateSmapStatus(statusD)
statusD = {'smapid': smapid,'column':'exploded', 'status': 'Y'}
self.session._UpdateSmapStatus(statusD)
else:
pass
def DumpExportDB(self):
#(host_name,database_name,user_name,database_password):
if self.process.params.dataonly and self.process.params.schemaonly:
exit('BOTh dataonly and schemaonly can not be set to TRUE')
username, account, password = self.session._SelectUserSecrets()
if self.process.dstpath.volume.lower() == 'none':
vol = ''
else:
vol = '/volumes/%s' % (self.process.dstpath.volume)
FP = path.join(vol, 'SQLdump')
today = kt_dt.DateToStrDate(kt_dt.Today())
#Set filename
FN = 'sqldump-%s' % (self.process.params.format)
if self.process.params.dataonly:
FN += '-dataonly'
elif self.process.params.schemaonly:
FN += '-schemaonly'
FN += '_%s.sql' % (today)
sqlFPN = path.join(FP, FN)
if not path.exists(FP):
makedirs(FP)
if path.isfile(sqlFPN) and not self.process.overwrite:
return
host = 'localhost'
db = 'postgres'
if self.process.params.cmdpath == 'None':
cmd = 'pg_dump'
else:
cmd = '%s/pg_dump' % (self.process.params.cmdpath)
if self.process.params.dataonly:
cmd += ' -h {0} -p 5432 -U {1} -F {2} -f {3} -a {4}'\
.format(host,username,self.process.params.format,sqlFPN,db)
elif self.process.params.schemaonly:
cmd += ' -h {0} -p 5432 -U {1} -F {2} -f {3} -s {4}'\
.format(host,username,self.process.params.format,sqlFPN,db)
else:
cmd += ' -h {0} -p 5432 -U {1} -F {2} -f {3} {4}'\
.format(host,username,self.process.params.format,sqlFPN,db)
system(cmd)
def _SearchSmapProducts(self):
'''IMPORTANT the user credentials must be in a hidden file in users home directory called ".netrc"
'''
#Set todays date
today = mj_dt.Today()
#Set the serverurl, pand the SMAP roduct and version to search for
self.serverurl = self.process.params.serverurl
self.product = self.process.params.product
self.version = self.process.params.version
#check that the version is correctly stated
if not len(self.version) == 3:
exit('The smap version must be 3 digits, e.g. "005" or "006"')
if not self.version.isdigit():
exit('The smap version must be 3 digits, e.g. "005" or "006"')
#Set the sensorpath on the server
sensorurl = 'SMAP'
#put the remote search path for the requested dates together
prodPath ='%s.%s' %(self.product,self.version)
#create the localpath where the search data (html) will be saved
localPath = os.path.join('/volumes',self.process.dstpath.volume,'DAAC-SMAP',prodPath)
if not os.path.exists(localPath):
os.makedirs(localPath)
#change to the local directory
cmd ='cd %s;' %(localPath)
os.system(cmd)
#Loop over the dates defined in process
for datum in self.process.srcperiod.datumD:
print ('searching',datum)
#search for the data
if self.process.srcperiod.datumD[datum]['acqdate'] > today:
#skip all dates later than today (there can be no images from the future)
continue
#convert date to pointed string used on the server
dateStr = mj_dt.DateToStrPointDate(self.process.srcperiod.datumD[datum]['acqdate'])
#define the complete url to the SMAP data
url = os.path.join(self.serverurl,sensorurl,prodPath,dateStr)
#
localFPN = os.path.join(localPath,dateStr)
if os.path.exists(localFPN) and not self.process.overwrite:
continue
#Run the wget command including definition of the cookie needed for accessing the server
cmd ='cd %s;' %(localPath)
cmd ='%(cmd)s /usr/local/bin/wget -L --load-cookies --spider --no-parent ~/.cookies --save-cookies ~/.cookies %(url)s' %{'cmd':cmd, 'url':url}
os.system(cmd)
def ExportTableDataCsvSql(self, schema, table):
import csv
schematab = '%s.%s' % (schema, table)
if self.process.dstpath.volume.lower() == 'none':
vol = ''
else:
vol = '/volumes/%s' % (self.process.dstpath.volume)
FP = path.join(vol, 'SQLdump', schema, table)
today = kt_dt.DateToStrDate(kt_dt.Today())
FN = '%s_sqldump_%s.csv' % (table, today)
csvFPN = path.join(FP, FN)
if not path.exists(FP):
makedirs(FP)
if path.isfile(csvFPN) and not self.process.overwrite:
return
#Get the columns
tableColumnDefL = self.session._GetTableColumns(schema, table)
tableColNameL = [row[0] for row in tableColumnDefL]
query = {'schematab': schematab, 'items': ",".join(tableColNameL)}
#Get all the data in the table
print("SELECT %(items)s FROM %(schematab)s;" % query)
recs = self.session._SelectAllTableRecs(query)
#self.cursor.execute("SELECT %(items)s FROM %(schematab)s;" %query)
#records = self.cursor.fetchall()
if len(recs) == 0:
warnstr = ' WARNING, empty sql dump: skipping export to %s' % (
csvFPN)
print(warnstr)
return
#open csv file for writing
print(' Dumping db records to', csvFPN)
F = open(csvFPN, 'w')
wr = csv.writer(F, delimiter=";")
wr.writerow(tableColNameL)
for row in recs:
wr.writerow(row)
F.close()
def DumpExportTable(self, schema, table):
#(host_name,database_name,user_name,database_password):
if self.process.params.dataonly and self.process.params.schemaonly:
exit('BOTh dataonly and schemaonly can not be set to TRUE')
schematab = '%s.%s' % (schema, table)
username, account, password = self.session._SelectUserSecrets()
if self.process.dstpath.volume.lower() == 'none':
vol = ''
else:
vol = '/volumes/%s' % (self.process.dstpath.volume)
FP = path.join(vol, 'SQLdump', schema, table)
today = kt_dt.DateToStrDate(kt_dt.Today())
#Set filename
#FN = '%s_sqldump_%s' %(table, today)
FN = '%s_sqldump-%s' % (table, self.process.params.format)
if self.process.params.dataonly:
FN += '-dataonly'
elif self.process.params.schemaonly:
FN += '-schemaonly'
FN += '_%s.sql' % (today)
sqlFPN = path.join(FP, FN)
if not path.exists(FP):
makedirs(FP)
if path.isfile(sqlFPN) and not self.process.overwrite:
return
host = 'localhost'
db = 'postgres'
#pg_dump -h localhost -p 5432 -U karttur -t process.subprocesses -f dbtab.sql postgres
'''
cmd = 'pg_dump -h {0} -p 5432 -U {1} -t {2} -Fc -f {3} {4}'\
.format(host,username,schematab,sqlFPN,db)
#print (cmd)
#BALLE
#oscmd = '%(cmdpath)s karttur -U postgres -t %(schematab)s -F c > %(sqlfpn)s' %{'cmdpath':cmdpath,'schematab':schematab, 'sqlfpn':sqlFPN}
#print (oscmd)
#p = Popen(cmd,shell=True,stdin=PIPE,stdout=PIPE,stderr=PIPE)
#p = Popen(cmd,stdin=PIPE,stdout=PIPE,stderr=PIPE)
proc = Popen(['pg_dump', '-h', host, '-U', username, '-p', '5432',
'-t', schematab, '-f', sqlFPN, db],
shell=True, stdin=PIPE)
proc.wait()
proc = Popen(['pg_dump', '-h', host, '-U', username, '-p', 5432,
'-F', 't', '-f', sqlFPN, db],
cwd=directory, shell=True, stdin=PIPE)
print ('old',cmd)
'''
if self.process.params.cmdpath == 'None':
cmd = 'pg_dump'
else:
#print (self.process.params.cmdpath)
cmd = '%s/pg_dump' % (self.process.params.cmdpath)
if self.process.params.dataonly:
cmd += ' -h {0} -p 5432 -U {1} -t {2} -F {3} -f {4} -a {5}'\
.format(host,username,schematab,self.process.params.format,sqlFPN,db)
elif self.process.params.schemaonly:
cmd += ' -h {0} -p 5432 -U {1} -t {2} -F {3} -f {4} -s {5}'\
.format(host,username,schematab,self.process.params.format,sqlFPN,db)
else:
cmd += ' -h {0} -p 5432 -U {1} -t {2} -F {3} -f {4} {5}'\
.format(host,username,schematab,self.process.params.format,sqlFPN,db)
system(cmd)