def update(self, xml):
"""Update record."""
elt, nsDict = getXmlEtree(xml)
key = elt.xpath('./%s' % self.keyCol)[0].text
kwargs = {}
for thisElt in elt:
if thisElt.text == key:
continue
else:
if thisElt.tag == 'localtime':
fieldName = 'ltime'
else:
fieldName = thisElt.tag
typ = self.fieldInfoDict[fieldName]['type']
if typ in ('float', 'double') or \
typ.startswith('double'):
kwargs[fieldName] = float(thisElt.text)
elif typ == 'datetime':
kwargs[fieldName] = getDatetimeFromString(thisElt.text)
else:
kwargs[fieldName] = thisElt.text
try:
res = eval("self.table.%s(key)" % self.keyColSelectMethodStr)
res.set(**kwargs)
except SQLObjectNotFound:
kwargs[self.keyCol] = key
res = self.table(**kwargs)
except:
raise
return True
def getSqlCreateInfo(xml, recordTag):
"""Return list of (column name, sqltype, key) tuples from xml. Specify recordTag
to specify tag that enumerates each record."""
root, nsDict = getXmlEtree(xml)
if root.tag == recordTag:
recElt = root
else:
try:
recElt = root.xpath('.//_default:%s' % recordTag,
namespaces=nsDict)[0]
except:
recElt = root.xpath('.//%s' % recordTag)[0]
retList = []
for subElt in recElt:
retList.append((subElt.tag, subElt.get('sqltype', 'text')))
return retList
def aggregateOverObjectidListUnaltered(xmlString):
"""
Usage: aggregateOverObjectidList(xmlString)
This function takes an arbitrary xml string and
extracts the datasetids via the <objectid> tag.
For each datasetid, it determines whether or not
a corresponding L2 txt file exists locally and creates
a CDAT generic grid of all profiles in NetCDF. It
returns the xml result set of the url pointing to the
NetCDF file."""
#get dir & url config
scp = ScifloConfigParser()
webUrl = scp.getParameter('htmlBaseHref')
dataUrl = webUrl.replace('/web/', '/data/')
dataDir = os.path.join(sys.prefix, 'share', 'sciflo', 'data')
gpsGenDir = os.path.join(dataDir, 'gps', 'genGrids')
gpsGenUrl = os.path.join(dataUrl, 'gps', 'genGrids')
if not os.path.isdir(gpsGenDir): os.makedirs(gpsGenDir)
#get occids
e, nsDict = getXmlEtree(xmlString)
print xmlString
objectids = [i for i in e.xpath('.//_default:objectid/text()', nsDict)]
print objectids
#objectids.sort()
#sort and generate hash
hash = hashlib.md5(','.join(objectids)).hexdigest() + '.nc'
ncFile = os.path.join(gpsGenDir, hash)
ncUrl = os.path.join(gpsGenUrl, hash)
#create generic grid if it doesn't exist
if not os.path.isfile(ncFile):
#create generic grid
g = GpsGenGrid(objectids,
addModel=['ncep', 'ecmwf'],
root=ROOT_GLEVELS_DIR,
doSort=False)
n = g.writeNcUnaltered(ncFile)
return simpleList2Xml([ncUrl], 'ResultSet', 'url')
def getIndexedFields(xml, recordTag):
"""Return list of indexed fields from xml. Specify recordTag
to specify tag that enumerates each record."""
root, nsDict = getXmlEtree(xml)
if root.tag == recordTag:
recElt = root
else:
try:
recElt = root.xpath('.//_default:%s' % recordTag,
namespaces=nsDict)[0]
except:
recElt = root.xpath('.//%s' % recordTag)[0]
retList = []
for subElt in recElt:
sqltype = subElt.get('sqltype', 'text')
if re.search(r'(index|key)', sqltype, re.IGNORECASE):
retList.append(subElt.tag)
if len(retList) == 0:
raise NoIndexedFieldsInXmlError(
"Unable to find indexed/keyed fields from xml: %s" % xml)
return retList
def getConversionFunctionString(fromType, toType, namespacePrefixDict={}):
"""Return a string indicating the proper conversion function.
Namespace prefixes are resolved from the namespacePrefixDict of the
xml doc."""
# force xpath type coersion
if isinstance(fromType, str) and \
re.search(r'^(sf:)?(xpath:)', fromType, re.IGNORECASE):
fromType = '*:*'
# get conv func xml elt and ns prefix dict
convFuncElt, convNsDict = getXmlEtree(CONVERSION_FUNCTIONS_CONFIG)
convNsToPrefixDict = dict(
list(zip(list(convNsDict.values()), list(convNsDict.keys()))))
# get overwrite conv funcs from user config
(userName, homeDir, userScifloDir, userConfigFile) = getUserInfo()
userConfElt, userConfNsDict = getXmlEtree(userConfigFile)
userConfNsToPrefixDict = dict(
list(zip(list(userConfNsDict.values()), list(userConfNsDict.keys()))))
ops = userConfElt.xpath('.//_default:conversionOperators/_default:op',
namespaces=userConfNsDict)
for op in ops:
opFromNs, opFromVal = parseNamespacePrefixAndTypeString(op.get('from'))
if opFromNs is None:
opFromNs = userConfNsToPrefixDict[namespacePrefixDict['_default']]
elif opFromNs == '*':
pass
else:
opFromNs = userConfNsToPrefixDict[namespacePrefixDict[opFromNs]]
opToNs, opToVal = parseNamespacePrefixAndTypeString(op.get('to'))
if opToNs is None:
opToNs = userConfNsToPrefixDict[namespacePrefixDict['_default']]
elif opToNs == '*':
raise RuntimeError(
"Cannot convert to type %s with unspecified namespace prefix %s."
% (opToVal, opToNs))
else:
opToNs = userConfNsToPrefixDict[namespacePrefixDict[opToNs]]
overwriteMatch = convFuncElt.xpath(
".//_default:op[@from='%s:%s' and @to='%s:%s']" %
(opFromNs, opFromVal, opToNs, opToVal),
namespaces=convNsDict)
if len(overwriteMatch) == 1:
if str(overwriteMatch[0].text) != str(op.text):
convFuncElt.remove(overwriteMatch[0])
convFuncElt.append(op)
else:
continue
elif len(overwriteMatch) == 0:
convFuncElt.append(op)
else:
raise RuntimeError("Found more than one match.")
# get fromType namespace prefix, type, and namespace
(fromNsPrefix, fromTypeVal) = parseNamespacePrefixAndTypeString(fromType)
if fromNsPrefix is None:
fromNs = convNsToPrefixDict[namespacePrefixDict['_default']]
elif fromNsPrefix == '*':
fromNs = '*'
else:
fromNs = convNsToPrefixDict[namespacePrefixDict[fromNsPrefix]]
# get toType namespace prefix, type and namespace
(toNsPrefix, toTypeVal) = parseNamespacePrefixAndTypeString(toType)
if toNsPrefix is None:
toNs = convNsToPrefixDict[namespacePrefixDict['_default']]
elif toNsPrefix == '*':
raise RuntimeError(
"Cannot convert to type %s with unspecified namespace prefix %s." %
(toTypeVal, toNsPrefix))
else:
toNs = convNsToPrefixDict[namespacePrefixDict[toNsPrefix]]
# return xpath conversion
if namespacePrefixDict.get(
toNsPrefix,
None) == SCIFLO_NAMESPACE and toTypeVal.startswith('xpath:'):
return toTypeVal
# xpath to find match
matches = convFuncElt.xpath(
".//_default:op[@from='%s:%s' and @to='%s:%s']" %
(fromNs, fromTypeVal, toNs, toTypeVal),
namespaces=convNsDict)
if len(matches) == 0:
matches = convFuncElt.xpath(
".//_default:op[@from='*:*' and @to='%s:%s']" % (toNs, toTypeVal),
namespaces=convNsDict)
if len(matches) == 0:
raise RuntimeError(
"Cannot find conversion function for %s:%s -> %s:%s." %
(fromNs, fromTypeVal, toNs, toTypeVal))
return str(matches[0].text)
def _execute(self, result, workDir):
"""Perform the post execution function. Pass in the entire result
of the work unit and this handler will run the conversion function
on the specified indexed result or on the entire result (if the
resultIndex is None). Return the result of the function.
"""
# get the result to perform function on
if self._resultIndex is None:
inputResult = result
else:
inputResult = result[self._resultIndex]
# special case conversions
xpathMatch = re.search(r'^xpath:(.+)$', self._conversionFuncStr)
if xpathMatch:
resultDoc, resultNs = getXmlEtree(inputResult)
return resultDoc.xpath(xpathMatch.group(1), namespaces=resultNs)
# eval the conversion function
convFunc = getFunction(self._conversionFuncStr,
addToSysPath=getUserConversionFunctionsDir())
# if FunctionWrapper, get local files
tempDir = None
tmpInputRes = []
if isinstance(convFunc, LocalizingFunctionWrapper):
tempDir = os.path.join(workDir, 'fileConversions')
validateDirectory(tempDir)
if not isinstance(inputResult, (list, tuple)):
singleArgFlag = True
tmpInput = [inputResult]
else:
singleArgFlag = False
tmpInput = inputResult
for ip in tmpInput:
if not isinstance(ip, str):
raise PostExecutionHandlerError(
"Cannot localize input %s. Please check return value of work unit."
% str(ip))
match = re.search(r'^\w*?://', ip)
if match:
filebase = urllib.parse.urlparse(ip)[2].split('/')[-1]
else:
filebase = os.path.basename(ip)
tempFile = os.path.join(tempDir, filebase)
(ip, headers) = urllib.request.urlretrieve(ip, tempFile)
tmpInputRes.append(tempFile)
if singleArgFlag:
inputResult = tmpInputRes[0]
else:
inputResult = tmpInputRes
# do conversion
if isinstance(convFunc, FileConversionFunction):
if singleArgFlag:
retVal = convFunc(inputResult)
else:
retVal = [convFunc(ip) for ip in inputResult]
else:
retVal = convFunc(inputResult)
return retVal
def __init__(self, location, xmlSchema):
"""Constructor."""
self.location = location
self.connection = connectionForURI(location)
sqlhub.processConnection = self.connection
self.schemaElt, self.schemaNsDict = getXmlEtree(xmlSchema)
self.tableName = self.schemaElt.xpath('./TableName')[0].text
self.fieldElts = self.schemaElt.xpath('.//Field')
classDefStrList = ["class %s(SQLObject):" % self.tableName]
self.keyCol = None
self.keyColSelectMethodStr = None
self.fieldInfoDict = {}
for fieldElt in self.fieldElts:
id = fieldElt.xpath('./FieldName')[0].text
typ = fieldElt.xpath('./Type')[0].text
null = fieldElt.xpath('./Null')[0].text
key = fieldElt.xpath('./Key')[0].text
default = fieldElt.xpath('./Default')[0].text
# get nullable flag
nullable = None
if null is None:
pass
elif re.search(r'yes', null, re.IGNORECASE):
nullable = False
else:
pass
# get key spec
alternateID = None
unique = None
if key == '' or re.search(r'mul', key, re.IGNORECASE):
pass
elif re.search(r'pri', key, re.IGNORECASE):
alternateID = True
nullable = None
self.keyCol = id
self.keyColSelectMethodStr = 'by' + id[0].upper() + id[1:]
elif re.search(r'uni', key, re.IGNORECASE):
unique = True
elif re.search(r'yes', key, re.IGNORECASE):
pass
else:
raise ScifloDbTableError("Unknown key: %s" % key)
# get default
defaultVal = None
if default:
defaultVal = default
# get column type
colLen = None
if typ.startswith('char'):
colClass = "StringCol"
matchChar = re.search(r'^char\((\d+)\)$', typ)
if matchChar:
colLen = matchChar.group(1)
elif typ == 'datetime':
colClass = "DateTimeCol"
elif typ.startswith('double'):
colClass = "FloatCol"
elif typ == 'time':
colClass = "StringCol"
else:
raise ScifloDbTableError("Unknown column type: %s" % typ)
# add col string for class
colStr = " %s = %s(" % (id, colClass)
colArgsList = []
if alternateID == True:
colArgsList.append("alternateID=True")
if nullable:
colArgsList.append("notNone=True")
if unique == True:
colArgsList.append("unique=True")
if colLen is not None:
colArgsList.append("length=%s" % colLen)
if defaultVal is not None:
colArgsList.append("default='%s'" % defaultVal)
self.fieldInfoDict[id] = {
'type': typ,
'alternateID': alternateID,
'nullable': nullable,
'unique': unique,
'colLen': colLen,
'key': key,
'defaultVal': defaultVal
}
# build col string
colStr = colStr + ','.join(colArgsList) + ')'
# append to class def str
classDefStrList.append(colStr)
# class def string
classDefStr = "\n".join(classDefStrList)
# exec class code and create table
try:
exec(classDefStr)
self.table = eval(self.tableName)
except ValueError as e:
if 'is already in the registry' in str(e):
self.table = classregistry.findClass(self.tableName)
else:
raise
except:
raise
self.table._connection.debug = False
self.table.createTable(ifNotExists=True)
def resolveDocumentInput(self, wuConfig, docArgList):
"""Resolve process input doc and return new args list."""
def addCDATAPlaceholders(txt):
return "CDATA_BEGIN_PLACEHOLDER%sCDATA_END_PLACEHOLDER" % txt
def replaceCDATAPlaceholders(txt):
return txt.replace('CDATA_BEGIN_PLACEHOLDER', '<![CDATA[').replace(
'CDATA_END_PLACEHOLDER',
']]>').replace('<CDATA_TPL>', '<![CDATA[').replace(
'</CDATA_TPL>', ']]>').replace('&', '&')
inputsElt, inputsNsDict = getXmlEtree(docArgList.docStr)
wuType = wuConfig.getType()
if wuType in ('soap', 'post'):
argIdx = 1
else:
argIdx = 0
for elt in inputsElt.getiterator():
if docArgList[argIdx] is not None:
format = elt.get('format', None)
eltTxt = str(docArgList[argIdx])
# interpolate CDATA
if format == 'CDATA':
if isXml(eltTxt):
eltChild, eltNs = getXmlEtree(eltTxt)
cdataBlockElt = lxml.etree.SubElement(elt, 'CDATA_TPL')
cdataBlockElt.append(eltChild)
eltTxt = None
cdataBlockElt.text = None
else:
eltTxt = addCDATAPlaceholders(eltTxt)
else:
# interpolate xml element
if isXml(eltTxt):
# create elt tree and append as child
eltChild, eltNs = getXmlEtree(eltTxt)
elt.append(eltChild)
eltTxt = None
# interpolate xml fragment
else:
tmpXml = '<tmp>%s</tmp>' % eltTxt
if isXml(tmpXml):
eltChild, eltNs = getXmlEtree(tmpXml)
eltChildKids = eltChild.getchildren()
if len(eltChildKids) == 0:
pass # no children to add
else:
for i in eltChild.getchildren():
elt.append(i)
eltTxt = None
# interpolate string
else:
pass
elt.text = eltTxt
if elt.get('from', None):
del elt.attrib['from']
if format:
del elt.attrib['format']
argIdx += 1
docList = [
replaceCDATAPlaceholders(
lxml.etree.tostring(i, pretty_print=True, encoding='unicode'))
for i in inputsElt
]
doc = '\n'.join(docList)
if wuType in ('soap', 'post'):
return [docArgList[0], doc]
else:
return [doc]