def getHexDigest(args):
"""Return a md5 hex digest of the objects passed in."""
# create string from args
argsString = str(getArgsString(getListFromUnknownObject(args)))
# get md5 hex digest
return hashlib.md5(argsString.encode('utf-8')).hexdigest()
def generateWorkUnitHexDigest(owner, type, call, args, stageFiles,
postExecIds):
"""Return a md5 hex digest of the objects passed in."""
# get md5 hex digest
return hashlib.md5(
"%s %s %s %s %s %s" %
(owner, type, call, getArgsString(args),
getStageFilesString(getListFromUnknownObject(stageFiles)),
getArgsString(postExecIds))).hexdigest()
def getDeleteSql(tableName, xml, recordTag, keyTags=[], database='mysql'):
"""Return SQL delete statement string."""
keyTags = getListFromUnknownObject(keyTags)
t = Table(tableName)
retStrList = []
lod = getPyLoD(xml, recordTag)
if len(keyTags) == 0:
keyTags = getIndexedFields(xml, recordTag)
for d in lod:
whereClause = AND(*[d.get(i) == getattr(t, i) for i in keyTags])
retStrList.append(sqlrepr(Delete(t, where=whereClause), database))
return '\n'.join(retStrList)
def query(self, queryField, queryValue, returnField=None, getFieldsListFlag=None):
"""Return a list of result sets of the specified field values of records
matching the query value. If no return fields are specified, it just
returns the result set list of value of the query field. If the optional
getFieldsListFlag flag is set, will return a tuple (resultSetList, returnFieldsList).
"""
# return columns
if returnField is None:
returnFieldsList = [queryField]
else:
returnFieldsList = getListFromUnknownObject(returnField)
# get result set
resultSetList = self._query(queryField, queryValue, returnFieldsList)
# return final list of field names and the result set
if getFieldsListFlag:
return (resultSetList, returnFieldsList)
else:
return resultSetList
def getScifloidInfo(self, scifloId, fields):
"""Return sorted list of dicts containing field info. Sorted by
process number in sciflo.
"""
# fields
fields = getListFromUnknownObject(fields)
# append procNum automatically if not in there already
fields.insert(0, 'index')
# get wuConfigIds
wuConfigIdList = self._managerStore.query('scifloid', scifloId,
'wuConfigId')
# loop over and build result list
retList = []
for wuConfigIds in wuConfigIdList:
if len(wuConfigIds) != 1:
raise ScheduleStoreHandlerError(
"Cannot handle number of wuConfigIds found: %s" %
wuConfigIds)
retList.append(
self.getValueByWorkUnitConfigId(wuConfigIds[0], fields))
# sort
retList.sort()
# loop over create dict and set order
newList = []
for ret in retList:
retDict = {}
for i in range(1, len(fields)):
fname = fields[i]
val = ret[i]
retDict[fname] = val
newList.append(retDict)
# return
return newList
def getValueByWorkUnitConfigId(self, wuConfigId, field):
"""Return the value of a field in the store by work unit config id.
Resolve any cached entries to their values.
"""
# if work unit config is here, check if it was cached by checking digest
if self.isWorkUnitConfigIdHere(wuConfigId):
# set store
self.setStore(wuConfigId)
# get digest and status
digest, status = self.queryUnique('wuConfigId',
wuConfigId, ['digest', 'status'],
dataStore=True)
# print "digest,status:",digest,status
# if it is a sciflowuid, it is cached
fields = getListFromUnknownObject(field)
if digest is not None and re.search(
r'^workunitconfigid-', digest) and status == cachedStatus:
self.setStore(digest, cachedStore=True)
if len(fields) == 1:
if fields[0] in cachedFields:
return self.queryUnique('wuConfigId',
digest,
fields[0],
dataStore=True,
cachedStore=True)
else:
return self.queryUnique('wuConfigId',
wuConfigId,
fields[0],
dataStore=True)
else:
cFields = []
dFields = []
for f in fields:
if f in cachedFields:
cFields.append(f)
else:
dFields.append(f)
cResults = getListFromUnknownObject(
self.queryUnique('wuConfigId',
digest,
cFields,
dataStore=True,
cachedStore=True))
dResults = getListFromUnknownObject(
self.queryUnique('wuConfigId',
wuConfigId,
dFields,
dataStore=True))
results = []
for f in fields:
if f in cFields:
results.append(cResults[cFields.index(f)])
elif f in dFields:
results.append(dResults[dFields.index(f)])
else:
raise ScheduleStoreHandlerError(
"Cannot find field %s in results." % f)
return results
# otherwise return the value from this work unit
else:
if len(fields) == 1:
return self.queryUnique('wuConfigId',
wuConfigId,
fields[0],
dataStore=True)
else:
return self.queryUnique('wuConfigId',
wuConfigId,
fields,
dataStore=True)
# if not here return None
else:
return None
def getValueByWorkUnitId(self, wuid, field):
"""Return the value of a field in the store by work unit id.
Resolve any cached entries to their values.
"""
# if work unit is here, check if it was cached by checking digest for a sciflo work unit id
if self.isWorkUnitIdHere(wuid):
# set store
self.setStore(wuid)
# get digest and status
digest, status = (None, None)
qTryRes = None
for qTry in range(2):
qTryRes = self.queryUnique('wuid',
wuid, ['digest', 'status'],
dataStore=True)
if qTryRes is not None:
digest, status = qTryRes
break
time.sleep(1)
if qTryRes is None:
raise WorkUnitStoreHandlerError(
"Couldn't get digest/status for wuid %s" % wuid)
# print "digest,status:",digest,status
# if it is a sciflowuid, it is cached
fields = getListFromUnknownObject(field)
if digest is not None and re.search(
r'^sciflowuid-', digest) and status == cachedStatus:
self.setStore(digest, cachedStore=True)
if len(fields) == 1:
if fields[0] in cachedFields:
return self.queryUnique('wuid',
digest,
fields[0],
dataStore=True,
cachedStore=True)
else:
return self.queryUnique('wuid',
wuid,
fields[0],
dataStore=True)
else:
cFields = []
dFields = []
for f in fields:
if f in cachedFields:
cFields.append(f)
else:
dFields.append(f)
cResults = getListFromUnknownObject(
self.queryUnique('wuid',
digest,
cFields,
dataStore=True,
cachedStore=True))
dResults = getListFromUnknownObject(
self.queryUnique('wuid', wuid, dFields,
dataStore=True))
results = []
for f in fields:
if f in cFields:
results.append(cResults[cFields.index(f)])
elif f in dFields:
results.append(dResults[dFields.index(f)])
else:
raise WorkUnitStoreHandlerError(
"Cannot find field %s in results." % f)
return results
# otherwise return the value from this work unit
else:
if len(fields) == 1:
return self.queryUnique('wuid',
wuid,
fields[0],
dataStore=True)
else:
return self.queryUnique('wuid',
wuid,
fields,
dataStore=True)
# if not here return None
else:
return None