def formatDict(self, result):
"""
Returns an array of dictionaries representing the results
"""
dictOut = []
for r in result:
descriptions = r.keys
for i in r.fetchall():
# WARNING: this can generate errors for some stupid reason
# in both oracle and mysql.
entry = {}
for index in range(0, len(descriptions)):
# WARNING: Oracle returns table names in CAP!
if isinstance(descriptions[index], (str, bytes)):
keyName = decodeBytesToUnicodeConditional(
descriptions[index], condition=PY3)
else:
keyName = descriptions[index]
if isinstance(i[index], (str, bytes)):
entry[
keyName.lower()] = decodeBytesToUnicodeConditional(
i[index], condition=PY3)
else:
entry[keyName.lower()] = i[index]
dictOut.append(entry)
r.close()
return dictOut
def runtime(self):
"""
_runtime_
Scram runtime command
"""
if self.projectArea is None:
msg = "Scram Runtime called with Project Area not set"
self.stdout = msg
self.stderr = ""
return 1
try:
proc = subprocess.Popen(
["/bin/bash"], shell=True, cwd=self.projectArea,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
)
except Exception as ex:
msg = "Error thrown while invoking subprocess for scram runtime\n"
msg += "%s\n" % str(ex)
msg += "Opening subprocess shell in %s" % self.projectArea
self.stdout = msg
self.stderr = ""
return 1
# write via process writer method
self.procWriter(proc, self.library_path)
self.procWriter(proc, self.preCommand())
self.procWriter(proc, "%s ru -sh\n" % self.command)
self.procWriter(proc, """if [ "$?" -ne "0" ]; then exit 4; fi\n""")
self.procWriter(proc, "eval `%s ru -sh`\n" % self.command)
self.stdout, self.stderr = proc.communicate()
self.stdout = decodeBytesToUnicodeConditional(self.stdout, condition=PY3)
self.stderr = decodeBytesToUnicodeConditional(self.stderr, condition=PY3)
if proc.returncode == 0:
for l in self.stdout.split(";\n"):
if l.strip() == "":
continue
if l.strip().startswith('unset'):
continue
l = l.replace("export ", "")
try:
var, val = l.split("=", 1)
except ValueError as ex:
raise ValueError("Couldn't split line: %s" % l)
self.runtimeEnv[var] = val
if self.test_mode:
# ensure that runtime env isnt empty in test mode
# as that will cause an exception in __call__
self.runtimeEnv['TEST_MODE'] = 1
self.code = proc.returncode
self.lastExecuted = "eval `%s ru -sh`" % self.command
return proc.returncode
def methodTest(verb,
url,
request_input={},
accept='text/json',
contentType=None,
output={},
expireTime=0,
secure=False,
secureParam={}):
data, code, content_type, response = makeRequest(url, request_input, verb,
accept, contentType,
secure, secureParam)
data = decodeBytesToUnicodeConditional(data, condition=PY3)
keyMap = {
'code': code,
'data': data,
'type': content_type,
'response': response
}
for key, value in viewitems(output):
msg = 'Got a return %s != %s (got %s, type %s) (expected %s, type %s)' \
% (keyMap[key], value, keyMap[key], type(keyMap[key]), value, type(value))
assert keyMap[key] == value, msg
expires = response.getheader('Expires')
if expireTime != 0:
timeStamp = make_rfc_timestamp(expireTime)
assert expires == timeStamp, \
'Expires header incorrect (%s) != (%s)' % (expires, timeStamp)
return data, expires
def to_string(self, data):
"""
Since json.dumps returns unicode in py3 and bytes in py2 (it behaves
returning "native str" in both versions), then we do the same with all
the data that is not GeneratorType, dict nor list by calling str(data)
Moreover, we need to properly decode bytes.
:returns: "native str" (unicode in py3, bytes in py2)
"""
if isinstance(data, GeneratorType):
return self.json(data)
if isinstance(data, dict) or isinstance(data, list):
return json.dumps(data)
if not isinstance(data, bytes):
return str(data)
return decodeBytesToUnicodeConditional(data, condition=PY3)
def setCondorChirpAttrDelayed(self,
key,
value,
compress=False,
maxLen=5120):
"""
_setCondorChirpAttrDelayed_
Util to call condor_chirp and publish the key/value pair
"""
if compress:
value = zipEncodeStr(value, maxLen=maxLen)
# bytes object is not JSON serializable in Python3
value = decodeBytesToUnicodeConditional(value, condition=PY3)
# construct condor_chirp binary location from CONDOR_CONFIG
# Note: This works when we do not use containers.
condor_chirp_bin = None
condor_config = os.getenv('CONDOR_CONFIG', None)
if condor_config:
condor_config_dir = os.path.dirname(condor_config)
condor_chirp_bin = os.path.join(
condor_config_dir, 'main/condor/libexec/condor_chirp')
# If the above fails, look for the executable in the environment
# This is the usual case for containers
if not condor_chirp_bin or not os.path.isfile(condor_chirp_bin):
condor_chirp_bin = getFullPath("condor_chirp")
if condor_chirp_bin and os.access(condor_chirp_bin, os.X_OK):
args = [
condor_chirp_bin, 'set_job_attr_delayed', key,
json.dumps(value)
]
subprocess.call(args)
else:
if condor_chirp_bin and not os.access(condor_chirp_bin, os.X_OK):
msg = 'condor_chirp was found in: %s, but it was not an executable.' % condor_chirp_bin
else:
msg = 'condor_chirp was not found in the system.'
self.logger.warning(msg)
return
def formatList(self, result):
"""
Returns a flat array with the results.
Ideally used for single column queries
"""
listOut = []
for r in result:
descriptions = r.keys
for i in r.fetchall():
for index in range(0, len(descriptions)):
if isinstance(i[index], (str, bytes)):
listOut.append(
decodeBytesToUnicodeConditional(i[index],
condition=PY3))
else:
listOut.append(i[index])
r.close()
return listOut
def _check_str(argname, val, rx, custom_err=None):
"""
This is not really check val is ASCII.
2021 09: we are now using version 17.4.0 -> we do not need to convert to
bytes here anymore, we are using a recent verison of cherrypy.
We merged the funcionality of _check_str and _check_ustr into a single function
:type val: str or bytes (only utf8 encoded string) in py3, unicode or str in py2
:type rx: regex, compiled from native str (unicode in py3, bytes in py2)
"""
val = decodeBytesToUnicodeConditional(val, condition=PY3)
val = encodeUnicodeToBytesConditional(val, condition=PY2)
# `val` should now be a "native str" (unicode in py3, bytes in py2)
# here str has not been redefined. it is default `str` in both py2 and py3.
if not isinstance(val, str) or not rx.match(val):
raise InvalidParameter(
return_message(
"Incorrect '%s' parameter %s %s" % (argname, type(val), val),
custom_err))
return val
def check_name(dbname):
match = re.match("^[a-z0-9_$()+-/]+$", urllib.parse.unquote_plus(dbname))
if not match:
msg = '%s is not a valid database name'
raise ValueError(msg % urllib.parse.unquote_plus(dbname))
def check_server_url(srvurl):
good_name = srvurl.startswith('http://') or srvurl.startswith('https://')
if not good_name:
raise ValueError('You must include http(s):// in your servers address')
PY3_STR_DECODER = lambda x: decodeBytesToUnicodeConditional(x, condition=PY3)
class Document(dict):
"""
Document class is the instantiation of one document in the CouchDB
"""
def __init__(self, id=None, inputDict=None):
"""
Initialise our Document object - a dictionary which has an id field
inputDict - input dictionary to initialise this instance
"""
inputDict = inputDict or {}
dict.__init__(self)
self.update(inputDict)