def __init__(self, command, arguments, environs, workingDir, state, timeout, stdout=None, stderr=None, displayName=None, **kwargs):
"""Create an instance of the process wrapper.
@param command: The full path to the command to execute
@param arguments: A list of arguments to the command
@param environs: A dictionary of environment variables (key, value) for the process context execution
@param workingDir: The working directory for the process
@param state: The state of the process (L{pysys.constants.FOREGROUND} or L{pysys.constants.BACKGROUND}
@param timeout: The timeout in seconds to be applied to the process
@param stdout: The full path to the filename to write the stdout of the process
@param stderr: The full path to the filename to write the sdterr of the process
@param displayName: Display name for this process
"""
CommonProcessWrapper.__init__(self, command, arguments, environs, workingDir,
state, timeout, stdout, stderr, displayName, **kwargs)
# private instance variables
self.__hProcess = None
self.__hThread = None
self.__tid = None
self.__lock = threading.Lock() # to protect access to the fields that get updated
# set the stdout|err file handles
self.fStdout = 'nul'
self.fStderr = 'nul'
try:
if stdout is not None: self.fStdout = _stringToUnicode(stdout)
except Exception:
log.info("Unable to create file to capture stdout - using the null device")
try:
if stderr is not None: self.fStderr = _stringToUnicode(stderr)
except Exception:
log.info("Unable to create file to capture stdout - using the null device")
def wait(self, interval):
"""Wait for a specified period of time.
@param interval: The time interval in seconds to wait
"""
log.info('Waiting for %0.1f seconds'%interval)
time.sleep(interval)
def getDriver(database):
"""Downloads database drivers to the staging area, and return the full path to the driver."""
download_dir = os.path.join(REMOTE_STAGING, 'drivers')
target = os.path.join(download_dir, database.driver)
if not os.path.exists(download_dir): os.makedirs(download_dir)
if not os.path.exists(target):
log.info('Downloading driver %s' % database.driver)
database.copyDriver(download_dir)
return target
def __init__(self, requests_queue, results_queue, poll_timeout=5, **kwds):
"""Class constructor.
@param requests_queue: Reference to the threadpool's request queue
@param results_queue: Reference to the threadpool's results queue
@param poll_timeout: The timeout when trying to obtain a request from the request queue
@param kwds: Variable arguments to be passed to the threading.Thread constructor
"""
threading.Thread.__init__(self, **kwds)
log.info("[%s] Creating thread for test execution" % self.getName())
self.setDaemon(1)
self._requests_queue = requests_queue
self._results_queue = results_queue
self._poll_timeout = poll_timeout
self._dismissed = threading.Event()
self.start()
def clean(self):
descriptors = createDescriptors(self.arguments, None, [], [], None,
self.workingDir)
for descriptor in descriptors:
if self.all:
if sys.version_info >= (3, ):
cache = os.path.join(os.path.dirname(descriptor.module),
"__pycache__")
if os.path.exists(cache):
log.info("Deleting pycache: " + cache)
self.purgeDirectory(cache, True)
else:
path = descriptor.module + ".pyc"
try:
mode = os.stat(path)[stat.ST_MODE]
if stat.S_ISLNK(mode):
os.unlink(path)
if stat.S_ISREG(mode):
os.remove(path)
log.info("Deleting compiled module: " + path)
except Exception:
log.debug("Error deleting compiled module: " + path)
pathToDelete = os.path.join(descriptor.output, self.outsubdir)
if os.path.exists(pathToDelete):
log.info("Deleting output directory: " + pathToDelete)
self.purgeDirectory(pathToDelete, True)
else:
log.debug("Output directory does not exist: " + pathToDelete)
def change_stream_listen(self, log, args):
connStr = args[0]
log.info("Listen and log")
# Connect to mongo
client = MongoClient(host=connStr)
db = client.get_default_database()
coll = db.get_collection('test_coll')
total = 0.0
count = 0
# Connect to elastic
# elastic_conn_str = [self.project.ELASTIC_URL]
# es = Elasticsearch(elastic_conn_str)
# index_name = 'mongosearch'
# doc_type = 'test'
# es.indices.delete(index=index_name, ignore=[400, 404])
self.thread_started = True
try:
for changed_doc in coll.watch([{
'$match': {
'operationType': 'insert'
}
}]):
# doc = changed_doc['fullDocument']
# doc.pop('_id')
# res = es.index(index=index_name, doc_type=doc_type, id=count, body=doc)
# count += 1
# log.info(res['result'])
self.records_to_receive -= 1
log.info(self.records_to_receive)
except PyMongoError as ex:
# The ChangeStream encountered an unrecoverable error or the
# resume attempt failed to recreate the cursor.
print(ex)
def clean(self):
Project.findAndLoadProject(outdir=self.outsubdir)
descriptors = createDescriptors(self.arguments,
None, [], [],
None,
self.workingDir,
expandmodes=False)
supportMultipleModesPerRun = Project.getInstance().getProperty(
'supportMultipleModesPerRun', True)
for descriptor in descriptors:
if self.all:
modulepath = os.path.join(descriptor.testDir,
descriptor.module)
cache = os.path.join(os.path.dirname(modulepath),
"__pycache__")
if os.path.isdir(cache):
log.info("Deleting pycache: " + cache)
deletedir(cache)
else:
log.debug('__pycache__ does not exist: %s', cache)
path = modulepath + ".pyc"
if os.path.exists(path):
log.info("Deleting compiled Python module: " + path)
os.remove(path)
else:
log.debug('.pyc does not exist: %s', path)
for mode in (descriptor.modes or [None]):
pathToDelete = os.path.join(descriptor.testDir,
descriptor.output, self.outsubdir)
if os.path.isabs(self.outsubdir
): # must delete only the selected testcase
pathToDelete += "/" + descriptor.id
if supportMultipleModesPerRun and mode:
pathToDelete += '~' + mode
if os.path.exists(pathToDelete):
log.info("Deleting output directory: " + pathToDelete)
deletedir(pathToDelete)
else:
log.debug("Output directory does not exist: " +
pathToDelete)
def makeTest(self,
input=None,
output=None,
reference=None,
descriptor=None,
testclass=None,
module=None,
group="",
constantsImport=None,
basetestImport=None,
basetest=None,
teststring=None):
if input == None: input = DEFAULT_INPUT
if output == None: output = DEFAULT_OUTPUT
if reference == None: reference = DEFAULT_REFERENCE
if descriptor == None: descriptor = DEFAULT_DESCRIPTOR[0]
if testclass == None: testclass = DEFAULT_TESTCLASS
if module == None: module = DEFAULT_MODULE
if constantsImport == None:
constantsImport = "from pysys.constants import *"
if basetestImport == None:
basetestImport = "from pysys.basetest import BaseTest"
if basetest == None: basetest = "BaseTest"
log.info("Creating testcase %s ..." % self.testId)
try:
os.makedirs(os.path.join(self.testdir, self.testId))
log.info("Created directory %s" %
os.path.join(self.testdir, self.testId))
except OSError:
log.info("Error creating testcase " +
os.path.join(self.testdir, self.testId) +
" - directory already exists")
return
else:
os.makedirs(os.path.join(self.testdir, self.testId, input))
log.info("Created directory %s " %
os.path.join(self.testdir, self.testId, input))
os.makedirs(os.path.join(self.testdir, self.testId, output))
log.info("Created directory %s " %
os.path.join(self.testdir, self.testId, output))
os.makedirs(os.path.join(self.testdir, self.testId, reference))
log.info("Created directory %s " %
os.path.join(self.testdir, self.testId, reference))
descriptor_fp = open(
os.path.join(self.testdir, self.testId, descriptor), "w")
descriptor_fp.write(DESCRIPTOR_TEMPLATE %
(self.type, group, testclass, module))
descriptor_fp.close()
log.info("Created descriptor %s " %
os.path.join(self.testdir, self.testId, descriptor))
testclass_fp = open(
os.path.join(self.testdir, self.testId, "%s.py" % module), "w")
if teststring == None:
testclass_fp.write(
TEST_TEMPLATE %
(constantsImport, basetestImport, testclass, basetest))
else:
testclass_fp.write(teststring)
testclass_fp.close()
log.info("Created test class module %s " %
os.path.join(self.testdir, self.testId, "%s.py" % module))
def parseArgs(self, args, printXOptions=None):
# add any default args first; shlex.split does a great job of providing consistent parsing from str->list,
# but need to avoid mangling \'s on windows; since this env var will be different for each OS no need for consistent win+unix behaviour
if os.getenv('PYSYS_DEFAULT_ARGS', ''):
log.info('Using PYSYS_DEFAULT_ARGS = %s' %
os.environ['PYSYS_DEFAULT_ARGS'])
args = shlex.split(os.environ['PYSYS_DEFAULT_ARGS'].replace(
os.sep, os.sep * 2 if os.sep == '\\' else os.sep)) + args
printLogsDefault = PrintLogs.ALL
if '--ci' in args:
# to ensure identical behaviour, set these as if on the command line
# (printLogs we don't set here since we use the printLogsDefault mechanism to allow it to be overridden
# by CI writers and/or the command line; note that setting --mode=ALL would be incorrect if
# supportMultipleModesPerRun=false but that's a legacy options so we raise an exception later if this happened)
args = [
'--purge', '--record', '-j0', '--type=auto', '--mode=ALL',
'-XcodeCoverage'
] + args
printLogsDefault = PrintLogs.FAILURES
try:
optlist, self.arguments = getopt.gnu_getopt(
args, self.optionString, self.optionList)
except Exception:
log.warn("Error parsing command line arguments: %s" %
(sys.exc_info()[1]))
sys.exit(1)
log.debug('PySys arguments: tests=%s options=%s', self.arguments,
optlist)
EXPR1 = re.compile("^[\w\.]*=.*$")
EXPR2 = re.compile("^[\w\.]*$")
printLogs = None
ci = False
defaultAbortOnError = None
logging.getLogger('pysys').setLevel(logging.INFO)
# as a special case, set a non-DEBUG log level for the implementation of assertions
# so that it doesn't get enabled with -vDEBUG only -vassertions=DEBUG
# as it is incredibly verbose and slow and not often useful
logging.getLogger('pysys.assertions').setLevel(logging.INFO)
for option, value in optlist:
if option in ("-h", "--help"):
self.printUsage(printXOptions)
elif option in ['--ci']:
continue # handled above
elif option in ("-r", "--record"):
self.record = True
elif option in ("-p", "--purge"):
self.purge = True
elif option in ("-v", "--verbosity"):
verbosity = value
if '=' in verbosity:
loggername, verbosity = value.split('=')
assert not loggername.startswith(
'pysys.'
), 'The "pysys." prefix is assumed and should not be explicitly specified'
if loggername.startswith('python:'):
loggername = loggername[len('python:'):]
assert not loggername.startswith(
'pysys'
), 'Cannot use python: with pysys.*' # would produce a duplicate log handler
# in the interests of performance and simplicity we normally only add the pysys.* category
logging.getLogger(loggername).addHandler(
pysys.internal.initlogging.pysysLogHandler)
else:
loggername = 'pysys.' + loggername
else:
loggername = None
if verbosity.upper() == "DEBUG":
verbosity = logging.DEBUG
elif verbosity.upper() == "INFO":
verbosity = logging.INFO
elif verbosity.upper() == "WARN":
verbosity = logging.WARN
elif verbosity.upper() == "CRIT":
verbosity = logging.CRITICAL
else:
log.warn('Invalid log level "%s"' % verbosity)
sys.exit(1)
if loggername is None:
# when setting global log level to a higher level like WARN etc we want to affect stdout but
# not necessarily downgrade the root level (would make run.log less useful and break
# some PrintLogs behaviour)
stdoutHandler.setLevel(verbosity)
if verbosity == logging.DEBUG:
logging.getLogger('pysys').setLevel(logging.DEBUG)
else:
# for specific level setting we need the opposite - only change stdoutHandler if we're
# turning up the logging (since otherwise it wouldn't be seen) but also change the specified level
logging.getLogger(loggername).setLevel(verbosity)
elif option in ("-a", "--type"):
self.type = value
if self.type not in ["auto", "manual"]:
log.warn(
"Unsupported test type - valid types are auto and manual"
)
sys.exit(1)
elif option in ("-t", "--trace"):
self.trace = value
elif option in ("-i", "--include"):
self.includes.append(value)
elif option in ("-e", "--exclude"):
self.excludes.append(value)
elif option in ("-c", "--cycle"):
try:
self.cycle = int(value)
except Exception:
print(
"Error parsing command line arguments: A valid integer for the number of cycles must be supplied"
)
sys.exit(1)
elif option in ("-o", "--outdir"):
value = os.path.normpath(value)
if os.path.isabs(value) and not value.startswith('\\\\?\\'):
value = fromLongPathSafe(toLongPathSafe(value))
self.outsubdir = value
elif option in ("-m", "--mode", "--modeinclude"):
self.modeinclude = self.modeinclude + [
x.strip() for x in value.split(',')
]
elif option in ["--modeexclude"]:
self.modeexclude = self.modeexclude + [
x.strip() for x in value.split(',')
]
elif option in ["-n", "-j", "--threads"]:
N_CPUS = multiprocessing.cpu_count()
if value.lower() == 'auto': value = '0'
if value.lower().startswith('x'):
self.threads = max(1, int(float(value[1:]) * N_CPUS))
else:
self.threads = int(value)
if self.threads <= 0:
self.threads = int(
os.getenv('PYSYS_DEFAULT_THREADS', N_CPUS))
elif option in ("-b", "--abort"):
defaultAbortOnError = str(value.lower() == 'true')
elif option in ["-g", "--progress"]:
self.progress = True
elif option in ["--printLogs"]:
printLogs = getattr(PrintLogs, value.upper(), None)
if printLogs is None:
print(
"Error parsing command line arguments: Unsupported --printLogs value '%s'"
% value)
sys.exit(1)
elif option in ["-X"]:
if '=' in value:
key, value = value.split('=', 1)
else:
key, value = value, 'true'
# best not to risk unintended consequences with matching of other types, but for boolean
# it's worth it to resolve the inconsistent behaviour of -Xkey=true and -Xkey that existed until 1.6.0,
# and because getting a bool where you expected a string is a bit more likely to give an exception
# and be noticed that getting a string where you expected a boolean (e.g. the danger of if "false":)
if value.lower() == 'true':
value = True
elif value.lower() == 'false':
value = False
self.userOptions[key] = value
elif option in ("-y", "--validateOnly"):
self.userOptions['validateOnly'] = True
elif option in ("-G", "--grep"):
self.grep = value
else:
print("Unknown option: %s" % option)
sys.exit(1)
# log this once we've got the log levels setup
log.debug('PySys is installed at: %s; python from %s',
os.path.dirname(pysys.__file__), sys.executable)
# retained for compatibility, but PYSYS_DEFAULT_ARGS is a better way to achieve the same thing
if os.getenv('PYSYS_PROGRESS', '').lower() == 'true':
self.progress = True
# special hidden dict of extra values to pass to the runner, since we can't change
# the public API now
self.userOptions['__extraRunnerOptions'] = {
'progressWritersEnabled': self.progress,
'printLogs': printLogs,
'printLogsDefault':
printLogsDefault, # to use if not provided by a CI writer or cmdline
}
# load project AFTER we've parsed the arguments, which opens the possibility of using cmd line config in
# project properties if needed
Project.findAndLoadProject(outdir=self.outsubdir)
if defaultAbortOnError is not None:
setattr(Project.getInstance(), 'defaultAbortOnError',
defaultAbortOnError)
if '--ci' in args and not Project.getInstance().getProperty(
'supportMultipleModesPerRun', True):
raise UserError(
'Cannot use --ci option with a legacy supportMultipleModesPerRun=false project'
)
descriptors = createDescriptors(self.arguments,
self.type,
self.includes,
self.excludes,
self.trace,
self.workingDir,
modeincludes=self.modeinclude,
modeexcludes=self.modeexclude,
expandmodes=True)
descriptors.sort(
key=lambda d: [d.executionOrderHint, d._defaultSortKey])
# No exception handler above, as any createDescriptors failure is really a fatal problem that should cause us to
# terminate with a non-zero exit code; we don't want to run no tests without realizing it and return success
if self.grep:
regex = re.compile(self.grep, flags=re.IGNORECASE)
descriptors = [
d for d in descriptors
if (regex.search(d.id) or regex.search(d.title))
]
runnermode = self.modeinclude[0] if len(
self.modeinclude
) == 1 else None # used when supportMultipleModesPerRun=False
return self.record, self.purge, self.cycle, runnermode, self.threads, self.outsubdir, descriptors, self.userOptions
