def create_loghandler(self):
from ZConfig.components.logger import loghandler
path = self.section.path
max_bytes = self.section.max_size
old_files = self.section.old_files
when = self.section.when
interval = self.section.interval
if path == "STDERR":
if max_bytes or old_files:
raise ValueError("cannot rotate STDERR")
handler = loghandler.StreamHandler(sys.stderr)
elif path == "STDOUT":
if max_bytes or old_files:
raise ValueError("cannot rotate STDOUT")
handler = loghandler.StreamHandler(sys.stdout)
elif when or max_bytes or old_files or interval:
if not old_files:
raise ValueError("old-files must be set for log rotation")
if when:
if max_bytes:
raise ValueError("can't set *both* max_bytes and when")
if not interval:
interval = 1
handler = loghandler.TimedRotatingFileHandler(
path, when=when, interval=interval,
backupCount=old_files)
elif max_bytes:
handler = loghandler.RotatingFileHandler(
path, maxBytes=max_bytes, backupCount=old_files)
else:
raise ValueError(
"max-bytes or when must be set for log rotation")
else:
handler = loghandler.FileHandler(path)
return handler
def create_loghandler(self):
from ZConfig.components.logger import loghandler
path = self.section.path
if path == "STDERR":
handler = loghandler.StreamHandler(sys.stderr)
elif path == "STDOUT":
handler = loghandler.StreamHandler(sys.stdout)
else:
handler = loghandler.FileHandler(path)
return handler
def __init__(self):
self.event_logger = logging.getLogger()
# We log events to the root logger, which is backed by a
# "StartupHandler" log handler. The "StartupHandler" buffers
# log messages. When the "real" loggers are set up, we flush
# accumulated messages in StartupHandler's buffers to the real
# logger.
formatter = logging.Formatter(
"%(asctime)s %(levelname)s %(name)s %(message)s",
"%Y-%m-%d %H:%M:%S")
self.debug_handler = loghandler.StreamHandler()
self.debug_handler.setFormatter(formatter)
self.debug_handler.setLevel(logging.WARN)
self.startup_handler = loghandler.StartupHandler()
self.event_logger.addHandler(self.debug_handler)
self.event_logger.addHandler(self.startup_handler)
def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
if "zeo_client" in os.environ and "zeo_server" in os.environ:
_print("conflicting options: --zeo_client and --zeo_server")
sys.exit(1)
instance_home = os.environ['INSTANCE_HOME']
os.environ.setdefault('EVENT_LOG_FILE',
os.path.join(log_directory, 'zLOG.log'))
os.environ.setdefault('EVENT_LOG_SEVERITY', '-300')
# For numpy parallel-primitives such as numpy.dot() that is used in
# testReceiptRecognition for example.
os.environ.setdefault('OMP_NUM_THREADS', '3')
_print("Loading Zope ... \n")
_start = time.time()
import Testing
# the above import changes cfg.testinghome. Reset it to where our
# custom_zodb.py can be found. This must be done before importing
# ZopeTestCase below (Leo: I hate import side-effects with a passion).
import App.config
cfg = App.config.getConfiguration()
# Zope do not call setDefaultBehaviors() for testing instance as it does for
# a normal instance so `cfg.skip_ownership_checking = True` would not work
# here...
import AccessControl
AccessControl.setDefaultBehaviors(
# ownerous: ERP5 default (zope.conf: `skip-ownership-checking true`)
False,
# authenticated: Zope default (zope.conf: `skip_authentication_checking`)
True,
# verbose: Zope default
False)
cfg.testinghome = instance_home
cfg.instancehome = instance_home
from Zope2.Startup.datatypes import DBTab
cfg.dbtab = DBTab({}, {})
App.config.setConfiguration(cfg)
if WIN:
products_home = os.path.join(real_instance_home, 'Products')
import Products
Products.__path__.insert(0, products_home)
else:
products_home = os.path.join(instance_home, 'Products')
from Testing.ZopeTestCase import layer, PortalTestCase, ZopeLite
_apply_patches = layer._deferred_setup.pop(0)[0]
assert _apply_patches.__name__ == '_apply_patches'
# Set debug mode after importing ZopeLite that resets it to 0
cfg.debug_mode = debug
from ZConfig.components.logger import handlers, logger, loghandler
import logging
root_logger = logging.getLogger()
# On recent Zope, ZopeTestCase does not have any logging facility.
# So we must emulate the usual Zope startup code to catch log messages.
from ZConfig.matcher import SectionValue
section = SectionValue(
{
'dateformat': '%Y-%m-%d %H:%M:%S',
'format':
'%(asctime)s.%(msecs)03d %(levelname)s %(name)s %(message)s',
'level': logging.INFO,
'path': os.environ['EVENT_LOG_FILE'],
'max_size': None,
'old_files': None,
'when': None,
'interval': None,
'formatter': None,
}, None, None)
section.handlers = [handlers.FileHandlerFactory(section)]
root_logger.handlers = []
logger.EventLogFactory(section)()
# Make sure that locally overridden python modules are used
sys.path.insert(0, os.path.join(real_instance_home, 'lib', 'python'))
# allow unit tests of our Products or business templates to be reached.
sys.path += glob(os.path.join(products_home, '*', 'tests'))
bt5_path_list = os.environ['erp5_tests_bt5_path'].split(',')
bt5_path_list += [os.path.join(path, "*") for path in bt5_path_list]
for path in bt5_path_list:
path = os.path.join(path, '*', 'TestTemplateItem')
sys.path += glob(path)
sys.path += glob(os.path.join(path, 'portal_components'))
sys.path += (os.path.join(real_instance_home,
'tests'), tests_home, instance_home)
# change current directory to the test home, to create zLOG.log in this dir.
os.chdir(tests_home)
from Products.ERP5Type.patches import noZopeHelp
from OFS.Application import AppInitializer
AppInitializer.install_session_data_manager = lambda self: None
# import ERP5TypeTestCase before calling layer.ZopeLite.setUp
# XXX What if the unit test itself uses 'onsetup' ? We should be able to call
# remaining 'onsetup' hooks just before executing the test suite.
from Products.ERP5Type.tests.ERP5TypeTestCase import \
ProcessingNodeTestCase, ZEOServerTestCase, dummy_setUp, dummy_tearDown
# Since we're not using the zope.testing testrunner, we need to set up
# the layer ourselves
# FIXME: We should start using Zope layers. Our setup will probably
# be faster and we could drop most of this code we currently maintain
# ourselves
layer.ZopeLite.setUp() # this will import custom_zodb.py
def assertFalse():
assert False
layer.onsetup = assertFalse
ZopeLite._theApp._p_jar.close()
ZopeLite._theApp = None
from Products.ERP5Type.tests.utils import DbFactory
root_db_name, = cfg.dbtab.databases.keys()
db_factory = DbFactory(root_db_name)
db_factory.addMountPoint('/')
TestRunner = unittest.TextTestRunner
import Lifetime
from Zope2.custom_zodb import Storage, save_mysql, \
node_pid_list, neo_cluster, zeo_server_pid, wcfs_server
def shutdown(signum, frame, signum_set=set()):
Lifetime.shutdown(0)
signum_set.add(signum)
if node_pid_list is None and len(signum_set) > 1:
# in case of ^C, a child should also receive a SIGHUP from the parent,
# so we merge the first 2 different signals in a single exception
signum_set.remove(signal.SIGHUP)
else:
raise KeyboardInterrupt
if signal.getsignal(signal.SIGINT) is not signal.SIG_IGN:
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGHUP, shutdown)
coverage_config = os.environ.get('coverage', None)
if coverage_config:
coverage_process = coverage(config_file=coverage_config)
coverage_process.start()
try:
save = int(os.environ.get('erp5_save_data_fs', 0))
load = int(os.environ.get('erp5_load_data_fs', 0))
dummy = save and (int(os.environ.get('update_business_templates', 0))
or not load)
if zeo_server_pid == 0:
suite = ZEOServerTestCase('asyncore_loop')
elif node_pid_list is None or not test_list:
suite = ProcessingNodeTestCase('processing_node')
if not (dummy or load):
_print(
'WARNING: either --save or --load should be used because static'
' files are only reloaded by the node installing business'
' templates.')
else:
if dummy:
# Skip all tests and monkeypatch PortalTestCase to skip
# afterSetUp/beforeTearDown.
ERP5TypeTestLoader._testMethodPrefix = 'dummy_test'
PortalTestCase.setUp = dummy_setUp
PortalTestCase.tearDown = dummy_tearDown
elif debug:
# Hack the profiler to run only specified test methods,
# and wrap results when running in debug mode.
class DebugTextTestRunner(TestRunner):
def _makeResult(self):
result = super(DebugTextTestRunner, self)._makeResult()
return DebugTestResult(result)
TestRunner = DebugTextTestRunner
loader = ERP5TypeTestLoader()
if run_only:
ERP5TypeTestLoader.filter_test_list = [
re.compile(x).search for x in run_only.split(',')
]
suite = loader.loadTestsFromNames(test_list)
if run_only:
ERP5TypeTestLoader.filter_test_list = None
if node_pid_list is None:
result = suite()
else:
if not test_list:
root_logger.handlers.append(
loghandler.StreamHandler(sys.stderr))
_print('done (%.3fs)\n' % (time.time() - _start))
result = TestRunner(verbosity=verbosity).run(suite)
transaction.commit()
except:
import traceback
print "runUnitTestList Exception : %r" % (traceback.print_exc(), )
# finally does not expect opened transaction, even in the
# case of a Ctrl-C.
transaction.abort()
raise
finally:
ProcessingNodeTestCase.unregisterNode()
db_factory.close()
Storage.close()
if node_pid_list is not None:
# Wait that child processes exit. Stop ZEO storage (if any) after all
# other nodes disconnected.
for pid in node_pid_list:
os.kill(pid, signal.SIGHUP)
for pid in node_pid_list:
os.waitpid(pid, 0)
if zeo_server_pid:
os.kill(zeo_server_pid, signal.SIGHUP)
os.waitpid(zeo_server_pid, 0)
if neo_cluster:
neo_cluster.stop()
if wcfs_server is not None:
# Stop WCFS server (if any) after all Zope nodes are stopped and
# disconnected from it.
wcfs_server.stop()
if coverage_config:
coverage_process.stop()
coverage_process.save()
coverage_process.html_report()
if save and save_mysql:
save_mysql(verbosity)
return result
def factory():
return loghandler.StreamHandler(sys.stdout)
def runUnitTestList(test_list, verbosity=1, debug=0, run_only=None):
if "zeo_client" in os.environ and "zeo_server" in os.environ:
_print("conflicting options: --zeo_client and --zeo_server")
sys.exit(1)
instance_home = os.environ['INSTANCE_HOME']
os.environ.setdefault('EVENT_LOG_FILE', os.path.join(tests_home, 'zLOG.log'))
os.environ.setdefault('EVENT_LOG_SEVERITY', '-300')
_print("Loading Zope ... \n")
_start = time.time()
import Testing
# the above import changes cfg.testinghome. Reset it to where our
# custom_zodb.py can be found. This must be done before importing
# ZopeTestCase below (Leo: I hate import side-effects with a passion).
import App.config
cfg = App.config.getConfiguration()
cfg.testinghome = instance_home
cfg.instancehome = instance_home
from Zope2.Startup.datatypes import DBTab
cfg.dbtab = DBTab({}, {})
App.config.setConfiguration(cfg)
if WIN:
products_home = os.path.join(real_instance_home, 'Products')
import Products
Products.__path__.insert(0, products_home)
else:
products_home = os.path.join(instance_home, 'Products')
from Testing.ZopeTestCase import layer, PortalTestCase, ZopeLite
_apply_patches = layer._deferred_setup.pop(0)[0]
assert _apply_patches.__name__ == '_apply_patches'
from ZConfig.components.logger import handlers, logger, loghandler
import logging
root_logger = logging.getLogger()
# On recent Zope, ZopeTestCase does not have any logging facility.
# So we must emulate the usual Zope startup code to catch log messages.
from ZConfig.matcher import SectionValue
section = SectionValue({'dateformat': '%Y-%m-%d %H:%M:%S',
'format': '%(asctime)s.%(msecs)03d %(levelname)s %(name)s %(message)s',
'level': logging.INFO,
'path': os.environ['EVENT_LOG_FILE'],
'max_size': None,
'old_files': None,
'when': None,
'interval': None,
'formatter': None,
},
None, None)
section.handlers = [handlers.FileHandlerFactory(section)]
root_logger.handlers = []
logger.EventLogFactory(section)()
# allow unit tests of our Products or business templates to be reached.
product_test_list = glob(os.path.join(products_home, '*', 'tests'))
sys.path.extend(product_test_list)
erp5_tests_bt5_path = os.environ.get('erp5_tests_bt5_path',
os.path.join(instance_home, 'bt5'))
bt5_path_list = erp5_tests_bt5_path.split(",")
bt5_test_list = []
project_bt5_test_list = []
for bt5_path in bt5_path_list:
bt5_test_list.extend(glob(os.path.join(bt5_path,'*','TestTemplateItem')))
bt5_test_list.extend(glob(os.path.join(bt5_path,'*','TestTemplateItem',
'portal_components')))
# also suport instance_home/bt5/project_bt5/*
project_bt5_test_list.extend(glob(os.path.join(bt5_path, '*', '*',
'TestTemplateItem')))
project_bt5_test_list.extend(glob(os.path.join(bt5_path, '*', '*',
'TestTemplateItem',
'portal_components')))
sys.path.extend(bt5_test_list)
sys.path.extend(project_bt5_test_list)
sys.path.extend((os.path.join(real_instance_home, 'tests'), tests_home))
sys.path.append(instance_home)
# Make sure that locally overridden python modules are used
sys.path.insert(0, os.path.join(real_instance_home, 'lib', 'python'))
# XXX Allowing to load modules from here is a wrong idea. use the above path
# instead.
# Add tests_framework_home as first path element.
# this allows to bypass psyco by creating a dummy psyco module
# it is then possible to run the debugger by "import pdb; pdb.set_trace()"
sys.path.insert(0, tests_framework_home)
# change current directory to the test home, to create zLOG.log in this dir.
os.chdir(tests_home)
from Products.ERP5Type.patches import noZopeHelp
from OFS.Application import AppInitializer
AppInitializer.install_session_data_manager = lambda self: None
# import ERP5TypeTestCase before calling layer.ZopeLite.setUp
# XXX What if the unit test itself uses 'onsetup' ? We should be able to call
# remaining 'onsetup' hooks just before executing the test suite.
from Products.ERP5Type.tests.ERP5TypeTestCase import \
ProcessingNodeTestCase, ZEOServerTestCase, dummy_setUp, dummy_tearDown
# Since we're not using the zope.testing testrunner, we need to set up
# the layer ourselves
# FIXME: We should start using Zope layers. Our setup will probably
# be faster and we could drop most of this code we currently maintain
# ourselves
layer.ZopeLite.setUp() # this will import custom_zodb.py
def assertFalse():
assert False
layer.onsetup = assertFalse
ZopeLite._theApp._p_jar.close()
ZopeLite._theApp = None
from Products.ERP5Type.tests.utils import DbFactory
root_db_name, = cfg.dbtab.databases.keys()
DbFactory(root_db_name).addMountPoint('/')
TestRunner = backportUnittest.TextTestRunner
import Lifetime
from Zope2.custom_zodb import Storage, save_mysql, \
node_pid_list, neo_cluster, zeo_server_pid
def shutdown(signum, frame, signum_set=set()):
Lifetime.shutdown(0)
signum_set.add(signum)
if node_pid_list is None and len(signum_set) > 1:
# in case of ^C, a child should also receive a SIGHUP from the parent,
# so we merge the first 2 different signals in a single exception
signum_set.remove(signal.SIGHUP)
else:
raise KeyboardInterrupt
if signal.getsignal(signal.SIGINT) is not signal.SIG_IGN:
signal.signal(signal.SIGINT, shutdown)
signal.signal(signal.SIGHUP, shutdown)
coverage_config = os.environ.get('coverage', None)
if coverage_config:
coverage_process = coverage(config_file=coverage_config)
coverage_process.start()
try:
save = int(os.environ.get('erp5_save_data_fs', 0))
load = int(os.environ.get('erp5_load_data_fs', 0))
dummy = save and (int(os.environ.get('update_business_templates', 0))
or not load)
if zeo_server_pid == 0:
suite = ZEOServerTestCase('asyncore_loop')
elif node_pid_list is None or not test_list:
suite = ProcessingNodeTestCase('processing_node')
if not (dummy or load):
_print('WARNING: either --save or --load should be used because static'
' files are only reloaded by the node installing business'
' templates.')
else:
if dummy:
# Skip all tests and monkeypatch PortalTestCase to skip
# afterSetUp/beforeTearDown.
ERP5TypeTestLoader._testMethodPrefix = 'dummy_test'
PortalTestCase.setUp = dummy_setUp
PortalTestCase.tearDown = dummy_tearDown
elif debug:
# Hack the profiler to run only specified test methods,
# and wrap results when running in debug mode.
class DebugTextTestRunner(TestRunner):
def _makeResult(self):
result = super(DebugTextTestRunner, self)._makeResult()
return DebugTestResult(result)
TestRunner = DebugTextTestRunner
loader = ERP5TypeTestLoader()
if run_only:
ERP5TypeTestLoader.filter_test_list = [re.compile(x).search for x in
run_only.split(',')]
suite = loader.loadTestsFromNames(test_list)
if run_only:
ERP5TypeTestLoader.filter_test_list = None
if node_pid_list is None:
result = suite()
else:
if not test_list:
root_logger.handlers.append(loghandler.StreamHandler(sys.stderr))
_print('done (%.3fs)\n' % (time.time() - _start))
result = TestRunner(verbosity=verbosity).run(suite)
transaction.commit()
except:
import traceback
print "runUnitTestList Exception : %r" % (traceback.print_exc(),)
# finally does not expect opened transaction, even in the
# case of a Ctrl-C.
transaction.abort()
raise
finally:
ProcessingNodeTestCase.unregisterNode()
Storage.close()
if node_pid_list is not None:
# Wait that child processes exit. Stop ZEO storage (if any) after all
# other nodes disconnected.
for pid in node_pid_list:
os.kill(pid, signal.SIGHUP)
for pid in node_pid_list:
os.waitpid(pid, 0)
if zeo_server_pid:
os.kill(zeo_server_pid, signal.SIGHUP)
os.waitpid(zeo_server_pid, 0)
if neo_cluster:
neo_cluster.stop()
if coverage_config:
coverage_process.stop()
coverage_process.save()
coverage_process.html_report()
if save:
os.chdir(instance_home)
if save_mysql:
save_mysql(verbosity)
if suite.__class__ not in (ProcessingNodeTestCase, ZEOServerTestCase):
# Static files are modified by the node installing business templates,
# i.e. by the node running the unit test. There is no point saving them
# on a ZEO server, or on nodes that only process activities: this has to
# be done manually.
if verbosity:
_print('Dumping static files...\n')
live_instance_path = os.environ.get('live_instance_path')
for static_dir in static_dir_list:
if os.path.islink(static_dir):
continue
if live_instance_path:
backup_path = os.path.join(live_instance_path, static_dir)
else:
backup_path = static_dir + '.bak'
try:
shutil.rmtree(backup_path)
except OSError, e:
if e.errno != errno.ENOENT:
raise
os.rename(static_dir, backup_path)
elif node_pid_list is not None:
_print('WARNING: No static files saved. You will have to do it manually.')
