def test_get_config_data():
configdata = get_config_data(None)
assert configdata is None
configdata = get_config_data('/tmp/asdxz.ini')
assert configdata is None
tests_configini = '%s/config.ini' % TESTSDIR
tests_configdata = get_config_data(tests_configini)
assert tests_configdata is not None
prod_configdata = get_config_data(framework_config())
assert prod_configdata is not None
assert tests_configdata == prod_configdata
def get_logdir(fw_cfg):
log_writeable = True
if not fw_cfg:
cfgini = framework_config()
fw_cfg = get_config_data(cfgini)
logdir = '%s' % framework_dir()
if fw_cfg and 'LOGGING' in fw_cfg:
fwconf = fw_cfg['LOGGING']
if 'logFolder' in fwconf and fwconf['logFolder'] and os.path.isdir(logdir):
logdir = '%s/%s' % (logdir, fwconf['logFolder'])
try:
if not os.path.exists(logdir):
os.makedirs(logdir)
except:
log_writeable = False
try:
if log_writeable:
from pathlib import Path
testfile = '%s/%d' % (logdir, int(time.time()))
Path(testfile).touch()
if os.path.exists(testfile):
os.remove(testfile)
else:
log_writeable = False
except:
log_writeable = False
return log_writeable, logdir
def ini_logging_config(fwconfigfile):
"""logging config"""
from processor.helper.config.config_utils import framework_config, get_config_data, framework_dir, get_base_log_dir
if not fwconfigfile:
fwconfigfile = framework_config()
fw_cfg = get_config_data(fwconfigfile)
log_config = {
"level": logging.INFO,
"propagate": True,
"size": 10,
"backups": 10,
"db": None,
'logpath': None
}
if fw_cfg and 'LOGGING' in fw_cfg:
base_log_dir = get_base_log_dir()
if base_log_dir is None:
base_log_dir = framework_dir()
logwriteable, logpath = get_logdir(fw_cfg, base_log_dir)
if logwriteable and logpath:
log_config['logpath'] = logpath
fwconf = fw_cfg['LOGGING']
log_config['level'] = get_loglevel(fwconf)
log_config['size'] = fwconf.getint('size') if 'size' in fwconf else 10
log_config['backups'] = fwconf.getint('backups') if 'backups' in fwconf else 10
log_config['propagate'] = fwconf.getboolean('propagate') if 'propagate' in fwconf else True
log_config['db'] = fwconf['dbname'] if 'dbname' in fwconf else None
return log_config
def logging_fw(fwconfigfile, dbargs):
"""Framework file logging"""
global FWLOGFILENAME, dbhandler
fwlogfile = '%Y%m%d-%H%M%S'
if not fwconfigfile:
fwconfigfile = framework_config()
fw_cfg = get_config_data(fwconfigfile)
log_config = {
"level": logging.INFO,
"propagate": True,
"size": 10,
"backups": 10,
"db": None
}
unittest = os.getenv('UNITTEST', "false")
if fw_cfg and 'LOGGING' in fw_cfg:
fwconf = fw_cfg['LOGGING']
# log_config['level'] = logging.getLevelName(fwconf['level']) \
# if 'level' in fwconf and fwconf['level'] else logging.INFO
# Fixed issue of only CRITICAL Logs were getting store
log_config['level'] = fwconf['level'].upper() \
if 'level' in fwconf and fwconf['level'] else logging.INFO
log_config['size'] = fwconf.getint('size') if 'size' in fwconf else 10
log_config['backups'] = fwconf.getint('backups') if 'backups' in fwconf else 10
log_config['propagate'] = fwconf.getboolean('propagate') if 'propagate' in fwconf \
else True
log_config['db'] = fwconf['dbname'] if 'dbname' in fwconf else None
log_config['dburl'] = get_dburl() if 'dbname' in fwconf and unittest != "true" else None
level = os.getenv('LOGLEVEL', None)
loglevel = level if level and level in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG'] \
else log_config['level']
logformat = '%(asctime)s(%(module)s:%(lineno)4d) - %(message)s'
# logging.basicConfig(level=loglevel, format=logformat)
logging.basicConfig(format=logformat)
logger = logging.getLogger(__name__)
logger.propagate = log_config['propagate']
logger.setLevel(loglevel)
# logpath = '%s/log/' % get_logdir(fw_cfg)
_, logpath = get_logdir(fw_cfg)
FWLOGFILENAME = '%s/%s.log' % (logpath, datetime.datetime.today().strftime(fwlogfile))
handler = RotatingFileHandler(
FWLOGFILENAME,
maxBytes=1024 * 1024 * log_config['size'],
backupCount=log_config['backups']
)
handler.setFormatter(logging.Formatter(logformat))
# handler.setLevel(log_config['level'])
handler.setLevel(loglevel)
logger.addHandler(handler)
# print(log_config)
if log_config['db'] and unittest != "true" and dbargs:
dblogformat = '%(asctime)s-%(message)s'
dbhandler = MongoDBHandler(log_config['dburl'], log_config['db'])
dbhandler.setFormatter(logging.Formatter(dblogformat))
dbhandler.setLevel(loglevel)
logger.addHandler(dbhandler)
return logger
def valid_config_ini(config_ini):
""" Valid config ini path and load the file and check """
error = None
if exists_file(config_ini):
config_data = get_config_data(config_ini)
if config_data:
# TODO: can also check for necessary sections and fields.
pass
else:
error = "Configuration(%s) INI file is invalid, correct it and try again!" % config_ini
else:
error = "Configuration(%s) INI file does not exist!" % config_ini
return error
def validator_main(arg_vals=None, delete_rundata=True):
"""
Main driver utility for running validator tests
The arg_vals, if passed should be array of string. A set of examples are:
1) arg_vals = ['container1'] - Use container1 to process test files from filesystem
2) args_vals = ['container1', '--db'] - Use container1 to process test documents from database.
When arg_vals is None, the argparse library parses from sys.argv array list.
The successful argument parsing initializes the system for the run.
On exit will run cleanup. The return values of this main entry function are as:
0 - Success, tests executed.
1 - Failure, Tests execution error.
2 - Exception, missing config.ini, Mongo connection failure or http connection exception,
the tests execution could not be started or completed.
"""
cmd_parser = argparse.ArgumentParser(
"prancer",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''\
Example: prancer collection1
Runs the prancer framework based on the configuration files available in collection1 folder
''')
cmd_parser.add_argument('-v',
'--version',
action='version',
version=("Prancer %s" % __version__),
help='Show prancer version')
cmd_parser.add_argument(
'container',
metavar='collection',
action='store',
help=
'The name of the folder which contains the collection of files related to one scenario'
)
cmd_parser.add_argument(
'--db',
action='store',
default=None,
choices=['NONE', 'SNAPSHOT', 'FULL'],
help=
'''NONE - Database will not be used, all the files reside on file system,
SNAPSHOT - Resource snapshots will be stored in db, everything else will be on file system,
FULL - tests, configurations, outputs and snapshots will be stored in the database'''
)
cmd_parser.add_argument('--crawler',
action='store_true',
default=False,
help='Crawls and generates snapshot files only')
cmd_parser.add_argument('--test',
action='store',
default=None,
help='Run a single test in NODB mode')
cmd_parser.add_argument('--customer',
action='store',
default=None,
help='customer name for config')
cmd_parser.add_argument(
'--connector',
action='store',
default=None,
help=
'specify the name of the connector which you want to run from the collection'
)
cmd_parser.add_argument(
'--branch',
action='store',
default=None,
help=
'specify the name of the branch to populate snapshots, for the filesystem connector'
)
args = cmd_parser.parse_args(arg_vals)
retval = 2
set_customer()
cfg_error, config_ini = search_config_ini()
if cfg_error:
return retval
if args.customer:
set_customer(args.customer)
if args.db:
if args.db.upper() in DBVALUES:
args.db = DBVALUES.index(args.db.upper())
else:
args.db = DBVALUES.index(SNAPSHOT)
else:
nodb = config_value(TESTS, DBTESTS)
if nodb and nodb.upper() in DBVALUES:
args.db = DBVALUES.index(nodb.upper())
else:
args.db = DBVALUES.index(SNAPSHOT)
if args.test:
args.db = DBVALUES.index(NONE)
# Check if we want to run in NO DATABASE MODE
if args.db:
# returns the db connection handle and status, handle is ignored.
from processor.database.database import init_db, TIMEOUT
_, db_init_res = init_db()
if not db_init_res:
msg = "Mongo DB connection timed out after %d ms, check the mongo server, exiting!....." % TIMEOUT
console_log(msg, currentframe())
return retval
# Check the log directory and also check if it is writeable.
from processor.logging.log_handler import init_logger, get_logdir, default_logging, add_file_logging
fw_cfg = get_config_data(framework_config())
log_writeable, logdir = get_logdir(fw_cfg, framework_dir())
if not log_writeable:
console_log(
'Logging directory(%s) is not writeable, exiting....' % logdir,
currentframe())
return retval
# Alls well from this point, check container exists in the directory configured
retval = 0
logger = init_logger(args.db, framework_config())
# logger = add_file_logging(config_ini)
logger.info("START: Argument parsing and Run Initialization. Version %s",
__version__)
from processor.connector.snapshot import populate_container_snapshots
from processor.connector.validation import run_container_validation_tests
from processor.crawler.master_snapshot import generate_container_mastersnapshots
try:
from processor_enterprise.notifications.notification import check_send_notification
except:
check_send_notification = lambda container, db: None
logger.info("Command: '%s %s'",
sys.executable.rsplit('/', 1)[-1], ' '.join(sys.argv))
try:
from processor.helper.config.rundata_utils import init_currentdata, \
delete_currentdata, put_in_currentdata
# Delete the rundata at the end of the script as per caller, default is True.
if delete_rundata:
atexit.register(delete_currentdata)
init_currentdata()
logger.info("Using Framework dir: %s", framework_dir())
logger.info("Args: %s", args)
logger.debug("Running tests from %s.", DBVALUES[args.db])
fs = True if args.db > DBVALUES.index(SNAPSHOT) else False
put_in_currentdata('jsonsource', fs)
put_in_currentdata(DBTESTS, args.db)
put_in_currentdata('container', args.container)
# if args.db == DBVALUES.index(FULL):
# from processor.logging.log_handler import get_dblogger
# log_name = get_dblogger()
# if log_name:
# pid = open('/tmp/pid_%s' % os.getpid(), 'w')
# pid.write(log_name)
# pid.close()
if args.customer:
put_in_currentdata(CUSTOMER, args.customer)
if args.test:
put_in_currentdata(SINGLETEST, args.test)
else:
put_in_currentdata(SINGLETEST, False)
if args.connector:
put_in_currentdata("connector", args.connector)
if args.branch:
put_in_currentdata("branch", args.branch)
if not args.db:
retval = 0 if container_exists(args.container) else 2
if retval:
logger.critical(
"Container(%s) is not present in Framework dir: %s",
args.container,
framework_dir(),
extra={"type": "critical"})
# TODO: Log the path the framework looked for.
return retval
if args.crawler:
# Generate snapshot files from here.
generate_container_mastersnapshots(args.container, fs)
else:
# Normal flow
snapshot_status = populate_container_snapshots(args.container, fs)
logger.debug(json.dumps(snapshot_status, indent=2))
if snapshot_status:
status = run_container_validation_tests(
args.container, fs, snapshot_status)
retval = 0 if status else 1
else:
retval = 1
check_send_notification(args.container, args.db)
except (Exception, KeyboardInterrupt) as ex:
logger.error("Execution exception: %s", ex)
print(traceback.format_exc())
retval = 2
return retval