def connect_db(self,
user,
db_server='',
db_name='',
db_user='',
db_passwd='',
shared_db=False):
"""
Connect to database.
Must know the IP + Database, the user and if the DB is shared or not,
to be able to decrypt the password.
"""
# Get the path to DB.XML
db_file = self.project.get_user_info(user, 'db_config')
if not db_file:
logError(
'Database: Null DB.XML file for user `{}`! Cannot connect!'.
format(user))
return False
c_time = time.time()
usr_server = '{}_{}_{}'.format(user, db_server, db_name)
# Existing connection ?
if self.connections.get(usr_server):
# This is a very fresh connection !
if c_time - self.connections[usr_server]['dt'] < 1.0:
return self.connections[usr_server]['conn']
# Use shared DB or not ?
use_shared_db = self.project.get_user_info(user, 'use_shared_db')
if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
use_shared_db = True
else:
use_shared_db = False
# DB.xml + Shared DB parser
users_groups = self.project._parse_users_and_groups()
shared_db_path = users_groups['shared_db_cfg']
db_config = DBParser(user, db_file, shared_db_path,
use_shared_db).db_config
# Try to use the default pair
if not db_server and not db_name:
db_server, db_name, db_user, db_passwd, _ = db_config[
'default_server']
# Need to magically identify the correct key; the first pair is from private DB.xml;
if shared_db:
# Shared DB is True
encr_key = users_groups.get('shared_db_key', 'Luxoft')
else:
# Fallback to shared DB
encr_key = None
# Decode database password
db_password = self.project.decrypt_text(user, db_passwd, encr_key)
logDebug('User `{}` connecting to {} MySQL `{} @ {} / {} : {}[...]`.'.\
format(user, 'shared' if encr_key else 'user', db_user, db_server,\
db_name, db_passwd[:3]))
try:
conn = MySQLdb.connect(host=db_server,
db=db_name,
user=db_user,
passwd=db_password)
conn.autocommit = False
except MySQLdb.Error as exp_err:
logError('MySQL error for user `{}`: `{} - {}`!'.\
format(user, exp_err.args[0], exp_err.args[1]))
return False
# Keep connection
self.connections[usr_server] = {'conn': conn, 'dt': c_time}
return conn
def load_config(self, usr, force=False):
"""
Read DB Config File for 1 user.
"""
if not os.path.isdir(userHome(usr) + '/twister/config'):
logError(
'Report Server: Cannot find Twister for user `{}` !'.format(
usr))
return False
# Get the path to DB.XML
db_file = self.project.get_user_info(usr, 'db_config')
if not db_file:
logError(
'Report Server: Null DB.XML file for user `{}`! Nothing to do!'
.format(usr))
return False
# Current timer
c_time = time.time()
# Create database parser IF necessary, or FORCED, or old connection...
if force or (usr not in self.db_parser) or (
c_time - self.timers.get(usr, 0) > 5.0):
# logDebug('Rebuilding fields, reports and redirects for user `{}`...'.format(usr))
self.timers[usr] = c_time
self.db_parser[usr] = True
self.glob_fields[usr] = OrderedDict()
self.glob_reports[usr] = OrderedDict()
self.glob_redirects[usr] = OrderedDict()
self.glob_links[usr] = [{
'name': 'Home',
'link': 'Home',
'type': 'link'
}]
# DB.xml + Shared DB parser
users_groups = self.project._parse_users_and_groups()
shared_db_path = users_groups['shared_db_cfg']
db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][usr][
'roles']
# Use shared DB or not ?
use_shared_db = self.project.get_user_info(usr, 'use_shared_db')
if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
use_shared_db = True
else:
use_shared_db = False
dbp = DBParser(usr, db_file, shared_db_path, use_shared_db)
self.db_servers = dbp.db_config['servers']
report_queries = dbp.get_reports(db_cfg_role)
del dbp
for host_db in report_queries:
self.glob_fields[usr].update(report_queries[host_db]['fields'])
self.glob_redirects[usr].update(
report_queries[host_db]['redirects'])
for report_name, report_data in report_queries[host_db][
'reports'].iteritems():
srv_name = report_data['srv_name']
# Add the new server name in reports ?
if srv_name not in self.glob_reports[usr]:
self.glob_reports[usr][srv_name] = OrderedDict()
# Add the report inside the server
self.glob_reports[usr][srv_name][report_name] = report_data
# There are more types of reports:
# Normal links, like Home, Help and other normal reports
# Redirect links, that don't contain reports
# Folders, that don't go anywhere, are just labels for reports
for rname, rval in report_queries[host_db][
'reports'].iteritems():
# Shared report ?
srv_name = rval['srv_name']
# Link name used in reports
link = ('S&' if srv_name == 'Shared' else 'U&') + rname
# Server, user, password
db_server, db_name, db_user, _, _ = host_db
srv_db = '{} server: {}/ {}/ {}'.format(
srv_name, db_server, db_name, db_user)
report = {
'name': rname,
'link': link,
'type': 'link',
'folder': rval.get('folder', ''),
'srvr': srv_db
}
self.glob_links[usr].append(report)
for rname, rval in report_queries[host_db][
'redirects'].iteritems():
link = ('S&'
if rval['srv_name'] == 'Shared' else 'U&') + rname
self.glob_links[usr].append({
'name': rname,
'link': link,
'type': 'redir',
'folder': ''
})
# Append the Help link at the end
self.glob_links[usr].append({
'name': 'Help',
'link': 'Help',
'type': 'link'
})
return True
def project_data(self, user, save_to_db=False):
"""
Collect all data from a user, using the DB.XML for the current project.
If save to DB is active, the function will also save.
"""
# Get the path to DB.XML
db_file = self.project.get_user_info(user, 'db_config')
if not db_file:
logError(
'Database: Null DB.XML file for user `{}`! Nothing to do!'.
format(user))
return False
# DB.xml + Shared DB parser
users_groups = self.project._parse_users_and_groups()
shared_db_path = users_groups['shared_db_cfg']
db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][user]['roles']
# Use shared DB or not ?
use_shared_db = self.project.get_user_info(user, 'use_shared_db')
if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
use_shared_db = True
else:
use_shared_db = False
dbp = DBParser(user, db_file, shared_db_path, use_shared_db)
all_inserts = dbp.get_inserts(db_cfg_role)
del dbp
if not all_inserts:
logWarning('Database: Cannot use inserts defined for user `{}`!'.\
format(user))
return False
# UserScript cache
usr_script_cache_s = {} # Suite
usr_script_cache_p = {} # Project
# DbSelect cache
db_select_cache_s = {} # Suite
db_select_cache_p = {} # Project
conn, curs = None, None
# Pre-calculated data
all_data = []
for subst_data in self.static_project_data(user):
# Setup and Teardown files will not be saved to database!
if subst_data.get('setup_file') == 'true' or \
subst_data.get('teardown_file') == 'true':
logDebug("Ignoring `{}` file, because it's setup or teardown.".\
format(subst_data['twister_tc_name']))
continue
# Pre-Suite or Post-Suite files will not be saved to database
if subst_data.get('Pre-Suite') or subst_data.get('Post-Suite'):
continue
# If file has iterators, the iteration save==Failed and the status was Failed
if subst_data.get('_cfg_files') and subst_data.get('iterationNr') \
and subst_data.get('iterationSave') == 'failed' and \
subst_data['twister_tc_status'] == 'PASS':
continue
# For every host, build correct data...
for host_db in all_inserts:
c_inserts = all_inserts[host_db]['inserts']
c_fields = all_inserts[host_db]['fields']
shared_db = all_inserts[host_db]['shared_db']
db_server, db_name, db_user, db_passwd, _ = host_db
conn = self.connect_db(user, db_server, db_name, db_user,\
db_passwd, shared_db=shared_db)
if not conn:
continue
curs = conn.cursor()
# Escape all unicodes variables before SQL Statements!
subst_data = {k: MySQLdb.escape_string(v) if isinstance(v, unicode) else v for \
k, v in subst_data.iteritems()}
# For every query of the current host
for query in c_inserts:
# All variables of type `UserScript` must be replaced
# with the script result
try:
user_script_fields = re.findall(
'(\$.+?)[,\.\'"\s]', query)
except Exception:
user_script_fields = []
for field in user_script_fields:
field = field[1:]
# Invalid field ?
if field not in c_fields:
continue
# If the field is not `UserScript`, ignore it
if c_fields.get(field, {}).get('type') != 'UserScript':
continue
# Field level: Suite or Project
lvl = c_fields.get(field)['level']
# Get Script Path, or null string
u_script = subst_data.get(field, '')
if not u_script:
query = query.replace('$' + field, '')
continue
# Execute this script based on level
if lvl == 'Project':
if u_script not in usr_script_cache_p:
# Execute script and use result
res = execScript(u_script)
# logDebug('Database: UserScript for `{}` was executed at '\
# 'LVL `{}`.'.format(user, lvl))
# Save result in cache
usr_script_cache_p[u_script] = res
else:
# Get script result from cache
res = usr_script_cache_p[u_script]
# Execute for every suite
else:
suite_id = subst_data['twister_suite_id']
if suite_id not in usr_script_cache_s:
usr_script_cache_s[suite_id] = {}
if u_script not in usr_script_cache_s[suite_id]:
# Execute script and use result
res = execScript(u_script)
# logDebug('Database: UserScript for `{}` was executed at '\
# 'LVL `{}`.'.format(user, lvl))
# Save result in cache
usr_script_cache_s[suite_id][u_script] = res
else:
# Get script result from cache
res = usr_script_cache_s[suite_id][u_script]
# Replace UserScript with with real Script results
if not res:
res = ''
query = query.replace('$' + field, res)
# Adding user script fields
subst_data[field] = res
# All variables of type `DbSelect` must be replaced with the SQL result
try:
auto_insert_fields = re.findall('(@.+?@)', query)
except Exception:
auto_insert_fields = []
for field in auto_insert_fields:
# Delete the @ character
field = field[1:-1]
# Invalid field ?
if field not in c_fields:
continue
# Get Auto Query, or null string
u_query = c_fields.get(field, {}).get('query', '')
# Field level: Suite, Project, or Testcase
lvl = c_fields.get(field)['level']
if not u_query:
logError('User `{}`, file `{}`: Cannot build query! Field `{}` '\
'is not defined in the fields section!'.format(user, subst_data['file'], field))
return False
# Execute User Query based on level
if lvl == 'Project':
if u_query not in db_select_cache_p:
# Execute User Query
curs.execute(u_query)
q_value = curs.fetchone()[0]
# logDebug('Database: DbSelect for `{}` was executed at '\
# 'LVL `{}`.'.format(user, lvl))
# Save result in cache
db_select_cache_p[u_query] = q_value
else:
# Get script result from cache
q_value = db_select_cache_p[u_query]
# Execute User Query for every suite
elif lvl == 'Suite':
suite_id = subst_data['twister_suite_id']
if suite_id not in db_select_cache_s:
db_select_cache_s[suite_id] = {}
if u_query not in db_select_cache_s[suite_id]:
# Execute User Query
curs.execute(u_query)
q_value = curs.fetchone()[0]
# logDebug('Database: DbSelect for `{}` was executed at '\
# 'LVL `{}`.'.format(user, lvl))
# Save result in cache
db_select_cache_s[suite_id][u_query] = q_value
else:
# Get script result from cache
q_value = db_select_cache_s[suite_id][u_query]
else:
# Execute User Query
curs.execute(u_query)
q_value = curs.fetchone()[0]
# logDebug('Database: DbSelect for `{}` was executed at '\
# 'LVL `TestCase`.'.format(user))
# Replace @variables@ with real Database values
query = query.replace('@' + field + '@', str(q_value))
# Adding auto insert fields
subst_data[field] = str(q_value)
# String Template
tmpl = Template(query)
# Fix None values to NULL
subst_data = {
k: 'NULL' if v is None else v
for k, v in subst_data.iteritems()
}
# Build complete query
try:
query = tmpl.substitute(subst_data)
except Exception as exp_err:
logError('User `{}`, file `{}`: Cannot build query! \
Error on `{}`!'.format(user, subst_data['file'],
exp_err))
return False
# Save query in database ?
if save_to_db:
# Execute MySQL Query!
try:
curs.execute(query)
logDebug('Executed query\n\t``{}``\n\t on {} OK.'.
format(query.strip(), host_db))
conn.commit()
except MySQLdb.Error as exp_err:
logError('Error in query ``{}`` , for user \
`{}`!\n\t MySQL Error {}: {}!' .\
format(query, user, exp_err.args[0],\
exp_err.args[1]))
conn.rollback()
return False
# :: Debug ::
# import pprint ; pprint.pprint(subst_data)
# Append all data for current file
all_data.append(subst_data)
return all_data