def getBindingsConfig(self):
"""
Parse the bindings file that connects Roots from a config file, with SUTs.
"""
logFull('xmlparser:getBindingsConfig')
cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))
bindings = {}
if not os.path.isfile(cfg_file):
err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
logError(err)
return err
bind_xml = parseXML(self.user, cfg_file)
if bind_xml is None:
err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
return err
for binding in bind_xml.xpath('/root/binding'):
name = binding.find('name')
# Valid names ?
if name is None:
continue
name = name.text.strip()
if not name:
continue
bindings[name] = {}
# All binds cfg -> sut
for bind in binding.findall('bind'):
cfg = bind.get('config')
sut = bind.get('sut')
bindings[name][cfg] = sut
logDebug('Found `{}` bindings for user `{}`.'.format(len(bindings), self.user))
return bindings
def __init__(self, user, base_config='', files_config=''):
self.user = user
self.user_home = userHome(user)
if os.path.isfile(base_config):
base_config = localFs.read_user_file(user, base_config)
elif base_config and (type(base_config) == type('') or type(base_config) == type(u'')) \
and (base_config[0] == '<' and base_config[-1] == '>'):
pass
else:
raise Exception('Parser ERROR: Invalid config data : `{}`!'.format(base_config))
try:
self.xmlDict = etree.fromstring(base_config)
except Exception as e:
raise Exception('Parser ERROR: Cannot access XML config! `{}`'.format(e))
self.configTS = None
self.configHash = None
self.project_globals = {}
self.files_config = ''
self.updateConfigTS(files_config)
self.updateProjectGlobals()
def _connect(self):
# Searching for a free port in the safe range...
while 1:
free = False
port = random.randrange(59000, 60000)
try:
socket.create_connection((None, port), 1)
except:
free = True
if free: break
p_cmd = 'su {} -c "{} -u {}/plugins/ClearCaseSrv.py {}"'.format(self.user, sys.executable, TWISTER_PATH, port)
proc = subprocess.Popen(p_cmd, cwd='{}/twister'.format(userHome(self.user)), shell=True)
proc.poll()
time.sleep(0.5)
print('CC Srv for user `{}` launched on `127.0.0.1:{}` - PID `{}`.'.format(self.user, port, proc.pid))
try:
self.conn = xmlrpclib.ServerProxy('http://127.0.0.1:{}/'.format(port))
self.conn.hello()
except Exception as e:
print('Cannot connect to CC Srv on `127.0.0.1:{}` - `{}` !'.format(port, e))
proc.terminate()
self.conn = None
del self.data['clear_case_view']
return False
return True
def get_binding(self, fpath):
"""
Read a binding between a CFG and a SUT.
The result is XML.
"""
logFull('xmlparser:get_binding')
cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))
if not os.path.isfile(cfg_file):
err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
logError(err)
return err
bind_xml = parseXML(self.user, cfg_file)
if bind_xml is None:
err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
return err
# Find the old binding
found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))
if found:
xml_string = etree.tostring(found[0])
return xml_string.replace('binding>', 'root>')
else:
logWarning('*ERROR* Cannot find binding name `{}` for user {}!'.format(fpath, self.user))
return False
def del_binding(self, fpath):
"""
Delete a binding between a CFG and a SUT.
Return True/ False.
"""
logFull('xmlparser:del_binding')
cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))
if not os.path.isfile(cfg_file):
err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
logError(err)
return err
bind_xml = parseXML(self.user, cfg_file)
if bind_xml is None:
err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
return err
# Find the binding
found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))
# If found, delete it
if found:
bind_xml.remove(found[0])
logDebug('Removed binding `{}`, for user `{}`.'.format(fpath, self.user))
else:
err = '*WARN* Invalid binding `{}`, user `{}`! Cannot unbind!'.format(fpath, self.user)
# logDebug(err)
return False
return dumpXML(self.user, cfg_file, bind_xml)
def generate_xml(self, user, filename):
'''
Receives project file.
Creates testsuites.xml file by multiplying tests depending
on the suts number and eps.
'''
logDebug("CeParser: preparing to convert project file: `{}`,\
user `{}`.".format(filename, user))
data = self.project.read_project_file(user, filename)
if data.startswith('*ERROR*'):
logWarning(data)
return data
# try to parse the project file
try:
xml = etree.fromstring(data)
except Exception as e:
msg = "The file: '{}' it's not an xml file. Try again!\n{}".\
format(filename, e)
logDebug(msg)
return '*ERROR* ' + msg
self._expand_global_configs(user, xml)
self._expand_repeat_tag(xml)
self._expand_by_ep(user, xml)
repeated_dict = {}
self._change_ids(xml, repeated_dict)
self._resolve_dependencies(xml, repeated_dict)
self._inherit_suite_dependency(xml)
self._remove_empty_suites_tc(xml)
for suite in xml.findall('.//TestSuite'):
prop = suite.find('Property')
if prop:
suite.remove(prop)
# Final XML string
xml_string = etree.tostring(xml, pretty_print=True)
# write the xml file
xml_file = userHome(user) + '/twister/config/testsuites.xml'
resp = self.project.localFs.write_user_file(user, xml_file, xml_string,
'w')
if resp != True:
logError(resp)
return '*ERROR* ' + resp
logDebug('CeParser: Successfully generated: `{}`, user `{}`.'.\
format(xml_file, user))
return True
def help(self, usr=''):
"""
Help page.
"""
if not usr:
return '<br><b>Error! This link should be accessed by passing a username, eg: /help/some_user<b/>'
if not os.path.isdir(userHome(usr) + '/twister/config'):
return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(
usr)
self.load_config(usr) # Re-load all Database XML
output = Template(filename=TWISTER_PATH +
'/server/template/rep_help.htm')
return output.render(title='Help', usr=usr, links=self.glob_links[usr])
def updateConfigTS(self, files_config=''):
"""
Updates Test Suite Cofig file hash and recreates internal XML structure,
only if the XML file is changed.
The file number and suite number have to be unique.
"""
if files_config and (type(files_config) == type('') or type(files_config) == type(u'')) \
and (files_config[0] == '<' and files_config[-1] == '>'):
# This is pure XML data
config_ts = files_config
# Hash check the XML file, to see if is changed
newConfigHash = hashlib.md5(files_config).hexdigest()
else:
if not files_config or not os.path.isfile(files_config):
# Get path to Test-Suites XML from Master config
files_config = self.files_config
if files_config.startswith('~/'):
files_config = userHome(self.user) + files_config[1:]
if not os.path.isfile(files_config):
logError('User {}: Parser: Test-Suites XML file `{}` does '\
'not exist! Please check framework config XML file!'.format(self.user, files_config))
self.configTS = None
return -1
else:
config_ts = localFs.read_user_file(self.user, files_config)
# Hash check the XML file, to see if is changed
newConfigHash = hashlib.md5(config_ts).hexdigest()
if self.configHash != newConfigHash:
logDebug('User {}: Parser: Test-Suites XML file changed, '\
'rebuilding internal structure...\n'.format(self.user))
# Use the new hash
self.configHash = newConfigHash
# Create XML Soup from the new XML file
try:
self.configTS = etree.fromstring(config_ts)
except Exception:
logError('User {}: Parser ERROR: Cannot access Test-Suites XML data!'.format(self.user))
self.configTS = None
return -1
self.files_config = files_config
def __init__(self, user):
self.user = user
user_home = userHome(user)
if not os.path.isdir('{}/twister'.format(user_home)):
raise Exception('PluginParser: Cannot find Twister for user `{}`, '\
'in path `{}/twister`!'.format(user, user_home))
config_data = '{}/twister/config/plugins.xml'.format(user_home)
if not os.path.isfile(config_data):
raise Exception('PluginParser: Cannot find Plugins for user `{}`, '\
'in path `{}/twister/config`!'.format(user, user_home))
# Read directly from CE
xml_data = localFs.read_system_file(config_data)
self.config = OrderedDict()
try:
self.xmlTree = etree.fromstring(xml_data)
except Exception:
raise Exception('PluginParser: Cannot access plugins XML data!')
for plugin in self.xmlTree.xpath('Plugin'):
if not (plugin.xpath('name/text()') and plugin.xpath('pyfile') and plugin.xpath('jarfile')):
logWarning('User {}: PluginParser: Invalid config for plugin: `{}`!'.format(self.user, plugin))
continue
prop_keys = plugin.xpath('property/propname')
prop_vals = plugin.xpath('property/propvalue')
res = dict(zip([k.text for k in prop_keys], [v.text for v in prop_vals])) # Pack Name + Value
name = plugin.xpath('name')[0].text
self.config[name] = res
self.config[name]['jarfile'] = plugin.xpath('jarfile')[0].text.strip() \
if plugin.xpath('jarfile/text()') else ''
self.config[name]['pyfile'] = plugin.xpath('pyfile')[0].text.\
strip() if plugin.xpath('pyfile/text()') else ''
self.config[name]['status'] = plugin.xpath('status')[0].text.\
strip() if plugin.xpath('status/text()') else ''
def set_binding(self, fpath, content):
"""
Write a binding between a CFG and a SUT.
Return True/ False.
"""
logFull('xmlparser:set_binding')
cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user))
if not os.path.isfile(cfg_file):
err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user)
logError(err)
return err
bind_xml = parseXML(self.user, cfg_file)
if bind_xml is None:
err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user)
return err
# Find the old binding
found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath))
# If found, use it
if found:
found = found[0]
found.clear()
# Or create it
else:
found = etree.SubElement(bind_xml, 'binding')
name = etree.SubElement(found, 'name')
name.text = fpath
try:
replace_xml = etree.fromstring(content, parser)
except Exception:
err = '*ERROR* Invalid XML content, user {}! Cannot parse!'.format(self.user)
logWarning(err)
return err
for elem in replace_xml:
found.append(elem)
# Beautify XML ?
return etree.tostring(bind_xml, pretty_print=True)
def index(self, usr=''):
"""
The index page.
"""
if not usr:
users = self.project.list_users()
output = Template(filename=TWISTER_PATH +
'/server/template/rep_base.htm')
return output.render(title='Users',
usr='******' + '#'.join(users),
links=[])
if not os.path.isdir(userHome(usr) + '/twister/config'):
return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(
usr)
# FORCE re-load all Database XML on INDEX/ HOME links !
self.load_config(usr, True)
output = Template(filename=TWISTER_PATH +
'/server/template/rep_base.htm')
return output.render(title='Home', usr=usr, links=self.glob_links[usr])
def __init__(self, user):
self.user = user
user_home = userHome(user)
self.user_home = user_home
self.config = {}
if not os.path.isdir('{}/twister'.format(user_home)):
raise Exception('ClearCaseParser: Cannot find Twister for user `{}`, '\
'in path `{}/twister`!'.format(user, user_home))
config_data = '{}/twister/config/clearcaseconfig.xml'.format(user_home)
if not os.path.isfile(config_data):
raise Exception('ClearCaseParser: Cannot find Clearcase XML for user `{}`, '\
'in path `{}/twister/config`!'.format(user, user_home))
# Read directly from CE
xml_data = localFs.read_system_file(config_data)
try:
self.xmlTree = etree.fromstring(xml_data)
except Exception:
raise Exception('ClearCaseParser: Cannot access Clearcase XML data!')
def rep(self, report=None, usr=None, **kw):
"""
Reporting link.
"""
if not usr:
return '<br><b>Error! This link should be accessed by passing a username, eg: /rep/some_user<b/>'
if not os.path.isdir(userHome(usr) + '/twister/config'):
return '<br><b>Error! Username `{}` doesn\'t have a Twister config folder!</b>'.format(
usr)
self.load_config(usr) # Re-load all Database XML
cherrypy.response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
cherrypy.response.headers['Pragma'] = 'no-cache'
cherrypy.response.headers['Expires'] = 0
if not report:
raise cherrypy.HTTPRedirect('/error')
# The report name is like "U&..." or "S&..."
rlink = report
shared_db, report = rlink[0], rlink[2:]
if shared_db == 'S':
shared_db = True
srv_name = 'Shared'
else:
shared_db = False
srv_name = 'User'
if report in self.glob_redirects[usr]:
redirect_dict = self.glob_redirects[usr][report]['path']
raise cherrypy.HTTPRedirect(redirect_dict)
if srv_name not in self.glob_reports[usr]:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title='Missing server',
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg='Server `<b>{}</b>` is not defined!<br/><br/>'
'Go <a href="/report/home/{}">Home</a> ...'.format(
srv_name, usr))
if report not in self.glob_reports[usr][srv_name]:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title='Missing report',
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg='Report `<b>{}</b>` is not defined!<br/><br/>'
'Go <a href="/report/home/{}">Home</a> ...'.format(
report, usr))
logDebug('Prepare {} report `{}`, for user `{}`...'.format(
srv_name, report, usr))
# All info about the report, from DB XML
report_dict = self.glob_reports[usr][srv_name][report]
query = report_dict['sqlquery']
db_server, db_name, db_user, db_passwd, _ = self.db_servers[srv_name]
conn = self.project.dbmgr.connect_db(usr,
db_server,
db_name,
db_user,
db_passwd,
shared_db=shared_db)
if not conn:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title=report,
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg='Cannot connect to MySql server `{} / {}` !'.format(
db_server, db_name))
curs = conn.cursor()
# All variables that must be replaced in Query
vars_to_replace = re.findall('(@.+?@)', query)
# ------------------------------------------------------------------------------------------
# If the user didn't select fields YET :
# ------------------------------------------------------------------------------------------
if vars_to_replace and not cherrypy.request.params:
# Options are defined as: Type, Label, Data
u_options = OrderedDict()
for opt in vars_to_replace:
u_field = self.glob_fields[usr].get(opt.replace('@', ''))
this_option = {}
if not u_field:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(title=report, usr=usr, rlink=rlink,
links=self.glob_links[usr],
msg='Cannot build query!<br><br>Field `<b>{}</b>` '\
'is not defined in the fields section!'.format(opt.replace('@', '')))
this_option['type'] = u_field.get('type')
this_option['label'] = u_field.get('label')
# Field type : User Select
if this_option['type'] == 'UserSelect':
u_query = u_field.get('sqlquery')
if not u_query:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(title=report, usr=usr, rlink=rlink,
links=self.glob_links[usr],
msg='Cannot build query!<br><br>Field `<b>{}</b>` doesn\'t '\
'have a query!'.format(opt.replace('@', '')))
# Execute User Query
try:
curs.execute(u_query)
except MySQLdb.Error as err:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title=report,
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg=
'Error in query `{}`!<br><br><b>MySQL Error {}</b>: {}!'
.format(u_query, err.args[0], err.args[1]))
try:
u_vals = curs.fetchall()
except Exception as err:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title=report,
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg=
'Error in query `{}`!<br><br><b>Exception</b>: {}!'
.format(u_query, err))
# No data available
if not u_vals:
this_option['data'] = []
# Data has one column
elif len(u_vals[0]) == 1:
field_data = [(val[0], val[0]) for val in u_vals]
this_option['data'] = field_data
# Data has 2 or more columns
else:
field_data = [(str(val[0]),
str(val[0]) + ': ' + '| '.join(val[1:]))
for val in u_vals]
this_option['data'] = field_data
# Field type : User Text
elif this_option['type'] == 'UserText':
this_option['data'] = ''
else:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
title=report,
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
msg='Field `<b>{}</b>` is of unknown type: <b>{}</b>!'.
format(opt.replace('@', ''), this_option['type']))
u_options[opt] = this_option
output = Template(filename=TWISTER_PATH +
'/server/template/rep_base.htm')
return output.render(title=report,
usr=usr,
rlink=rlink,
links=self.glob_links[usr],
options=u_options)
# ------------------------------------------------------------------------------------------
# If the user has selected the fields :
# ------------------------------------------------------------------------------------------
ajax_links = []
# ... For normal Queries ...
for field in vars_to_replace:
# The value chosen by the user
u_select = cherrypy.request.params.get(field)
if not u_select:
u_select = ''
ajax_links.append(field + '=' + u_select)
# Replace @variables@ with user chosen value
query = query.replace(field, str(u_select))
ajax_links = sorted(set(ajax_links))
ajax_link = '/report/json/' + rlink + '/' + usr + '?' + '&'.join(
ajax_links)
user_choices = ('", '.join(ajax_links))
user_choices = user_choices.replace('@', '').replace('=', '="') + '"'
del ajax_links
try:
curs.execute(query)
except MySQLdb.Error as err:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(title=report, usr=usr, rlink=rlink, links=self.glob_links[usr],
msg='Error in query `{}`!<br><br>' \
'<b>MySQL Error {}</b>: {}!'.format(query, err.args[0], err.args[1]))
descr = [desc[0] for desc in curs.description]
# ... For Query Compare side by side, the table is double ...
query_compr = report_dict['sqlcompr']
if query_compr:
# All variables that must be replaced in Query
vars_to_replace = re.findall('(@.+?@)', query_compr)
for field in vars_to_replace:
# The value chosen by the user
u_select = cherrypy.request.params.get(field)
# Replace @variables@ with user chosen value
query_compr = query_compr.replace(field, str(u_select))
try:
curs.execute(query_compr)
except MySQLdb.Error as err:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(title=report, usr=usr, rlink=rlink, links=self.glob_links[usr],
msg='Error in query `{}`!<br><br>' \
'<b>MySQL Error {}</b>: {}!'.format(query_compr, err.args[0], err.args[1]))
headers_tot = [desc[0] for desc in curs.description]
# Update headers: must contain both headers.
descr = descr + ['vs.'] + headers_tot
# Write DEBUG
#DEBUG.write(report +' -> '+ user_choices +' -> '+ query_compr + '\n\n') ; DEBUG.flush()
output = Template(filename=TWISTER_PATH +
'/server/template/rep_base.htm')
return output.render(usr=usr,
title=report,
rlink=rlink,
links=self.glob_links[usr],
ajax_link=ajax_link,
user_choices=user_choices,
report=descr,
chart=report_dict['type'])
def _usr_service(self, user, oper='read'):
"""
Launch a user service.
"""
if oper not in ['read', 'write']:
logWarning(
'Invalid FS operation `{}`, for user `{}`! Will reset to "read".'
.format(oper, user))
oper = 'read'
# Must block here, so more users cannot launch Logs at the same time and lose the PID
with self._srv_lock:
# Try to re-use the logger server, if available
conn = self._services.get(user, {}).get('conn_' + oper, None)
if conn:
try:
conn.ping(data='Hello', timeout=30.0)
# logDebug('Reuse old {} User Service connection for `{}` OK.'.format(op, user))
return conn
except Exception as exp_err:
logWarning(
'Cannot reuse {} User Service for `{}`: `{}`.'.format(
oper, user, exp_err))
self._kill(user)
else:
logInfo('Launching a User Service for `{}`, the first time...'.
format(user))
port = None
# If the server is not available, search for a free port in the safe range...
while 1:
port = random.randrange(63000, 65000)
try:
socket.create_connection((None, port), 1)
except Exception:
break
p_cmd = 'su {} -c "{} -u {}/server/UserService.py {} {}"'.\
format(user, sys.executable, TWISTER_PATH, port, self.name)
proc = subprocess.Popen(p_cmd, cwd='{}/twister'.\
format(userHome(user)), shell=True, close_fds=True,\
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
proc.poll()
time.sleep(2.0)
config = {
'allow_pickle': True,
'allow_getattr': True,
'allow_setattr': True,
'allow_delattr': True
}
retry = 10
delay = 0.5
success = False
while retry > 0:
if success:
break
try:
stream_r = rpyc.SocketStream.connect('127.0.0.1',
port,
timeout=5.0)
conn_read = rpyc.connect_stream(stream_r, config=config)
conn_read.root.hello()
logDebug(
'Connected to User Service for `{}`, operation `read`.'
.format(user))
success = True
except Exception as exp_err:
logWarning('Cannot connect to User Service for `{}` - \
Exception: `{}`! Wait {}s...'.format(user, exp_err, delay))
if success:
try:
stream_w = rpyc.SocketStream.connect('127.0.0.1',
port,
timeout=5.0)
conn_write = rpyc.connect_stream(stream_w,
config=config)
conn_write.root.hello()
logDebug(
'Connected to User Service for `{}`, operation `write`.'
.format(user))
break
except Exception as exp_err:
logWarning('Cannot connect to User Service for `{}` \
- Exception: `{}`! Wait {}s...' .\
format(user, exp_err, delay))
success = False
time.sleep(delay)
retry -= 1
delay += 0.75
if not success:
logError(
'Error on starting User Service for `{}`!'.format(user))
return None
# Save the process inside the block. 99% of the time, this block is executed instantly!
self._services[user] = {
'proc': proc,
'conn_read': conn_read,
'conn_write': conn_write,
'port': port
}
logDebug(
'User Service for `{}` launched on `127.0.0.1:{}` - PID `{}`.'.
format(user, port, proc.pid))
return self._services[user].get('conn_' + oper, None)
def json(self, report, usr, **args):
"""
The report data, in json format.
"""
if not usr:
output = {'aaData':[], 'error':'Error! This link should be '\
'accessed by passing a username, eg: ' \
'/json/some_report/some_user'}
return json.dumps(output, indent=2)
if not os.path.isdir(userHome(usr) + '/twister/config'):
output = {'aaData':[], 'error':'Error! Username `{}` doesn\'t have '\
'a Twister config folder!'.format(usr)}
return json.dumps(output, indent=2)
self.load_config(usr) # Re-load all Database XML
cherrypy.response.headers[
'Content-Type'] = 'application/json; charset=utf-8'
cherrypy.response.headers[
'Cache-Control'] = 'no-cache, no-store, must-revalidate'
cherrypy.response.headers['Pragma'] = 'no-cache'
cherrypy.response.headers['Expires'] = 0
# The report name is like "U&..." or "S&..."
shared_db, report = report[0], report[2:]
if shared_db == 'S':
shared_db = True
srv_name = 'Shared'
else:
shared_db = False
srv_name = 'User'
if srv_name not in self.glob_reports[usr]:
output = {
'aaData': [],
'error':
'Server `{}` is not in the list of defined servers!'.format(
srv_name)
}
return json.dumps(output, indent=2)
if report not in self.glob_reports[usr][srv_name]:
output = {
'aaData': [],
'error':
'Report `{}` is not in the list of defined reports!'.format(
report)
}
return json.dumps(output, indent=2)
# All info about the report, from DB XML.
report_dict = self.glob_reports[usr][srv_name][report]
query = report_dict['sqlquery']
db_server, db_name, db_user, db_passwd, _ = self.db_servers[srv_name]
conn = self.project.dbmgr.connect_db(usr,
db_server,
db_name,
db_user,
db_passwd,
shared_db=shared_db)
if not conn:
output = Template(filename=TWISTER_PATH +
'/server/template/rep_error.htm')
return output.render(
links=self.glob_links[usr],
title=report,
usr=usr,
msg='Cannot connect to MySql server `{} / {}` !'.format(
db_server, db_name))
curs = conn.cursor()
# All variables that must be replaced in Query
vars_to_replace = re.findall('(@.+?@)', query)
for field in vars_to_replace:
# The value chosen by the user
u_select = cherrypy.request.params.get(field)
# Replace @variables@ with user chosen value
query = query.replace(field, str(u_select))
try:
curs.execute(query)
except MySQLdb.Error as err:
output = {'aaData':[], 'error':'Error in query `{}`! ' \
'MySQL Error {}: {}!'.format(query, err.args[0], err.args[1])}
return json.dumps(output, indent=2)
headers = [desc[0] for desc in curs.description]
rows = curs.fetchall()
del query
query_total = report_dict['sqltotal']
query_compr = report_dict['sqlcompr']
# ... Calculate SQL Query Total ...
if query_total:
# All variables that must be replaced in Query
vars_to_replace = re.findall('(@.+?@)', query_total)
for field in vars_to_replace:
# The value chosen by the user
u_select = cherrypy.request.params.get(field)
# Replace @variables@ with user chosen value
query_total = query_total.replace(field, str(u_select))
try:
curs.execute(query_total)
except MySQLdb.Error as err:
output = {'aaData':[], 'error':'Error in query total `{}`! ' \
'MySQL Error {}: {}!'.format(query_total, err.args[0], err.args[1])}
return json.dumps(output, indent=2)
headers_tot = [desc[0] for desc in curs.description]
rows_tot = curs.fetchall()
if len(headers) != len(headers_tot):
output = {
'aaData': [],
'error':
'The first query has {} columns and the second has {} columns!'
.format(len(headers), len(headers_tot))
}
return json.dumps(output, indent=2)
# Will calculate the new rows like this:
# The first column of the first query will not be changed
# The second row of the first query / the second row of the second query * 100
calc_rows = []
rows = {r[0]: r[1] for r in rows}
rows_tot = {r[0]: r[1] for r in rows_tot}
for rnb in rows_tot.keys():
if rnb in rows.keys():
# Calculate percent...
percent = '%.2f' % (float(rows[rnb]) / rows_tot[rnb] *
100.0)
# Using the header from Total, because it might be Null in the first query
calc_rows.append([rnb, float(percent)])
else:
calc_rows.append([rnb, 0.0])
# ... SQL Query Compare side by side ...
elif query_compr:
# All variables that must be replaced in Query
vars_to_replace = re.findall('(@.+?@)', query_compr)
for field in vars_to_replace:
# The value chosen by the user
u_select = cherrypy.request.params.get(field)
# Replace @variables@ with user chosen value
query_compr = query_compr.replace(field, str(u_select))
try:
curs.execute(query_compr)
except MySQLdb.Error as err:
output = {'aaData':[], 'error':'Error in query compare `{}`! '\
'MySQL Error {}: {}!'.format(query_total, err.args[0], err.args[1])}
return json.dumps(output, indent=2)
headers_tot = [desc[0] for desc in curs.description]
rows_tot = curs.fetchall()
if len(headers) != len(
headers_tot): # Must be the same number of columns
output = {
'aaData': [],
'error':
'The first query has {} columns and the second has {} columns!'
.format(len(headers), len(headers_tot))
}
return json.dumps(output, indent=2)
headers_len = len(headers)
rows_max_size = max(len(rows), len(rows_tot))
calc_rows = []
for i in range(rows_max_size):
row1 = rows[i:i + 1]
row2 = rows_tot[i:i + 1]
if not row1:
row1 = [' ' for i in range(headers_len)]
else:
row1 = row1[0]
if not row2:
row2 = [' ' for i in range(headers_len)]
else:
row2 = row2[0]
calc_rows.append(tuple(row1) + (' <---> ', ) + tuple(row2))
# Update headers: must contain both headers.
headers = headers + ['vs.'] + headers_tot
# ... Normal Query ...
else:
calc_rows = rows
del rows
if (not calc_rows) or (not calc_rows[0:1]):
output = {'aaData': [], 'error': 'The select is empty!'}
return json.dumps(output, indent=2)
if isinstance(calc_rows[0][0], datetime.datetime):
is_date = True
else:
is_date = False
dthandler = lambda obj: obj.strftime(
'%Y-%m-%d %H:%M:%S') if isinstance(obj, datetime.datetime
) else None
return json.dumps(
{
'headers': headers,
'type': report_dict['type'],
'isDate': is_date,
'aaData': calc_rows
},
indent=2,
default=dthandler)
def load_config(self, usr, force=False):
"""
Read DB Config File for 1 user.
"""
if not os.path.isdir(userHome(usr) + '/twister/config'):
logError(
'Report Server: Cannot find Twister for user `{}` !'.format(
usr))
return False
# Get the path to DB.XML
db_file = self.project.get_user_info(usr, 'db_config')
if not db_file:
logError(
'Report Server: Null DB.XML file for user `{}`! Nothing to do!'
.format(usr))
return False
# Current timer
c_time = time.time()
# Create database parser IF necessary, or FORCED, or old connection...
if force or (usr not in self.db_parser) or (
c_time - self.timers.get(usr, 0) > 5.0):
# logDebug('Rebuilding fields, reports and redirects for user `{}`...'.format(usr))
self.timers[usr] = c_time
self.db_parser[usr] = True
self.glob_fields[usr] = OrderedDict()
self.glob_reports[usr] = OrderedDict()
self.glob_redirects[usr] = OrderedDict()
self.glob_links[usr] = [{
'name': 'Home',
'link': 'Home',
'type': 'link'
}]
# DB.xml + Shared DB parser
users_groups = self.project._parse_users_and_groups()
shared_db_path = users_groups['shared_db_cfg']
db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][usr][
'roles']
# Use shared DB or not ?
use_shared_db = self.project.get_user_info(usr, 'use_shared_db')
if use_shared_db and use_shared_db.lower() in ['true', 'yes']:
use_shared_db = True
else:
use_shared_db = False
dbp = DBParser(usr, db_file, shared_db_path, use_shared_db)
self.db_servers = dbp.db_config['servers']
report_queries = dbp.get_reports(db_cfg_role)
del dbp
for host_db in report_queries:
self.glob_fields[usr].update(report_queries[host_db]['fields'])
self.glob_redirects[usr].update(
report_queries[host_db]['redirects'])
for report_name, report_data in report_queries[host_db][
'reports'].iteritems():
srv_name = report_data['srv_name']
# Add the new server name in reports ?
if srv_name not in self.glob_reports[usr]:
self.glob_reports[usr][srv_name] = OrderedDict()
# Add the report inside the server
self.glob_reports[usr][srv_name][report_name] = report_data
# There are more types of reports:
# Normal links, like Home, Help and other normal reports
# Redirect links, that don't contain reports
# Folders, that don't go anywhere, are just labels for reports
for rname, rval in report_queries[host_db][
'reports'].iteritems():
# Shared report ?
srv_name = rval['srv_name']
# Link name used in reports
link = ('S&' if srv_name == 'Shared' else 'U&') + rname
# Server, user, password
db_server, db_name, db_user, _, _ = host_db
srv_db = '{} server: {}/ {}/ {}'.format(
srv_name, db_server, db_name, db_user)
report = {
'name': rname,
'link': link,
'type': 'link',
'folder': rval.get('folder', ''),
'srvr': srv_db
}
self.glob_links[usr].append(report)
for rname, rval in report_queries[host_db][
'redirects'].iteritems():
link = ('S&'
if rval['srv_name'] == 'Shared' else 'U&') + rname
self.glob_links[usr].append({
'name': rname,
'link': link,
'type': 'redir',
'folder': ''
})
# Append the Help link at the end
self.glob_links[usr].append({
'name': 'Help',
'link': 'Help',
'type': 'link'
})
return True
def __init__(self,user,data):
BasePlugin.__init__(self,user,data)
self.cap_dir = helpers.userHome(user) + '/twister/pcap'
print 'CAP_DIR: {}'.format(self.cap_dir)