def setFileOwner(user, path): """ Update file ownership for 1 file or folder.\n `Chown` function works ONLY in Linux. """ logFull('helpers:setFileOwner user `{}`.'.format(user)) try: from pwd import getpwnam uid = getpwnam(user)[2] gid = getpwnam(user)[3] except: return False if os.path.isdir(path): try: proc = subprocess.Popen( ['chown', str(uid) + ':' + str(gid), path, '-R', '--silent'], ) proc.wait() except: logWarning( 'Cannot change owner! Cannot chown folder `{}:{}` on `{} -R`!'. format(uid, gid, path)) return False else: try: os.chown(path, uid, gid) except: logWarning( 'Cannot set owner! Cannot chown file `{}:{}` on `{}`!'.format( uid, gid, path)) return False return True
def list_user_files(self, user, fdir, hidden=True, recursive=True, accept=[], reject=[]): """ List the files in user directory. """ if not fdir: return '*ERROR* Empty `fdir` parameter on list files, user `{}`!'.format( user) srvr = self._usr_service(user) if srvr: try: files = srvr.root.list_files(fdir, hidden, recursive, accept, reject) return copy.copy(files) except Exception as exp_err: err = '*ERROR* Cannot list files `{}`, user `{}`! {}'.format( fdir, user, exp_err) logWarning(err) return err else: return '*ERROR* Cannot access the UserService on list files, user `{}`!'.format( user)
def write_system_file(fpath, fdata, flag='a'): """ Write data in a file. ROOT access. Overwrite or append, ascii or binary. """ if not fpath: return False if flag not in ['w', 'wb', 'a', 'ab']: err = '*ERROR* Invalid flag `{}`! Cannot read!'.format(flag) logWarning(err) return err try: with open(fpath, flag) as file_p: file_p.write(fdata) # if flag == 'w': # logDebug('Written `{}` chars in ascii file `{}`.'.format(len(fdata), fpath)) # elif flag == 'wb': # logDebug('Written `{}` chars in binary file `{}`.'.format(len(fdata), fpath)) # elif flag == 'a': # logDebug('Appended `{}` chars in ascii file `{}`.'.format(len(fdata), fpath)) # else: # logDebug('Appended `{}` chars in binary file `{}`.'.format(len(fdata), fpath)) return True except Exception as exp_err: err = '*ERROR* Cannot write into file `{}`! {}'.\ format(fpath, exp_err) logWarning(err) return err
def get_binding(self, fpath): """ Read a binding between a CFG and a SUT. The result is XML. """ logFull('xmlparser:get_binding') cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user)) if not os.path.isfile(cfg_file): err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user) logError(err) return err bind_xml = parseXML(self.user, cfg_file) if bind_xml is None: err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user) return err # Find the old binding found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath)) if found: xml_string = etree.tostring(found[0]) return xml_string.replace('binding>', 'root>') else: logWarning('*ERROR* Cannot find binding name `{}` for user {}!'.format(fpath, self.user)) return False
def service_stop(self, service): """ stop service """ logFull('CeServices:service_stop') rc = self.service_status(service) if not rc: logDebug('SM: Service name `{}` is not running.'.format(service['name'])) return False tprocess = service.get('pid', 0) if isinstance(tprocess, int): logError('SM: Cannot stop service `{}`!'.format(service['name'])) try: tprocess.terminate() except Exception as exp_err: logError('SM: Cannot stop service: `{}`, exception `{}`!'.format(service['name'], exp_err)) return False try: time.sleep(0.1) os.killpg(tprocess.pid, signal.SIGTERM) time.sleep(0.1) tprocess.kill() except: pass logWarning('SM: Stopped service: `{}`.'.format(service['name'])) return True
def generate_xml(self, user, filename): ''' Receives project file. Creates testsuites.xml file by multiplying tests depending on the suts number and eps. ''' logDebug("CeParser: preparing to convert project file: `{}`,\ user `{}`.".format(filename, user)) data = self.project.read_project_file(user, filename) if data.startswith('*ERROR*'): logWarning(data) return data # try to parse the project file try: xml = etree.fromstring(data) except Exception as e: msg = "The file: '{}' it's not an xml file. Try again!\n{}".\ format(filename, e) logDebug(msg) return '*ERROR* ' + msg self._expand_global_configs(user, xml) self._expand_repeat_tag(xml) self._expand_by_ep(user, xml) repeated_dict = {} self._change_ids(xml, repeated_dict) self._resolve_dependencies(xml, repeated_dict) self._inherit_suite_dependency(xml) self._remove_empty_suites_tc(xml) for suite in xml.findall('.//TestSuite'): prop = suite.find('Property') if prop: suite.remove(prop) # Final XML string xml_string = etree.tostring(xml, pretty_print=True) # write the xml file xml_file = userHome(user) + '/twister/config/testsuites.xml' resp = self.project.localFs.write_user_file(user, xml_file, xml_string, 'w') if resp != True: logError(resp) return '*ERROR* ' + resp logDebug('CeParser: Successfully generated: `{}`, user `{}`.'.\ format(xml_file, user)) return True
def _expand_by_ep(self, user, xml): """ @param: user: the twister authenticated user xml: the project/last_edited xml in etree format. When going into recursion, xml becomes a suite. @summary: The sut tag can contain multiple suts and a sut can contain multiple eps. Main suites must be duplicated to run on each ep from each sut. """ suites = xml.findall('TestSuite') for suite in suites: # get all suts chosen by user all_suts = suite.find('SutName').text suite_name = suite.find('tsName').text if not all_suts: err = 'User `{}`: Invalid SUT for suite `{}`! Cannot generate project!'.format(user, suite_name) logWarning(err) return '*ERROR* ' + err suts_list = [q.replace('(', '.').replace(')', '') for q in all_suts.split(';') if q] # added get_sut to load the suts in resources dict. get_sut_info\ # takes resources from the resources dict index = xml.index(suite) for sut in suts_list: self.project.sut.get_sut(sut, props={'__user': user}) sut = '/' + sut sut_eps = self.project.sut.get_info_sut(sut + ':_epnames_' + user, {'__user': user}) if sut_eps and '*ERROR*' in sut_eps: logError(sut_eps) return sut_eps if sut_eps and (isinstance(sut_eps, str) or isinstance(sut_eps, unicode)): sut_eps_list = [ep for ep in sut_eps.split(';') if ep] for ep_id in sut_eps_list: deep_copy = copy.deepcopy(suite) suite_st = etree.tostring(deep_copy) suite_copy = etree.fromstring(suite_st) if suite_copy.find('TestCase') is not None or \ suite_copy.find('TestSuite') is not None: index += 1 xml.insert(index, suite_copy) self._edit_suite(ep_id, sut, suite_copy) else: # Find Anonimous EP in the active EPs anonim_ep = self.project._find_anonim_ep(user) if isinstance(anonim_ep, bool): return anonim_ep deep_copy = copy.deepcopy(suite) suite_st = etree.tostring(deep_copy) suite_copy = etree.fromstring(suite_st) if suite_copy.find('TestCase') is not None or \ suite_copy.find('TestSuite') is not None: index += 1 xml.insert(index, suite_copy) self._edit_suite(anonim_ep, sut, suite_copy) xml.remove(suite)
def get_query(self, field_id): """ Used by the applet. """ logFull('dbparser:get_query') res = self.user_xml.xpath('insert_section/field[@ID="%s"]' % field_id) if not res: logWarning('User {}: Cannot find field ID `{}`!'.format(self.user, field_id)) return False query = res[0].get('SQLQuery') return query
def sys_file_size(fpath): """ Get file size for 1 file. ROOT access. """ if not fpath: return False try: fsize = os.stat(fpath).st_size # logDebug('File `{}` is size `{}`.'.format(fpath, fsize)) return fsize except Exception as exp_err: err = '*ERROR* Cannot find file `{}`! {}'.format(fpath, exp_err) logWarning(err) return err
def read_system_file(fpath, flag='r', fstart=0): """ Read 1 file. ROOT access. """ if not fpath: return False if flag not in ['r', 'rb']: err = '*ERROR* Invalid flag `{}`! Cannot read!'.format(flag) logWarning(err) return err if not os.path.isfile(fpath): err = '*ERROR* No such file `{}`!'.format(fpath) logWarning(err) return err try: with open(fpath, flag) as file_p: # logDebug('Reading file `{}`, flag `{}`.'.format(fpath, flag)) if fstart: file_p.seek(fstart) fdata = file_p.read() if len(fdata) > 20 * 1000 * 1000: err = '*ERROR* File data too long `{}`: {}!'.format( fpath, len(fdata)) logWarning(err) return err return fdata except Exception as exp_err: err = '*ERROR* Cannot read file `{}`! {}'.format(fpath, exp_err) logWarning(err) return err
def __init__(self, user): self.user = user user_home = userHome(user) if not os.path.isdir('{}/twister'.format(user_home)): raise Exception('PluginParser: Cannot find Twister for user `{}`, '\ 'in path `{}/twister`!'.format(user, user_home)) config_data = '{}/twister/config/plugins.xml'.format(user_home) if not os.path.isfile(config_data): raise Exception('PluginParser: Cannot find Plugins for user `{}`, '\ 'in path `{}/twister/config`!'.format(user, user_home)) # Read directly from CE xml_data = localFs.read_system_file(config_data) self.config = OrderedDict() try: self.xmlTree = etree.fromstring(xml_data) except Exception: raise Exception('PluginParser: Cannot access plugins XML data!') for plugin in self.xmlTree.xpath('Plugin'): if not (plugin.xpath('name/text()') and plugin.xpath('pyfile') and plugin.xpath('jarfile')): logWarning('User {}: PluginParser: Invalid config for plugin: `{}`!'.format(self.user, plugin)) continue prop_keys = plugin.xpath('property/propname') prop_vals = plugin.xpath('property/propvalue') res = dict(zip([k.text for k in prop_keys], [v.text for v in prop_vals])) # Pack Name + Value name = plugin.xpath('name')[0].text self.config[name] = res self.config[name]['jarfile'] = plugin.xpath('jarfile')[0].text.strip() \ if plugin.xpath('jarfile/text()') else '' self.config[name]['pyfile'] = plugin.xpath('pyfile')[0].text.\ strip() if plugin.xpath('pyfile/text()') else '' self.config[name]['status'] = plugin.xpath('status')[0].text.\ strip() if plugin.xpath('status/text()') else ''
def set_binding(self, fpath, content): """ Write a binding between a CFG and a SUT. Return True/ False. """ logFull('xmlparser:set_binding') cfg_file = '{}/twister/config/bindings.xml'.format(userHome(self.user)) if not os.path.isfile(cfg_file): err = '*ERROR* Bindings Config file `{}`, for user `{}` does not exist!'.format(cfg_file, self.user) logError(err) return err bind_xml = parseXML(self.user, cfg_file) if bind_xml is None: err = '*ERROR* Config file `{}`, for user `{}` cannot be parsed!'.format(cfg_file, self.user) return err # Find the old binding found = bind_xml.xpath('/root/binding/name[text()="{}"]/..'.format(fpath)) # If found, use it if found: found = found[0] found.clear() # Or create it else: found = etree.SubElement(bind_xml, 'binding') name = etree.SubElement(found, 'name') name.text = fpath try: replace_xml = etree.fromstring(content, parser) except Exception: err = '*ERROR* Invalid XML content, user {}! Cannot parse!'.format(self.user) logWarning(err) return err for elem in replace_xml: found.append(elem) # Beautify XML ? return etree.tostring(bind_xml, pretty_print=True)
def is_folder(self, user, fpath): """ Returns True of False. Client access via RPyc. """ if not fpath: return '*ERROR* Empty `fpath` parameter on is folder, user `{}`!'.format( user) srvr = self._usr_service(user) if srvr: try: return srvr.root.is_folder(fpath) except Exception as exp_err: err = '*ERROR* Cannot detect file/ folder `{}`, user `{}`! {}'.format( fpath, user, exp_err) logWarning(err) return err else: return '*ERROR* Cannot access the UserService on is folder, user `{}`!'.format( user)
def read_user_file(self, user, fpath, flag='r', fstart=0): """ Read 1 file. Client access via RPyc. """ if not fpath: return '*ERROR* Empty `fpath` parameter on read file, user `{}`!'.format( user) srvr = self._usr_service(user) if srvr: try: return srvr.root.read_file(fpath, flag, fstart) except Exception as exp_err: err = '*ERROR* Cannot read file `{}`, user `{}`! {}'.format( fpath, user, exp_err) logWarning(err) return err else: return '*ERROR* Cannot access the UserService on read file, user `{}`!'.format( user)
def set_tb(self, name, parent=None, props={}): """ Higher level wrapper over functions Create new TB, create component and update meta. """ pdata = self.get_resource(parent) user_info = self.user_info(props) if not isinstance(pdata, dict): logWarning('User `{}`: No such parent `{}`!'.format(user_info[0], parent)) return False if (parent == '/' or parent == '1') and name not in pdata['children']: return self.create_new_tb(name, parent, props) # If exists, update meta if name in pdata['children']: return self.update_meta_tb(name, parent, props) # This is a new component else: return self.create_component_tb(name, parent, props)
def execScript(script_path): """ Execute a user script and return the text printed on the screen. """ logFull('helpers:execScript') if not os.path.exists(script_path): logWarning( 'Exec script: The path `{}` does not exist!'.format(script_path)) return False try: os.system('chmod +x {}'.format(script_path)) except: pass logDebug('Executing script `{}`...'.format(script_path)) try: txt = subprocess.check_output(script_path, shell=True) return txt.strip() except Exception as exp_err: logWarning('Exec script `{}`: Exception - `{}`.'.\ format(script_path, exp_err)) return False
def write_user_file(self, user, fpath, fdata, flag='w'): """ Write 1 file. Client access via RPyc. """ if not fpath: return '*ERROR* Empty `fpath` parameter on write file, user `{}`!'.format( user) srvr = self._usr_service(user, 'write') if len(fdata) > 20 * 1000 * 1000: err = '*ERROR* File data too long `{}`: {}; User {}.'.format( fpath, len(fdata), user) logWarning(err) return err if srvr: try: return srvr.root.write_file(fpath, fdata, flag) except Exception as exp_err: err = '*ERROR* Cannot write into file `{}`, user `{}`! {}'.format( fpath, user, exp_err) logWarning(err) return err else: return '*ERROR* Cannot access the UserService on write file, user `{}`!'.format( user)
def getPlugins(self): """ Return all plugins info """ logFull('xmlparser:getPlugins') Base = BasePlugin.BasePlugin py_modules = [k +'::'+ os.path.splitext(self.config[k]['pyfile'])[0] for k in self.config if self.config[k]['status'] == 'enabled'] plugins = {} for module in py_modules: name = module.split('::')[0] if not name: continue mod = module.split('::')[1] if not mod: continue if not os.path.isfile('{}/plugins/{}.py'.format(TWISTER_PATH, mod)): continue plug = None try: # Import the plugin module mm = __import__('plugins.' + mod, fromlist=['Plugin']) # Reload all data, just in case mm = reload(mm) plug = mm.Plugin except Exception, e: logWarning('User {}: PluginParser ERROR: Unhandled exception' \ ' in plugin file `{}`! Exception: {}!'.\ format(self.user, mod, e)) continue if not plug: logWarning('User {}: PluginParser ERROR: Plugin `{}` cannot be Null!'.format(self.user, plug)) continue # Check plugin parent. Must be Base Plugin. if not issubclass(plug, Base): logWarning('User {}: PluginParser ERROR: Plugin `{}` must be' \ ' inherited from Base Plugin!'.format(self.user, plug)) continue # Append plugin classes to plugins list d = self.config[name] d['plugin'] = plug plugins[name] = d
def dirList(path): """ Create recursive list of folders and files from base path. The format of a node is: {"path": "/..." "data": "name", "folder":true|false, "children": []} """ # The node is valid ? if not path: return False # Cleanup '/' if path != '/': path = path.rstrip('/') # This is folder ? if os.path.isfile(path): return False len_path = len(base_path) + 1 dlist = [] # Folders list flist = [] # Files list try: names = sorted(os.listdir(path), key=str.lower) except Exception as exp_err: logWarning('*WARN* Cannot list folder `{}`: `{}`!'.\ format(path, exp_err)) return [] # Cycle a folder for fname in names: long_path = path + '/' + fname # If Accept is active and file doesn't match, ignore file if accept and os.path.isfile(long_path): valid = True if isinstance(accept, list): # If nothing from the Accept matches the file if True not in [(long_path.startswith(f) or long_path.endswith(f)) for f in accept]: valid = False elif isinstance(accept, str): if not (long_path.startswith(accept) or long_path.endswith(accept)): valid = False if not valid: continue # If Reject is active and file matches, ignore the file if reject and os.path.isfile(long_path): valid = True if isinstance(reject, list): # If nothing from the Reject matches the file if True in [(long_path.startswith(f) or long_path.endswith(f)) for f in reject]: valid = False elif isinstance(reject, str): if long_path.startswith(reject) or long_path.endswith( reject): valid = False if not valid: continue # Ignore hidden files if hidden and fname[0] == '.': continue # Meta info try: fstat = os.stat(long_path) try: uname = pwd.getpwuid(fstat.st_uid).pw_name except Exception: uname = fstat.st_uid try: gname = grp.getgrgid(fstat.st_gid).gr_name except Exception: gname = fstat.st_gid meta_info = '{}|{}|{}|{}'.\ format(uname, gname, fstat.st_size,\ time.strftime('%Y-%m-%d %H:%M:%S',\ time.localtime(fstat.st_mtime))) except Exception: meta_info = '' # Semi long path short_path = long_path[len_path:] # Data to append data = {'path': short_path, 'data': fname, 'meta': meta_info} if os.path.isdir(long_path): data['folder'] = True # Recursive ! if recursive: children = dirList(long_path) else: children = [] if children in [False, None]: continue data['children'] = children dlist.append(data) else: flist.append(data) # Folders first, files second return dlist + flist
def _usr_service(self, user_view_actv, op='read'): """ Launch a user service. Open a ClearCase view first. """ if user_view_actv.count(':') == 1 and user_view_actv[-1] != ':': user_view_actv += ':' try: user, view, actv = user_view_actv.split(':') except Exception: # We don't really know the user in here ! msg = 'Invalid ClearCase user-view-activity parameter: `{}`!'.format( user_view_actv) logWarning(msg) return '*ERROR* ' + msg if op not in ['read', 'write']: logWarning( 'Invalid CC operation `{}`, for user `{}`! Will reset to "read".' .format(op, user)) op = 'read' view = view.strip() actv = actv.strip() user_view = user + ':' + view if not view: # We don't know the view in here ! msg = 'Empty view in `{}`!'.format(user_view_actv) logWarning(msg) return '*ERROR* ' + msg def pread(): """ Read proc stdout """ while 1: try: line = proc.readline().strip() if not line: continue plog.append(line) except: break # Must block here, so more users cannot launch Logs at the same time and lose the PID with self._srv_lock: # Try to re-use the FS, if available conn = self._services.get(user_view, {}).get('conn_' + op, None) if conn: try: conn.ping(data='Hello', timeout=30.0) # logDebug('Reuse old {} ClearCase Service connection for `{}` OK.'.format(op, user_view)) proc = self._services.get(user_view, {}).get('proc', None) old_actv = self._services.get(user_view, {}).get('actv', None) if actv != old_actv: logInfo('Changing activity to `{}`, for `{}`.'.format( actv, user_view)) # Set cc activity again ! proc.sendline('cleartool setactivity {}'.format(actv)) time.sleep(1.0) pread() self._services.get(user_view, {})['actv'] = actv return conn except Exception as exp_err: logWarning('Cannot connect to `{}` ClearCase Service \ for `{}`: `{}`.'.format(op, user_view, exp_err)) self._kill(user) proc = self._services.get(user_view, {}).get('proc', None) PID = proc.pid proc.terminate() logInfo('Terminated CC User Service `{}` for user `{}`.'.\ format(PID, user)) else: logInfo('Launching a ClearCase Service for `{}`, the first \ time...'.format(user_view)) proc = pexpect.spawn(['bash'], timeout=2.5, maxread=2048) time.sleep(1.0) plog = [] proc.sendline('su {}'.format(user)) time.sleep(1.0) pread() # User's home folder proc.sendline('cd ~/twister') pread() # Set cc view only the first time ! proc.sendline('cleartool setview {}'.format(view)) time.sleep(2.0) pread() # Empty line after set view proc.sendline('') pread() if actv: # Set cc activity for the first time ! proc.sendline('cleartool setactivity {}'.format(actv)) time.sleep(1.0) pread() port = None # If the server is not available, search for a free port in the safe range... while 1: port = random.randrange(63000, 65000) try: socket.create_connection((None, port), 1) except Exception: break # Launching 1 UserService inside the SSH terminal, with ClearCase View open p_cmd = '{} -u {}/server/UserService.py {} {} & '.format( sys.executable, TWISTER_PATH, port, self.name) proc.sendline(p_cmd) time.sleep(2.0) pread() # Empty line after proc start proc.sendline('') pread() logDebug( 'ClearCase startup log \n:: -------\n{}\n:: -------'.format( '\n'.join(plog))) config = { 'allow_pickle': True, 'allow_getattr': True, 'allow_setattr': True, 'allow_delattr': True } retry = 10 delay = 0.5 success = False while retry > 0: if success: break try: stream_r = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0) conn_read = rpyc.connect_stream(stream_r, config=config) conn_read.root.hello() logDebug( 'Connected to ClearCase Service for `{}`, operation `read`.' .format(user_view)) success = True except Exception as exp_err: logWarning('Cannot connect to ClearCase Service for `{}`!'\ 'Exception: `{}`! Retry...'.format(user_view, exp_err)) if success: try: stream_w = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0) conn_write = rpyc.connect_stream(stream_w, config=config) conn_write.root.hello() logDebug( 'Connected to ClearCase Service for `{}`, operation `write`.' .format(user_view)) break except Exception as exp_err: logWarning('Cannot connect to ClearCase Service \ for `{}`! Exception: `{}`! Retry...' .\ format(user_view, exp_err)) success = False time.sleep(delay) retry -= 1 delay += 0.75 if not success: logError( 'Error on starting ClearCase Service for `{}`!'.format( user_view)) return None # Save the process inside the block. self._services[user_view] = {'proc': proc, 'port': port,\ 'conn_read': conn_read, 'conn_write': conn_write, 'actv': actv} logDebug( 'ClearCase Service for `{}` launched on `127.0.0.1:{}`.'.format( user_view, port)) return self._services[user_view].get('conn_' + op, None)
def _get_iterator_values(self, config_file, iterator_default,\ interval_values, user, part_interval_values=[], default_values_list=[]): """ Calculates all the possible values for iterators for a test """ data = self.project.configs.read_config_file(user, config_file) if data.startswith('*ERROR*'): logWarning(data) return # Try to parse the project file try: xml_config = etree.fromstring(data) except: msg = "Config file `{}` is invalid!".format(config_file) logWarning(msg) return config_file_st = etree.tostring(xml_config) config_file_fst = etree.fromstring(config_file_st) # find all entries having tag = iterator config_types = config_file_fst.xpath('//type') config_types = [x for x in config_types if x.text == 'iterator'] # Iterators from config file for item in config_types: prop_iterator = item.getparent() # set the parent component(s) of the iterator comp_parent = prop_iterator.getparent() comp_parent_name = '' if comp_parent.find('fname') is not None: comp_parent_name = comp_parent.find('fname').text # if this is a sub-component, we need to go up to get all parents while True: comp_parent = comp_parent.getparent() if comp_parent is None: # we reached the root; need to break out from this loop break comp_fname = comp_parent.find('fname') if comp_fname is not None: comp_parent_name = comp_parent.find('fname').text + '/' +\ comp_parent_name config_name = config_file + '#' + comp_parent_name +\ '#' + prop_iterator.find('name').text values = prop_iterator.find('value').text if not values: continue values_list = values.replace(' ', '').split(',') # get the default value index_dot = values_list[0].find('..') if index_dot > -1: try: default_value = ast.literal_eval( values_list[0][:index_dot]) default_value = int(default_value) except: default_value = values_list[0][:index_dot] else: try: default_value = ast.literal_eval(values_list[0]) default_value = int(default_value) except: default_value = values_list[0] if iterator_default == 'true': part_interval_values.append(['{}={}'.\ format(config_name, default_value)]) default_values_list.append('{}={}'.\ format(config_name, default_value)) else: iter_interval_values = list() key_default_value = '{}={}'.format(config_name, default_value) for interv in values_list: re_intervals = re.search('(\w*\d*\.?\d+)\.+(\w*\d*\.?\d+)',\ interv) try: x_val = ast.literal_eval(re_intervals.group(1)) y_val = ast.literal_eval(re_intervals.group(2)) range_res = range(int(x_val), int(y_val) + 1) # avoid adding default value again ex: 2, 1...4 if default_value in range_res and \ key_default_value in iter_interval_values: del range_res[range_res.index(default_value)] for i in range_res: iter_interval_values.append('{}={}'.\ format(config_name, i)) except: try: x_val = re_intervals.group(1) y_val = re_intervals.group(2) # try to convert to int if possible try: value = int(ast.literal_eval(x_val)) iter_interval_values.append('{}={}'.\ format(config_name, value)) except: iter_interval_values.append('{}={}'.\ format(config_name, x_val)) try: value = int(ast.literal_eval(y_val)) iter_interval_values.append('{}={}'.\ format(config_name, value)) except: iter_interval_values.append('{}={}'.\ format(config_name, y_val)) except: try: interv = ast.literal_eval(interv) except: pass # avoid adding default value again ex: 2, 1, 2, 3 if default_value != interv or\ key_default_value not in iter_interval_values: iter_interval_values.append('{}={}'.\ format(config_name, interv)) part_interval_values.append(iter_interval_values) if part_interval_values: if config_name in interval_values.keys(): interval_values[config_name].extend(part_interval_values) else: interval_values[config_name] = part_interval_values
def get_reports(self, db_cfg_role=True): """ Used by Reporting Server. Returns a list with all report fields and queries. """ logFull('dbparser:get_reports') report_queries = OrderedDict() def get_fields(server_data, srv_name): """ All report fields. """ fields = OrderedDict() for field in server_data.xpath('reports_section/field'): data = {} data['id'] = field.get('ID', '') data['type'] = field.get('Type', '') data['label'] = field.get('Label', data['id']) data['sqlquery'] = field.get('SQLQuery', '') data['srv_name'] = srv_name fields[data['id']] = data return fields def get_reps(server_data, srv_name): """ All reports. """ reports = OrderedDict() for report in server_data.xpath('reports_section/report'): data = {} data['id'] = report.get('ID', '') data['type'] = report.get('Type', '') data['path'] = report.get('Path', '') data['folder'] = report.get('Folder', '') data['sqlquery'] = report.get('SQLQuery', '') data['sqltotal'] = report.get('SQLTotal', '') # SQL Total Query data['sqlcompr'] = report.get('SQLCompare', '') # SQL Query Compare side by side data['srv_name'] = srv_name # Save server name here reports[data['id']] = data return reports def get_redirects(server_data, srv_name): """ All redirects. """ redirects = OrderedDict() for redirect in server_data.xpath('reports_section/redirect'): data = {} data['id'] = redirect.get('ID', '') data['path'] = redirect.get('Path', '') data['srv_name'] = srv_name redirects[data['id']] = data return redirects # If the user has the roles AND Use Shared DB is disabled (user DB enabled) if db_cfg_role and not self.use_shared_db: # Insert user DB first and shared DB second db_pair = self.db_config['default_server'] # Reports and Redirects from private db.xml report_queries[db_pair] = { 'fields': get_fields(self.user_xml, 'User'), 'reports': get_reps(self.user_xml, 'User'), 'redirects': get_redirects(self.user_xml, 'User') } if not self.use_shared_db and not db_cfg_role: logInfo('Insufficient privileges to get user reports, for user `{}`!'.format(self.user)) # Valid shared db.xml if self.shared_xml is None: logWarning('Invalid shared DB XML on get reports, for user `{}`!'.format(self.user)) return report_queries # Invalid entry ? if not self.shared_xml.xpath('db_config/server/text()') or \ not self.shared_xml.xpath('db_config/database/text()'): logWarning('Invalid shared DB XML on get reports, for user `{}`!'.format(self.user)) return report_queries # Important MySQL server info db_server = self.shared_xml.xpath('db_config/server')[0].text db_name = self.shared_xml.xpath('db_config/database')[0].text db_user = self.shared_xml.xpath('db_config/user')[0].text db_passwd = self.shared_xml.xpath('db_config/password')[0].text db_pair = (db_server, db_name, db_user, db_passwd, 'S') # Overwrite all private fields, reports or redirects if db_pair in report_queries: report_queries[db_pair]['fields'].update(get_fields(self.shared_xml, 'Shared')) report_queries[db_pair]['reports'].update(get_reps(self.shared_xml, 'Shared')) report_queries[db_pair]['redirects'].update(get_redirects(self.shared_xml, 'Shared')) # Save this info else: report_queries[db_pair] = { 'fields': get_fields(self.shared_xml, 'Shared'), 'reports': get_reps(self.shared_xml, 'Shared'), 'redirects': get_redirects(self.shared_xml, 'Shared') } # Return after shared db inserts ! # import pprint ; pprint.pprint(dict(report_queries), width=40) return report_queries
def rename_tb(self, query, new_name, props={}): """ Rename a resource. """ logDebug('Rename TB `{}`, new name `{}`, props {}.'.format(query, new_name, props)) user_info = self.user_info(props) if ':' in query: meta = query.split(':')[1] query = query.split(':')[0] else: meta = '' _is_res_reserved = self.is_resource_reserved(query, props) if _is_res_reserved and _is_res_reserved != user_info[0]: msg = 'User {}: The resource is reserved for {} !'.format(user_info[0], _is_res_reserved) logWarning(msg) return '*ERROR* ' + msg _is_res_locked = self.is_resource_locked(query) if _is_res_locked and _is_res_locked != user_info[0]: msg = 'User {}: The resource is locked for {} !'.format(user_info[0], _is_res_locked) logWarning(msg) return '*ERROR* ' + msg if '/' in new_name or ':' in new_name: logWarning('New resource name cannot contain `/` or `:`!') return False # If no resources... if not self.resources.get('children'): msg = 'There are no resources defined !' logError(msg) return '*ERROR* ' + msg # Correct node path result = self.get_reserved_resource(query, props) if not result: logWarning('Cannot access reserved TB `{}` !') return False if result['path'][-1] == new_name: logWarning('Nothing to rename for TB `{}` !'.format(new_name)) return True with self.ren_lock: # If must rename a Meta info if meta: try: # Modify meta to the parent if len(result['path']) == 1: child = result # Modify to a component else: base_path = '/'.join(result['path'][1:]) child = self.get_path(base_path, result) child['meta'][new_name] = child['meta'].pop(meta) except: msg = 'Rename meta `{}` error, for TB `{}`!'.format(meta, result['path'][-1]) logWarning(msg) return 'false' return self.save_reserved_tb(query, props) # If must rename a normal node else: # The parent is directly from the root and we want to rename its immediate children if len(result['path']) == 2: result['children'][new_name] = result['children'].pop(result['path'][-1]) # The component we want to rename is deep in the tree elif len(result['path']) > 2: base_path = '/'.join(result['path'][1:-1]) parent = self.get_path(base_path, result) if not isinstance(parent, dict): msg = msg = 'Rename error for TB `{}`, invalid parent \ on {}!'.format(result['path'][-1], result['id']) logWarning(msg) return 'false' parent['children'][new_name] = parent['children'].pop(result['path'][-1]) else: result['path'] = [new_name] # Only have to change the current path and the path of the children result['path'] = [result['path'][0]] # Recursive update paths self.change_path(result, result['path']) return True
def project_data(self, user, save_to_db=False): """ Collect all data from a user, using the DB.XML for the current project. If save to DB is active, the function will also save. """ # Get the path to DB.XML db_file = self.project.get_user_info(user, 'db_config') if not db_file: logError( 'Database: Null DB.XML file for user `{}`! Nothing to do!'. format(user)) return False # DB.xml + Shared DB parser users_groups = self.project._parse_users_and_groups() shared_db_path = users_groups['shared_db_cfg'] db_cfg_role = 'CHANGE_DB_CFG' in users_groups['users'][user]['roles'] # Use shared DB or not ? use_shared_db = self.project.get_user_info(user, 'use_shared_db') if use_shared_db and use_shared_db.lower() in ['true', 'yes']: use_shared_db = True else: use_shared_db = False dbp = DBParser(user, db_file, shared_db_path, use_shared_db) all_inserts = dbp.get_inserts(db_cfg_role) del dbp if not all_inserts: logWarning('Database: Cannot use inserts defined for user `{}`!'.\ format(user)) return False # UserScript cache usr_script_cache_s = {} # Suite usr_script_cache_p = {} # Project # DbSelect cache db_select_cache_s = {} # Suite db_select_cache_p = {} # Project conn, curs = None, None # Pre-calculated data all_data = [] for subst_data in self.static_project_data(user): # Setup and Teardown files will not be saved to database! if subst_data.get('setup_file') == 'true' or \ subst_data.get('teardown_file') == 'true': logDebug("Ignoring `{}` file, because it's setup or teardown.".\ format(subst_data['twister_tc_name'])) continue # Pre-Suite or Post-Suite files will not be saved to database if subst_data.get('Pre-Suite') or subst_data.get('Post-Suite'): continue # If file has iterators, the iteration save==Failed and the status was Failed if subst_data.get('_cfg_files') and subst_data.get('iterationNr') \ and subst_data.get('iterationSave') == 'failed' and \ subst_data['twister_tc_status'] == 'PASS': continue # For every host, build correct data... for host_db in all_inserts: c_inserts = all_inserts[host_db]['inserts'] c_fields = all_inserts[host_db]['fields'] shared_db = all_inserts[host_db]['shared_db'] db_server, db_name, db_user, db_passwd, _ = host_db conn = self.connect_db(user, db_server, db_name, db_user,\ db_passwd, shared_db=shared_db) if not conn: continue curs = conn.cursor() # Escape all unicodes variables before SQL Statements! subst_data = {k: MySQLdb.escape_string(v) if isinstance(v, unicode) else v for \ k, v in subst_data.iteritems()} # For every query of the current host for query in c_inserts: # All variables of type `UserScript` must be replaced # with the script result try: user_script_fields = re.findall( '(\$.+?)[,\.\'"\s]', query) except Exception: user_script_fields = [] for field in user_script_fields: field = field[1:] # Invalid field ? if field not in c_fields: continue # If the field is not `UserScript`, ignore it if c_fields.get(field, {}).get('type') != 'UserScript': continue # Field level: Suite or Project lvl = c_fields.get(field)['level'] # Get Script Path, or null string u_script = subst_data.get(field, '') if not u_script: query = query.replace('$' + field, '') continue # Execute this script based on level if lvl == 'Project': if u_script not in usr_script_cache_p: # Execute script and use result res = execScript(u_script) # logDebug('Database: UserScript for `{}` was executed at '\ # 'LVL `{}`.'.format(user, lvl)) # Save result in cache usr_script_cache_p[u_script] = res else: # Get script result from cache res = usr_script_cache_p[u_script] # Execute for every suite else: suite_id = subst_data['twister_suite_id'] if suite_id not in usr_script_cache_s: usr_script_cache_s[suite_id] = {} if u_script not in usr_script_cache_s[suite_id]: # Execute script and use result res = execScript(u_script) # logDebug('Database: UserScript for `{}` was executed at '\ # 'LVL `{}`.'.format(user, lvl)) # Save result in cache usr_script_cache_s[suite_id][u_script] = res else: # Get script result from cache res = usr_script_cache_s[suite_id][u_script] # Replace UserScript with with real Script results if not res: res = '' query = query.replace('$' + field, res) # Adding user script fields subst_data[field] = res # All variables of type `DbSelect` must be replaced with the SQL result try: auto_insert_fields = re.findall('(@.+?@)', query) except Exception: auto_insert_fields = [] for field in auto_insert_fields: # Delete the @ character field = field[1:-1] # Invalid field ? if field not in c_fields: continue # Get Auto Query, or null string u_query = c_fields.get(field, {}).get('query', '') # Field level: Suite, Project, or Testcase lvl = c_fields.get(field)['level'] if not u_query: logError('User `{}`, file `{}`: Cannot build query! Field `{}` '\ 'is not defined in the fields section!'.format(user, subst_data['file'], field)) return False # Execute User Query based on level if lvl == 'Project': if u_query not in db_select_cache_p: # Execute User Query curs.execute(u_query) q_value = curs.fetchone()[0] # logDebug('Database: DbSelect for `{}` was executed at '\ # 'LVL `{}`.'.format(user, lvl)) # Save result in cache db_select_cache_p[u_query] = q_value else: # Get script result from cache q_value = db_select_cache_p[u_query] # Execute User Query for every suite elif lvl == 'Suite': suite_id = subst_data['twister_suite_id'] if suite_id not in db_select_cache_s: db_select_cache_s[suite_id] = {} if u_query not in db_select_cache_s[suite_id]: # Execute User Query curs.execute(u_query) q_value = curs.fetchone()[0] # logDebug('Database: DbSelect for `{}` was executed at '\ # 'LVL `{}`.'.format(user, lvl)) # Save result in cache db_select_cache_s[suite_id][u_query] = q_value else: # Get script result from cache q_value = db_select_cache_s[suite_id][u_query] else: # Execute User Query curs.execute(u_query) q_value = curs.fetchone()[0] # logDebug('Database: DbSelect for `{}` was executed at '\ # 'LVL `TestCase`.'.format(user)) # Replace @variables@ with real Database values query = query.replace('@' + field + '@', str(q_value)) # Adding auto insert fields subst_data[field] = str(q_value) # String Template tmpl = Template(query) # Fix None values to NULL subst_data = { k: 'NULL' if v is None else v for k, v in subst_data.iteritems() } # Build complete query try: query = tmpl.substitute(subst_data) except Exception as exp_err: logError('User `{}`, file `{}`: Cannot build query! \ Error on `{}`!'.format(user, subst_data['file'], exp_err)) return False # Save query in database ? if save_to_db: # Execute MySQL Query! try: curs.execute(query) logDebug('Executed query\n\t``{}``\n\t on {} OK.'. format(query.strip(), host_db)) conn.commit() except MySQLdb.Error as exp_err: logError('Error in query ``{}`` , for user \ `{}`!\n\t MySQL Error {}: {}!' .\ format(query, user, exp_err.args[0],\ exp_err.args[1])) conn.rollback() return False # :: Debug :: # import pprint ; pprint.pprint(subst_data) # Append all data for current file all_data.append(subst_data) return all_data
def _usr_service(self, user, oper='read'): """ Launch a user service. """ if oper not in ['read', 'write']: logWarning( 'Invalid FS operation `{}`, for user `{}`! Will reset to "read".' .format(oper, user)) oper = 'read' # Must block here, so more users cannot launch Logs at the same time and lose the PID with self._srv_lock: # Try to re-use the logger server, if available conn = self._services.get(user, {}).get('conn_' + oper, None) if conn: try: conn.ping(data='Hello', timeout=30.0) # logDebug('Reuse old {} User Service connection for `{}` OK.'.format(op, user)) return conn except Exception as exp_err: logWarning( 'Cannot reuse {} User Service for `{}`: `{}`.'.format( oper, user, exp_err)) self._kill(user) else: logInfo('Launching a User Service for `{}`, the first time...'. format(user)) port = None # If the server is not available, search for a free port in the safe range... while 1: port = random.randrange(63000, 65000) try: socket.create_connection((None, port), 1) except Exception: break p_cmd = 'su {} -c "{} -u {}/server/UserService.py {} {}"'.\ format(user, sys.executable, TWISTER_PATH, port, self.name) proc = subprocess.Popen(p_cmd, cwd='{}/twister'.\ format(userHome(user)), shell=True, close_fds=True,\ stdout=subprocess.PIPE, stderr=subprocess.PIPE) proc.poll() time.sleep(2.0) config = { 'allow_pickle': True, 'allow_getattr': True, 'allow_setattr': True, 'allow_delattr': True } retry = 10 delay = 0.5 success = False while retry > 0: if success: break try: stream_r = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0) conn_read = rpyc.connect_stream(stream_r, config=config) conn_read.root.hello() logDebug( 'Connected to User Service for `{}`, operation `read`.' .format(user)) success = True except Exception as exp_err: logWarning('Cannot connect to User Service for `{}` - \ Exception: `{}`! Wait {}s...'.format(user, exp_err, delay)) if success: try: stream_w = rpyc.SocketStream.connect('127.0.0.1', port, timeout=5.0) conn_write = rpyc.connect_stream(stream_w, config=config) conn_write.root.hello() logDebug( 'Connected to User Service for `{}`, operation `write`.' .format(user)) break except Exception as exp_err: logWarning('Cannot connect to User Service for `{}` \ - Exception: `{}`! Wait {}s...' .\ format(user, exp_err, delay)) success = False time.sleep(delay) retry -= 1 delay += 0.75 if not success: logError( 'Error on starting User Service for `{}`!'.format(user)) return None # Save the process inside the block. 99% of the time, this block is executed instantly! self._services[user] = { 'proc': proc, 'conn_read': conn_read, 'conn_write': conn_write, 'port': port } logDebug( 'User Service for `{}` launched on `127.0.0.1:{}` - PID `{}`.'. format(user, port, proc.pid)) return self._services[user].get('conn_' + oper, None)
if TWISTER_PATH not in sys.path: sys.path.append(TWISTER_PATH) from common.tsclogging import logDebug, logInfo, logWarning, logError, logCritical from common.tsclogging import setLogLevel from server.CeProject import Project from server.CeXmlRpc import CeXmlRpc from server.CeRpyc import CeRpycService from common import iniparser # if __name__ == "__main__": if os.getuid() != 0: logWarning('Twister Server should run as ROOT! If it doesn\'t, ' 'it won\'t be able to read config files and write logs for all users!') SERVER_PORT = sys.argv[1:2] if not SERVER_PORT: logCritical('Twister Server: Must start with parameter PORT number!') exit(1) else: try: SERVER_PORT = int(SERVER_PORT[0]) except Exception: logCritical('Twister Server: Must start with parameter PORT number!') exit(1) # Read verbosity from configuration CFG_PATH = '{}/config/server_init.ini'.format(TWISTER_PATH)
def __init__(self, user, config_data, shared_data=None, use_shared_db=True): self.user = user self.db_config = {} self.config_data = config_data self.user_xml = None self.shared_xml = None self.use_shared_db = use_shared_db if os.path.isfile(config_data): data = localFs.read_user_file(self.user, config_data) try: self.user_xml = etree.fromstring(data) # logDebug('User `{}` loaded priv DB config from file `{}`.'.format(user, config_data)) except Exception: raise Exception('Invalid DB config file `{}`, '\ 'for user `{}`!'.format(config_data, self.user)) elif config_data and isinstance(config_data, str) or isinstance(config_data, unicode): try: self.user_xml = etree.fromstring(config_data) # logDebug('User `{}` loaded priv DB config from a string.'.format(user)) except Exception: raise Exception('Cannot parse DB config data, for user `{}`!'.format(self.user)) else: raise Exception('Invalid config data type: `{}`, '\ 'for user `{}`!'.format(type(config_data), self.user)) if shared_data: if os.path.isfile(shared_data): data = localFs.read_user_file(self.user, shared_data) try: self.shared_xml = etree.fromstring(data) # logDebug('User `{}` loaded shared DB config from file `{}`.'.format(user, shared_data)) except Exception: raise Exception('Invalid shared DB config file `{}`, '\ 'for user `{}`!'.format(shared_data, self.user)) elif shared_data and isinstance(shared_data, str) or isinstance(shared_data, unicode): try: self.shared_xml = etree.fromstring(shared_data) # logDebug('User `{}` loaded shared DB config from a string.'.format(user)) except Exception: logWarning('Cannot parse shared DB config data, for user `{}`!'.format(self.user)) else: raise Exception('Invalid shared config data type: `{}`, '\ 'for user `{}`!'.format(type(shared_data), self.user)) # The servers list is used to know how to connect to a specific server name self.db_config['servers'] = {} if self.user_xml.xpath('db_config/server/text()') and self.user_xml.xpath('db_config/database/text()'): # User's server and database db_server = self.user_xml.xpath('db_config/server')[0].text db_name = self.user_xml.xpath('db_config/database')[0].text db_user = self.user_xml.xpath('db_config/user')[0].text db_passwd = self.user_xml.xpath('db_config/password')[0].text self.db_config['default_server'] = (db_server, db_name, db_user, db_passwd, 'U') self.db_config['servers']['User'] = self.db_config['default_server'] else: raise Exception('Invalid DB config, no server and DB, for user `{}`!'.format(self.user)) if shared_data and self.shared_xml is not None: # Servers list try: db_server = self.shared_xml.xpath('db_config/server')[0].text db_name = self.shared_xml.xpath('db_config/database')[0].text db_user = self.shared_xml.xpath('db_config/user')[0].text db_passwd = self.shared_xml.xpath('db_config/password')[0].text self.db_config['servers']['Shared'] = (db_server, db_name, db_user, db_passwd, 'S') except Exception as err: logWarning('Invalid shared DB XML, for user `{}`: {}!'.format(self.user, err)) self.shared_xml = None
from mako.template import Template TWISTER_PATH = os.getenv('TWISTER_PATH') if not TWISTER_PATH: print('\n$TWISTER_PATH environment variable is not set! Exiting!\n') exit(1) if TWISTER_PATH not in sys.path: sys.path.append(TWISTER_PATH) from common.helpers import userHome from common.tsclogging import logDebug, logInfo, logWarning, logError from common.xmlparser import DBParser if mako.__version__ < '0.7': logWarning( 'Warning! Mako-template version `{}` is old! Some pages might crash!\n' .format(mako.__version__)) class ReportingServer(object): """ Reporting server class. """ db_parser = {} db_servers = {} glob_fields = {} glob_reports = {} glob_redirects = {} glob_links = {} timers = {}
def get_inserts(self, db_cfg_role=True): """ Used by Database Manager. Returns a list with all insert fields and queries. """ logFull('dbparser:get_inserts') insert_queries = OrderedDict() # If user has the roles and Use Shared DB is disabled (user DB enabled) if db_cfg_role and not self.use_shared_db: # Fields and Inserts from private db.xml private_db = {} private_db['inserts'] = [q.text for q in self.user_xml.xpath('insert_section/sql_statement')] fields = OrderedDict() for field in self.user_xml.xpath('insert_section/field'): data = {} data['id'] = field.get('ID', '') data['type'] = field.get('Type', '') data['query'] = field.get('SQLQuery', '') data['level'] = field.get('Level', '') # Project / Suite / Testcase fields[data['id']] = data private_db['fields'] = fields private_db['shared_db'] = False # Add private db to inserts db_pair = self.db_config['default_server'] insert_queries[db_pair] = private_db # Return after user db inserts ! return insert_queries if self.shared_xml is None: logWarning('Invalid shared DB XML on get inserts, for user `{}`!'.format(self.user)) return insert_queries # Invalid entry ? if not self.shared_xml.xpath('db_config/server/text()') or \ not self.shared_xml.xpath('db_config/database/text()'): logWarning('Invalid shared DB XML on get inserts, for user `{}`!'.format(self.user)) return insert_queries # Important MySQL server info db_server = self.shared_xml.xpath('db_config/server')[0].text db_name = self.shared_xml.xpath('db_config/database')[0].text db_user = self.shared_xml.xpath('db_config/user')[0].text db_passwd = self.shared_xml.xpath('db_config/password')[0].text db_pair = (db_server, db_name, db_user, db_passwd, 'S') # Insert fields fields = OrderedDict() for field in self.shared_xml.xpath('insert_section/field'): data = {} data['id'] = field.get('ID', '') data['type'] = field.get('Type', '') data['query'] = field.get('SQLQuery', '') data['level'] = field.get('Level', '') # Project / Suite / Testcase fields[data['id']] = data # Insert queries inserts = [] for elem in self.shared_xml.xpath('insert_section/sql_statement'): inserts.append(elem.text.strip()) # Save this info insert_queries[db_pair] = { 'inserts': inserts, 'fields': fields, 'shared_db': True } # Return after shared db inserts ! return insert_queries