def send_update(self, datos=None): """ Sends an email update with status of run. :return: """ note_queue = common.global_vars['test_notes'] notes = '' while not note_queue.empty(): notes += note_queue.get() checkpoint_results = '' for checkpoint, result in common.global_vars['checkpoints']: checkpoint_results += '\n+ %s: %s' % (result, checkpoint) # Format completed message. msg = 'Test Update.\n\n' msg += ' Name: %s\n' % self.test_name msg += ' Config: %s\n' % self.config_name msg += ' Log: %s\n' % self.log_dir msg += ' Time: %s\n' % seconds_to_days_hours_min_sec_string( time.time() - self.start_time) if checkpoint_results: msg += '\nCheckpoints:\n%s\n' % checkpoint_results if notes: msg += '\nMessages:\n%s' % notes # Playing with the formatting a bit here to create better looking reports. msg = '* %s' % msg.strip('\n').replace('\n', '\n* ') report(msg, no_date=True, no_level=True) if common.global_vars['email']: email(msg, common.global_vars['email'], subject='Geppetto Test {%s} Update' % self.test_name)
def get_substrings(string, k, allowed_substrings): """Return all substrings of length >= <k> in <string> as a dict of substring:count where there are count occurrences of substring in <string> If <allowed_substrings> then only allow substrings from <allowed_substrings>. Performance ----------- The returned substring:count dict will no longer than <allowed_substrings> or <parent_keys> so the best way to guarantee performance is to find short key sets. """ common.report('get_substrings:k=%2d,allowed_substrings=%5d,size=%7d' % (k, len(allowed_substrings) if allowed_substrings else -1, len(string))) substrings = {} n = len(string) for i in range(n-k): pattern = string[i:i+k] if common.is_junk(pattern): continue if allowed_substrings: if not pattern in allowed_substrings: continue if not pattern in substrings: substrings[pattern] = 0 substrings[pattern] += 1 return substrings
def validate_child_offsets(file_names, offsets_dict, child_offsets_dict, k): """Check that a length k+1 substring is repeated no more than the length k substrings it contains""" substrings_list = _od_substrings(offsets_dict) child_substrings_list = _od_substrings(child_offsets_dict) common.report('validate_child_offsets(%d)' % k) for child_substring in child_substrings_list: assert(len(child_substring) == k+1) child_matching_offsets = get_matching_offsets(file_names, child_offsets_dict, child_substring) substring1 = child_substring[:-1] substring2 = child_substring[1:] if not substring1 in substrings_list: print 'No match for parent"%s", child="%s" in parent list' % (H(substring1), H(child_substring)) return False if not substring2 in substrings_list: print 'No match for parent"%s", child="%s" in parent list' % (H(substring2), H(child_substring)) return False matching_offsets1 = get_matching_offsets(file_names, offsets_dict, substring1) matching_offsets2 = get_matching_offsets(file_names, offsets_dict, substring1) if not is_offsets_greater(file_names, matching_offsets1, child_matching_offsets): print 'Mismatch on parent="%s", child="%s"' % (H(substring1), H(child_substring)) return False if not is_offsets_greater(file_names, matching_offsets2, child_matching_offsets): print 'Mismatch on parent="%s", child="%s"' % (H(substring2), H(child_substring)) return False return True
def handle_sysexit(self): """ Handle any sys exit exceptions ... tear down any processes necessary. :return: """ report(' ', no_date=True, no_level=True) update_status('Test Complete') self.complete()
def complete(self): """ Completely wrap up the test run. Do log moving, emailing, etc. """ note_queue = common.global_vars['test_notes'] notes = '' while not note_queue.empty(): notes += note_queue.get() checkpoint_results = '' for checkpoint, result in common.global_vars['checkpoints']: checkpoint_results += '\n+ %s: %s' % (result, checkpoint) # Set the overall test status if not set already. (Can also be set anywhere for test faults etc) if not common.global_vars['test_status']: common.global_vars['test_status'] = 'Passed' for checkpoint, status in common.global_vars['checkpoints']: if status.lower() != 'passed': common.global_vars['test_status'] = 'Failed' break # Format completed message. msg = ' Status: %s\n' % common.global_vars[ 'test_status'] #self.passed msg += ' Name: %s\n' % self.test_name msg += ' Config: %s\n' % self.config_name msg += ' Log: %s\n' % self.log_dir msg += ' Time: %s\n' % seconds_to_days_hours_min_sec_string( time.time() - self.start_time) if common.global_vars['commit_id']: msg += ' Commit: %s\n' % common.global_vars['commit_id'] msg += ' Traces: %s\n' % common.global_vars['traceback_count'] msg += ' Errors: %s\n' % common.global_vars['error_count'] if checkpoint_results: msg += '\nCheckpoints:\n%s\n' % checkpoint_results if notes: msg += '\nMessages:\n%s' % notes # Playing with the formatting a bit here to create better looking reports. msg = '* %s' % msg.strip('\n').replace('\n', '\n* ') msg = '*' * 70 + '\n' + msg + '\n' + '*' * 70 # Move geppetto logs to log dir. os.system('mv ./logs/*.log* %s/' % self.log_dir) if common.global_vars['email']: email(msg, common.global_vars['email'], subject='Geppetto Test {%s} Results' % self.test_name) # Do final printout of test to terminal. report(msg, no_date=True, no_level=True)
def do_start_message(self): msg = '...\n%s\n' % ('*' * 70) msg += '*\tTest: %s\n' % self.test_name msg += '*\tConfig: %s\n' % self.config_name msg += '*\tLog: %s\n' % self.log_dir email = common.global_vars['email'] if email: msg += '*\tEmails: %s\n' % email else: msg += '*\tEmails: Disabled.\n' msg += '%s' % ('*' * 70) report(msg)
def main(): box_num = raw_input("input box number:") over_time = float(raw_input("input work time:")) report_datas = common.report() log_path = path + "/log/" + box_num cases_run = [] for case in case_files: cases_run.append(case + ".run(log_path)") threads = [] threads.append(threading.Thread(target=common.heart_beat, args=(over_time,))) for t in threads: t.setDaemon(True) t.start() time_flag = 0 while 1: for t in threads: if t.is_alive(): for case_run in cases_run: response = eval(case_run) if response[response.keys()[0]]["state"]: print ".", else: print "F", report_datas.append_data(response) else: common.get_report_html(box_num, report_datas, report_path) time_flag = 1 print "" if time_flag: break print 99999
def set_init_params(self, configuration_dict, args, test_name, config_name): self.test_name = test_name self.config_name = config_name self.configuration_dict = configuration_dict self.start_time = time.time() # Require certain parameters to be in configuration file. Do all parsing here!!!!! EARLLLYYYY !!! required_params = [] for param in required_params: assert (param in self.configuration_dict), 'Param not found: %s' % param # Check that email given is valid. if args.email: if self._check_email_format(args.email): common.global_vars['email'] = args.email else: report('Invalid email address given. Emails disabled.', 'critical')
def _handle_stopped_runner(self, runner): try: common.report(runner, self._mail_kwargs) except: logger.exception("")
def modify_names(directory, action, position, input_string, replace_string=None, recursive=False, exclude=None, pattern=None, ignore_case=True, regex_syntax=False, report_file=None, ignore_symlinks=False, strip_chars=None): """ Modify the base name of files by adding, removing or replacing a user-defined string. """ pv.path(directory, "given", False, True) pv.compstr(action, "action", ["add", "remove", "replace"]) pv.compstr(position, "position", ["any", "prefix", "suffix"]) pv.string(input_string, "input string", False, common.get_invalid_chars()) action = action.lower() position = position.lower() directory = os.path.abspath(directory) if not directory.endswith(os.path.sep): directory += os.path.sep if report_file == None: simulate = False else: pv.path(report_file, "report", True, False) report_file = os.path.abspath(report_file) simulate = True if not replace_string == None: if not action == "replace": raise Exception("The replace string argument can only be used " \ "together with the action 'replace'.") else: pv.string(replace_string, "string False", False, common.get_invalid_chars()) if action == "add" and position == "any": raise Exception("The position 'any' cannot be used together with " \ "the action 'add'.") if len(input_string) == 0: raise Exception("The input string must not be empty.") else: pv.string(input_string, "input string", False, common.get_invalid_chars()) if not strip_chars == None: pv.string(strip_chars, "strip chars string", False, common.get_invalid_chars()) time_start = dt.now() list_content = [] list_excluded = [] list_renamed = [] list_skipped = [] regex = None if not pattern == None: regex = common.compile_regex(pattern, ignore_case, regex_syntax) list_content, list_excluded = \ common.get_files(directory, recursive, ignore_case, regex, exclude, ignore_symlinks) for item in list_content: list_files = item[1] __modify_names(list_files, list_renamed, list_skipped, action, position, input_string, replace_string, strip_chars) if simulate: explicit = None if exclude == None: exclude = False explicit = False elif exclude: explicit = False else: explicit = True list_header = [] list_header.append("Nomen File Name Modifier simulation report") list_header.append(["Report file name:", report_file]) list_header.append(["Directory:", directory]) list_header.append(["Recursive:", recursive]) list_header.append(["Ignore symlinks:", ignore_symlinks]) list_header.append(["Action to perform:", action.capitalize()]) list_header.append(["Position:", position.capitalize()]) list_header.append(["Input string:", "\"" + input_string + "\" " \ "(without double quotes)"]) if not replace_string == None: list_header.append(["Replace string:", "\"" + replace_string + \ "\" (without double quotes)"]) if strip_chars == None: list_header.append(["Strip chars:", "None"]) else: list_header.append(["Strip chars:", "\"" + strip_chars + "\" " \ "(without double quotes)"]) list_header.append(["Exclude files:", exclude]) list_header.append(["Explicit files:", explicit]) list_header.append(["Pattern:", pattern]) list_header.append(["Ignore case:", ignore_case]) list_header.append(["Regex syntax:", regex_syntax]) common.report(report_file, list_header, list_renamed, list_excluded, list_skipped, time_start) else: common.rename(list_renamed)
def rename_files(directory, rename_mode, separator=" ", recursive=False, padding=0, exclude=None, pattern=None, ignore_case=True, regex_syntax=False, report_file=None, ignore_symlinks=False, ignore_file_ext=False, custom_name=None, step=1, order_by=None): """ Rename the base name of files based on the name of the directory where they are stored in and add a numeric ID. """ pv.path(directory, "given", False, True) pv.compstr(rename_mode, "rename mode", ["fill-gaps", "increase", "keep-order", "rename-new"]) pv.intrange(padding, "padding", 0, 12, True) pv.string(separator, "seperator", False, common.get_invalid_chars()) pv.intvalue(step, "step", True, False, False) if not order_by == None: pv.compstr(order_by, "order by", ["accessed", "created", "modified"]) if not rename_mode == "keep-order": raise Exception("The order-by argument can only be used in " \ "combination with keep-order mode.") step = int(step) rename_mode = rename_mode.lower() directory = os.path.abspath(directory) if not directory.endswith(os.path.sep): directory += os.path.sep if report_file == None: simulate = False else: pv.path(report_file, "report", True, False) report_file = os.path.abspath(report_file) simulate = True if not custom_name == None: pv.string(custom_name, "custom file name", False, common.get_invalid_chars()) time_start = dt.now() list_content = [] list_excluded = [] list_renamed = [] list_skipped = [] regex = None if not pattern == None: regex = common.compile_regex(pattern, ignore_case, regex_syntax) list_content, list_excluded = \ common.get_files(directory, recursive, ignore_case, regex, exclude, ignore_symlinks, order_by) for item in list_content: list_files = item[1] if rename_mode == "fill-gaps": list_renamed, list_skipped = \ __rename_files_fill(list_files, list_renamed, list_skipped, separator, padding, True, ignore_file_ext, custom_name, step) elif rename_mode == "rename-new": list_renamed, list_skipped = \ __rename_files_fill(list_files, list_renamed, list_skipped, separator, padding, False, ignore_file_ext, custom_name, step) elif rename_mode == "keep-order": list_renamed, list_skipped = \ __rename_files_keep_order(list_files, list_renamed, list_skipped, separator, padding, ignore_file_ext, custom_name, step, order_by) else: raise Exception("An invalid rename mode was given.") if simulate: if padding == 0: padding = "Set automatically" else: padding = str(padding) explicit = None if exclude == None: exclude = False explicit = False elif exclude: explicit = False else: explicit = True if order_by == "accessed": order_by = "Access time" elif order_by == "created": order_by = "Creation time" elif order_by == "modified": order_by = "Modification time" else: order_by = "False" list_header = [] list_header.append("Nomen File Renamer simulation report") list_header.append(["Report file name:", report_file]) list_header.append(["Directory:", directory]) list_header.append(["Recursive:", recursive]) list_header.append(["Ignore symlinks:", ignore_symlinks]) list_header.append(["Rename mode:", rename_mode.capitalize()]) list_header.append(["Order by time:", order_by]) list_header.append(["Separator:", "\"" + separator + "\" " \ "(without double quotes)"]) list_header.append(["Numeric padding:", padding]) list_header.append(["Step size:", step]) list_header.append(["Exclude files:", exclude]) list_header.append(["Explicit files:", explicit]) list_header.append(["Pattern:", pattern]) list_header.append(["Ignore case:", ignore_case]) list_header.append(["Regex syntax:", regex_syntax]) common.report(report_file, list_header, list_renamed, list_excluded, list_skipped, time_start) else: common.rename(list_renamed)
if simulate: list_header = [] list_header.append("Nomen File Name Case Converter simulation report") list_header.append(["Report file name:", report_file]) list_header.append(["Directory:", directory]) list_header.append(["Recursive:", recursive]) list_header.append(["Ignore symlinks:", ignore_symlinks]) list_header.append(["Conflict mode:", conflict_mode.capitalize()]) list_header.append(["Case:", case.capitalize()]) list_header.append(["Lowercase config:", cfg_lower]) list_header.append(["Mixed case config:", cfg_mixed]) list_header.append(["Title case config:", cfg_title]) list_header.append(["Uppercase config:", cfg_upper]) common.report(report_file, list_header, list_renamed, list_excluded, list_skipped, time_start) else: common.rename(list_renamed) def get_version(): """ Return the version of this module. """ return __version__ def modify_names(directory, action, position, input_string, replace_string=None, recursive=False, exclude=None, pattern=None, ignore_case=True, regex_syntax=False, report_file=None, ignore_symlinks=False, strip_chars=None): """ Modify the base name of files by adding, removing or replacing a
def rename_extensions(directory, conflict_mode, extension, extension_target, recursive=False, ignore_case=True, report_file=None, ignore_symlinks=False): """ Rename the file extensions in the given directory and all of its sub-directories (if requested). """ pv.path(directory, "given", False, True) pv.compstr(conflict_mode, "conflict mode", ["rename", "skip"]) pv.string(extension, "extension", False, common.get_invalid_chars()) pv.string(extension_target, "target extension", False, common.get_invalid_chars()) conflict_mode = conflict_mode.lower() directory = os.path.abspath(directory) if not directory.endswith(os.path.sep): directory += os.path.sep if report_file == None: simulate = False else: pv.path(report_file, "report", True, False) report_file = os.path.abspath(report_file) simulate = True time_start = dt.now() list_content = [] list_excluded = [] list_extensions = [] list_renamed = [] list_skipped = [] if ";" in extension: while (";" * 2) in extension: extension = extension.replace((";" * 2), ";") list_temp = extension.split(";") for extension in list_temp: if not extension == "": list_extensions.append(extension) if len(list_extensions) == 0: raise Exception("The given extension list does not contain any " \ "extensions.") else: list_extensions.append(extension) pattern = "" for extension in list_extensions: pattern += "(.*\." + str(extension) + "$)|" pattern = pattern.rstrip("|") if ignore_case: regex = re.compile(pattern, re.IGNORECASE) else: regex = re.compile(pattern) list_content, list_excluded = \ common.get_files(directory, recursive, ignore_case, regex, False, ignore_symlinks) for item in list_content: list_files = item[1] list_renamed, list_skipped = \ __rename_extensions(list_files, list_extensions, list_renamed, list_skipped, conflict_mode, extension_target) if simulate: list_header = [] list_header.append("Nomen Extension Renamer simulation report") list_header.append(["Report file name:", report_file]) list_header.append(["Directory:", directory]) list_header.append(["Recursive:", recursive]) list_header.append(["Ignore symlinks:", ignore_symlinks]) list_header.append(["Conflict mode:", conflict_mode.capitalize()]) list_header.append(["Extensions:", extension]) list_header.append(["Target extension:", extension_target]) list_header.append(["Ignore case:", ignore_case]) common.report(report_file, list_header, list_renamed, list_excluded, list_skipped, time_start) else: common.rename(list_renamed)
def find_repeated_substrings(test_files): """Return the longest substring(s) s that is repeated in <test_files> according to rule: For each x in test_files: s occurs at least x['repeats'] times in x['text'] test_files[name] = {'text':text, 'repeats':repeats} """ common.note_time('start searching strings') common.report('find_repeated_substrings(%d,%d,%d)' % (len(test_files.keys()), _MIN_K, _MAX_K)) if not test_files: print 'no test files' return # Find the substrings that are repeated >= k times in files with k repeats # It is important to test shorter files first file_names = [x for x in test_files.keys()] file_names.sort(key = lambda x: len(test_files[x]['text'])) common.report('file_names:\n%s' % '\n'.join(['%8d:%3d: %s' % (len(test_files[name]['text']),test_files[name]['repeats'],name) for name in file_names])) # Start by finding all substrings of length _MIN_K which is typically 4 k = _MIN_K if False: print 'Pure Python' pattern_offsets_list = get_offsets_from_texts(file_names, test_files, k) else: print 'Cython rolling hash' text_list, repeats_list = test_files_to_text_repeats(file_names, test_files) # Get rid of expensive references to big strings for name in file_names: test_files[name]['text'] = None pattern_offsets_list = rolling_hash.get_offsets_from_texts(text_list, repeats_list, k, _JUNK_KEY_THRESHOLD) text_repeats_to_test_files(file_names, test_files, text_list, repeats_list) text_list = None min_repeats_list = None if False: # Does not work. !@#$ Find out why while k >= _MIN_K: pattern_offsets_list = rolling_hash.get_offsets_from_texts(text_list, repeats_list, k, _JUNK_KEY_THRESHOLD) if pattern_offsets_list[0]: break print 'reducing k %d=>%d' % (k, k // 2) k = k // 2 common.note_time('got substrings') offsets_dict = dict(zip(file_names, pattern_offsets_list)) # Work in increasing length of substrings, +1 per round offsets_dict_dict = {} k_list = [] for k in range(_MIN_K, _MAX_K): #sparsify the text for name in file_names: test_files[name]['text'] = common.sparsify_by_offsets(test_files[name]['text'], offsets_dict[name], k) offsets_dict_dict[k] = offsets_dict k_list.append(k) common.note_time('found %3d substrings of length >= %3d' % (len(offsets_dict[file_names[0]]), k)) child_offsets_dict = get_child_offsets(file_names, test_files, offsets_dict, k) if not child_offsets_dict: break offsets_dict = child_offsets_dict # The offsets dict may have too many repeats # Walk back through the offsets list to find the first one without excess repeats k_list.reverse() print '$' * 60 print 'k_list', k_list for k in k_list: print '-' * 60 offsets_dict = offsets_dict_dict[k] for key in sorted(offsets_dict[file_names[0]].keys()): print '%s:' % H(key), for name in file_names: if len(offsets_dict[name][key]) != test_files[name]['repeats']: print '"%s":%d,%d' % (name, len(offsets_dict[name][key]), test_files[name]['repeats']), print for name in file_names: for key, ofs_set in offsets_dict[name].items(): if len(ofs_set) > test_files[name]['repeats']: for n in file_names: del(offsets_dict[n][key]) #for name in file_names: # if not offsets_dict[name]: # del(offsets_dict[name]) print 'k=%d, offsets_dict=' % (k) for name in file_names: print ' ', name, ['"%s":%d'%(key, len(val)) for (key,val) in offsets_dict[name].items()] if all(offsets_dict.values()): break exit() # return last non-empty dict of offsets return offsets_dict
def get_child_offsets(file_names, test_files, offsets_dict, k): """ Given a set of substrings of length <k> defined by offsets in a set of test_files, return a dict of substrings of length k+1 where offsets_dict[<filename>][<substring>] is the set of offsets of <substring> in test_files[<filename>] <file_names> is keys of test_files in the desired sort order (shorter first) Performance ----------- This is the inner loop of the program. The returned dict will no longer than offsets_dict and string searches are on existing substrings + 1 character to left or right so there is not that much text to search. """ common.report('get_child_offsets(file_names=%d,test_files=%d,%d,substrings=%d,k=%d)' % (len(file_names), len(test_files), len(offsets_dict), len(offsets_dict.values()[0]), k)) parent_substrings = offsets_dict[file_names[0]].keys() child_offsets_dict = {} allowed_substrings = None for name in file_names: x = test_files[name] child_offsets_dict[name] = {} for key, ofs_set in offsets_dict[name].items(): # Use a list which unlike a set can be indexed and sorted ofs_list = sorted(ofs_set) # Remove parent offsets that would truncate substrings of length k+1 if ofs_list[0] == 0: del(ofs_list[0]) if ofs_list[-1]+k+1 == len(x['text']): del(ofs_list[-1]) # Create the child length k+1 substrings and add them to the child offsets dict # ofs1 is the offset of the k+1 substring key1 for ofs in ofs_list: for ofs1 in [ofs-1, ofs]: key1 = x['text'][ofs1:ofs1+k+1] #if len(key1) != k+1: # print 'key="%s", key1="%s"' % (key, key1) assert(len(key1) == k+1) if allowed_substrings: if not key1 in allowed_substrings: continue # Only allow keys with valid parents if not key1[1:] in parent_substrings or not key1[:-1] in parent_substrings: continue # Get rid of the junk too if common.is_junk(key1): continue # Got through all the filters. Add the new offset to the child dict if not key1 in child_offsets_dict[name].keys(): child_offsets_dict[name][key1] = set([]) child_offsets_dict[name][key1].add(ofs1) # Prune the entries with insufficient repeats unpruned_len = len(child_offsets_dict[name].keys()) for key, ofs_set in child_offsets_dict[name].items(): if len(ofs_set) < x['repeats']: del(child_offsets_dict[name][key]) # allowed_substrings is used as a filter in all but first pass through this loop allowed_substrings = child_offsets_dict[name].keys() common.report(' allowed_substrings=%3d,%3d,size=%7d' % (unpruned_len, len(allowed_substrings), len(x['text']))) # Need to go back and trim the substrings lists to allowed_substrings # If this results in a zero length list for any file then returns for name in file_names: for key in child_offsets_dict[name].keys(): if not key in allowed_substrings: del(child_offsets_dict[name][key]) if len(child_offsets_dict[name]) == 0: return None common.dump_dict('dumpfile_%03d' % (k+1), file_names, test_files, child_offsets_dict) if common.is_validate(): if not validate_child_offsets(file_names, offsets_dict, child_offsets_dict, k): raise ValueError for name in file_names: common.report('before=%3d,after=%3d,file=%s' % (len(offsets_dict[name]), len(child_offsets_dict[name]),name)) return child_offsets_dict