def send_update(self, datos=None):
"""
Sends an email update with status of run.
:return:
"""
note_queue = common.global_vars['test_notes']
notes = ''
while not note_queue.empty():
notes += note_queue.get()
checkpoint_results = ''
for checkpoint, result in common.global_vars['checkpoints']:
checkpoint_results += '\n+ %s: %s' % (result, checkpoint)
# Format completed message.
msg = 'Test Update.\n\n'
msg += ' Name: %s\n' % self.test_name
msg += ' Config: %s\n' % self.config_name
msg += ' Log: %s\n' % self.log_dir
msg += ' Time: %s\n' % seconds_to_days_hours_min_sec_string(
time.time() - self.start_time)
if checkpoint_results:
msg += '\nCheckpoints:\n%s\n' % checkpoint_results
if notes:
msg += '\nMessages:\n%s' % notes
# Playing with the formatting a bit here to create better looking reports.
msg = '* %s' % msg.strip('\n').replace('\n', '\n* ')
report(msg, no_date=True, no_level=True)
if common.global_vars['email']:
email(msg,
common.global_vars['email'],
subject='Geppetto Test {%s} Update' % self.test_name)
def get_substrings(string, k, allowed_substrings):
"""Return all substrings of length >= <k> in <string> as a dict of
substring:count where there are count occurrences of substring in
<string>
If <allowed_substrings> then only allow substrings from <allowed_substrings>.
Performance
-----------
The returned substring:count dict will no longer than
<allowed_substrings> or <parent_keys> so the best way to guarantee
performance is to find short key sets.
"""
common.report('get_substrings:k=%2d,allowed_substrings=%5d,size=%7d' %
(k, len(allowed_substrings) if allowed_substrings else -1, len(string)))
substrings = {}
n = len(string)
for i in range(n-k):
pattern = string[i:i+k]
if common.is_junk(pattern):
continue
if allowed_substrings:
if not pattern in allowed_substrings:
continue
if not pattern in substrings:
substrings[pattern] = 0
substrings[pattern] += 1
return substrings
def validate_child_offsets(file_names, offsets_dict, child_offsets_dict, k):
"""Check that a length k+1 substring is repeated no more than the length k substrings it
contains"""
substrings_list = _od_substrings(offsets_dict)
child_substrings_list = _od_substrings(child_offsets_dict)
common.report('validate_child_offsets(%d)' % k)
for child_substring in child_substrings_list:
assert(len(child_substring) == k+1)
child_matching_offsets = get_matching_offsets(file_names, child_offsets_dict, child_substring)
substring1 = child_substring[:-1]
substring2 = child_substring[1:]
if not substring1 in substrings_list:
print 'No match for parent"%s", child="%s" in parent list' % (H(substring1), H(child_substring))
return False
if not substring2 in substrings_list:
print 'No match for parent"%s", child="%s" in parent list' % (H(substring2), H(child_substring))
return False
matching_offsets1 = get_matching_offsets(file_names, offsets_dict, substring1)
matching_offsets2 = get_matching_offsets(file_names, offsets_dict, substring1)
if not is_offsets_greater(file_names, matching_offsets1, child_matching_offsets):
print 'Mismatch on parent="%s", child="%s"' % (H(substring1), H(child_substring))
return False
if not is_offsets_greater(file_names, matching_offsets2, child_matching_offsets):
print 'Mismatch on parent="%s", child="%s"' % (H(substring2), H(child_substring))
return False
return True
def handle_sysexit(self):
"""
Handle any sys exit exceptions ... tear down any processes necessary.
:return:
"""
report(' ', no_date=True, no_level=True)
update_status('Test Complete')
self.complete()
def complete(self):
"""
Completely wrap up the test run. Do log moving, emailing, etc.
"""
note_queue = common.global_vars['test_notes']
notes = ''
while not note_queue.empty():
notes += note_queue.get()
checkpoint_results = ''
for checkpoint, result in common.global_vars['checkpoints']:
checkpoint_results += '\n+ %s: %s' % (result, checkpoint)
# Set the overall test status if not set already. (Can also be set anywhere for test faults etc)
if not common.global_vars['test_status']:
common.global_vars['test_status'] = 'Passed'
for checkpoint, status in common.global_vars['checkpoints']:
if status.lower() != 'passed':
common.global_vars['test_status'] = 'Failed'
break
# Format completed message.
msg = ' Status: %s\n' % common.global_vars[
'test_status'] #self.passed
msg += ' Name: %s\n' % self.test_name
msg += ' Config: %s\n' % self.config_name
msg += ' Log: %s\n' % self.log_dir
msg += ' Time: %s\n' % seconds_to_days_hours_min_sec_string(
time.time() - self.start_time)
if common.global_vars['commit_id']:
msg += ' Commit: %s\n' % common.global_vars['commit_id']
msg += ' Traces: %s\n' % common.global_vars['traceback_count']
msg += ' Errors: %s\n' % common.global_vars['error_count']
if checkpoint_results:
msg += '\nCheckpoints:\n%s\n' % checkpoint_results
if notes:
msg += '\nMessages:\n%s' % notes
# Playing with the formatting a bit here to create better looking reports.
msg = '* %s' % msg.strip('\n').replace('\n', '\n* ')
msg = '*' * 70 + '\n' + msg + '\n' + '*' * 70
# Move geppetto logs to log dir.
os.system('mv ./logs/*.log* %s/' % self.log_dir)
if common.global_vars['email']:
email(msg,
common.global_vars['email'],
subject='Geppetto Test {%s} Results' % self.test_name)
# Do final printout of test to terminal.
report(msg, no_date=True, no_level=True)
def do_start_message(self):
msg = '...\n%s\n' % ('*' * 70)
msg += '*\tTest: %s\n' % self.test_name
msg += '*\tConfig: %s\n' % self.config_name
msg += '*\tLog: %s\n' % self.log_dir
email = common.global_vars['email']
if email:
msg += '*\tEmails: %s\n' % email
else:
msg += '*\tEmails: Disabled.\n'
msg += '%s' % ('*' * 70)
report(msg)
def main():
box_num = raw_input("input box number:")
over_time = float(raw_input("input work time:"))
report_datas = common.report()
log_path = path + "/log/" + box_num
cases_run = []
for case in case_files:
cases_run.append(case + ".run(log_path)")
threads = []
threads.append(threading.Thread(target=common.heart_beat, args=(over_time,)))
for t in threads:
t.setDaemon(True)
t.start()
time_flag = 0
while 1:
for t in threads:
if t.is_alive():
for case_run in cases_run:
response = eval(case_run)
if response[response.keys()[0]]["state"]:
print ".",
else:
print "F",
report_datas.append_data(response)
else:
common.get_report_html(box_num, report_datas, report_path)
time_flag = 1
print ""
if time_flag:
break
print 99999
def set_init_params(self, configuration_dict, args, test_name,
config_name):
self.test_name = test_name
self.config_name = config_name
self.configuration_dict = configuration_dict
self.start_time = time.time()
# Require certain parameters to be in configuration file. Do all parsing here!!!!! EARLLLYYYY !!!
required_params = []
for param in required_params:
assert (param
in self.configuration_dict), 'Param not found: %s' % param
# Check that email given is valid.
if args.email:
if self._check_email_format(args.email):
common.global_vars['email'] = args.email
else:
report('Invalid email address given. Emails disabled.',
'critical')
def _handle_stopped_runner(self, runner):
try:
common.report(runner, self._mail_kwargs)
except:
logger.exception("")
def _handle_stopped_runner(self, runner):
try:
common.report(runner, self._mail_kwargs)
except:
logger.exception("")
def modify_names(directory, action, position, input_string,
replace_string=None, recursive=False, exclude=None,
pattern=None, ignore_case=True, regex_syntax=False,
report_file=None, ignore_symlinks=False, strip_chars=None):
"""
Modify the base name of files by adding, removing or replacing a
user-defined string.
"""
pv.path(directory, "given", False, True)
pv.compstr(action, "action", ["add", "remove", "replace"])
pv.compstr(position, "position", ["any", "prefix", "suffix"])
pv.string(input_string, "input string", False, common.get_invalid_chars())
action = action.lower()
position = position.lower()
directory = os.path.abspath(directory)
if not directory.endswith(os.path.sep):
directory += os.path.sep
if report_file == None:
simulate = False
else:
pv.path(report_file, "report", True, False)
report_file = os.path.abspath(report_file)
simulate = True
if not replace_string == None:
if not action == "replace":
raise Exception("The replace string argument can only be used " \
"together with the action 'replace'.")
else:
pv.string(replace_string, "string False", False,
common.get_invalid_chars())
if action == "add" and position == "any":
raise Exception("The position 'any' cannot be used together with " \
"the action 'add'.")
if len(input_string) == 0:
raise Exception("The input string must not be empty.")
else:
pv.string(input_string, "input string", False,
common.get_invalid_chars())
if not strip_chars == None:
pv.string(strip_chars, "strip chars string", False,
common.get_invalid_chars())
time_start = dt.now()
list_content = []
list_excluded = []
list_renamed = []
list_skipped = []
regex = None
if not pattern == None:
regex = common.compile_regex(pattern, ignore_case, regex_syntax)
list_content, list_excluded = \
common.get_files(directory, recursive, ignore_case, regex, exclude,
ignore_symlinks)
for item in list_content:
list_files = item[1]
__modify_names(list_files, list_renamed, list_skipped, action,
position, input_string, replace_string, strip_chars)
if simulate:
explicit = None
if exclude == None:
exclude = False
explicit = False
elif exclude:
explicit = False
else:
explicit = True
list_header = []
list_header.append("Nomen File Name Modifier simulation report")
list_header.append(["Report file name:", report_file])
list_header.append(["Directory:", directory])
list_header.append(["Recursive:", recursive])
list_header.append(["Ignore symlinks:", ignore_symlinks])
list_header.append(["Action to perform:", action.capitalize()])
list_header.append(["Position:", position.capitalize()])
list_header.append(["Input string:", "\"" + input_string + "\" " \
"(without double quotes)"])
if not replace_string == None:
list_header.append(["Replace string:", "\"" + replace_string + \
"\" (without double quotes)"])
if strip_chars == None:
list_header.append(["Strip chars:", "None"])
else:
list_header.append(["Strip chars:", "\"" + strip_chars + "\" " \
"(without double quotes)"])
list_header.append(["Exclude files:", exclude])
list_header.append(["Explicit files:", explicit])
list_header.append(["Pattern:", pattern])
list_header.append(["Ignore case:", ignore_case])
list_header.append(["Regex syntax:", regex_syntax])
common.report(report_file, list_header, list_renamed, list_excluded,
list_skipped, time_start)
else:
common.rename(list_renamed)
def rename_files(directory, rename_mode, separator=" ", recursive=False,
padding=0, exclude=None, pattern=None, ignore_case=True,
regex_syntax=False, report_file=None, ignore_symlinks=False,
ignore_file_ext=False, custom_name=None, step=1,
order_by=None):
"""
Rename the base name of files based on the name of the directory where
they are stored in and add a numeric ID.
"""
pv.path(directory, "given", False, True)
pv.compstr(rename_mode, "rename mode",
["fill-gaps", "increase", "keep-order", "rename-new"])
pv.intrange(padding, "padding", 0, 12, True)
pv.string(separator, "seperator", False, common.get_invalid_chars())
pv.intvalue(step, "step", True, False, False)
if not order_by == None:
pv.compstr(order_by, "order by", ["accessed", "created", "modified"])
if not rename_mode == "keep-order":
raise Exception("The order-by argument can only be used in " \
"combination with keep-order mode.")
step = int(step)
rename_mode = rename_mode.lower()
directory = os.path.abspath(directory)
if not directory.endswith(os.path.sep):
directory += os.path.sep
if report_file == None:
simulate = False
else:
pv.path(report_file, "report", True, False)
report_file = os.path.abspath(report_file)
simulate = True
if not custom_name == None:
pv.string(custom_name, "custom file name", False,
common.get_invalid_chars())
time_start = dt.now()
list_content = []
list_excluded = []
list_renamed = []
list_skipped = []
regex = None
if not pattern == None:
regex = common.compile_regex(pattern, ignore_case, regex_syntax)
list_content, list_excluded = \
common.get_files(directory, recursive, ignore_case, regex, exclude,
ignore_symlinks, order_by)
for item in list_content:
list_files = item[1]
if rename_mode == "fill-gaps":
list_renamed, list_skipped = \
__rename_files_fill(list_files, list_renamed, list_skipped,
separator, padding, True, ignore_file_ext,
custom_name, step)
elif rename_mode == "rename-new":
list_renamed, list_skipped = \
__rename_files_fill(list_files, list_renamed, list_skipped,
separator, padding, False,
ignore_file_ext, custom_name, step)
elif rename_mode == "keep-order":
list_renamed, list_skipped = \
__rename_files_keep_order(list_files, list_renamed,
list_skipped, separator, padding,
ignore_file_ext, custom_name, step,
order_by)
else:
raise Exception("An invalid rename mode was given.")
if simulate:
if padding == 0:
padding = "Set automatically"
else:
padding = str(padding)
explicit = None
if exclude == None:
exclude = False
explicit = False
elif exclude:
explicit = False
else:
explicit = True
if order_by == "accessed":
order_by = "Access time"
elif order_by == "created":
order_by = "Creation time"
elif order_by == "modified":
order_by = "Modification time"
else:
order_by = "False"
list_header = []
list_header.append("Nomen File Renamer simulation report")
list_header.append(["Report file name:", report_file])
list_header.append(["Directory:", directory])
list_header.append(["Recursive:", recursive])
list_header.append(["Ignore symlinks:", ignore_symlinks])
list_header.append(["Rename mode:", rename_mode.capitalize()])
list_header.append(["Order by time:", order_by])
list_header.append(["Separator:", "\"" + separator + "\" " \
"(without double quotes)"])
list_header.append(["Numeric padding:", padding])
list_header.append(["Step size:", step])
list_header.append(["Exclude files:", exclude])
list_header.append(["Explicit files:", explicit])
list_header.append(["Pattern:", pattern])
list_header.append(["Ignore case:", ignore_case])
list_header.append(["Regex syntax:", regex_syntax])
common.report(report_file, list_header, list_renamed, list_excluded,
list_skipped, time_start)
else:
common.rename(list_renamed)
if simulate:
list_header = []
list_header.append("Nomen File Name Case Converter simulation report")
list_header.append(["Report file name:", report_file])
list_header.append(["Directory:", directory])
list_header.append(["Recursive:", recursive])
list_header.append(["Ignore symlinks:", ignore_symlinks])
list_header.append(["Conflict mode:", conflict_mode.capitalize()])
list_header.append(["Case:", case.capitalize()])
list_header.append(["Lowercase config:", cfg_lower])
list_header.append(["Mixed case config:", cfg_mixed])
list_header.append(["Title case config:", cfg_title])
list_header.append(["Uppercase config:", cfg_upper])
common.report(report_file, list_header, list_renamed, list_excluded,
list_skipped, time_start)
else:
common.rename(list_renamed)
def get_version():
"""
Return the version of this module.
"""
return __version__
def modify_names(directory, action, position, input_string,
replace_string=None, recursive=False, exclude=None,
pattern=None, ignore_case=True, regex_syntax=False,
report_file=None, ignore_symlinks=False, strip_chars=None):
"""
Modify the base name of files by adding, removing or replacing a
def rename_extensions(directory,
conflict_mode,
extension,
extension_target,
recursive=False,
ignore_case=True,
report_file=None,
ignore_symlinks=False):
"""
Rename the file extensions in the given directory and all of its
sub-directories (if requested).
"""
pv.path(directory, "given", False, True)
pv.compstr(conflict_mode, "conflict mode", ["rename", "skip"])
pv.string(extension, "extension", False, common.get_invalid_chars())
pv.string(extension_target, "target extension", False,
common.get_invalid_chars())
conflict_mode = conflict_mode.lower()
directory = os.path.abspath(directory)
if not directory.endswith(os.path.sep):
directory += os.path.sep
if report_file == None:
simulate = False
else:
pv.path(report_file, "report", True, False)
report_file = os.path.abspath(report_file)
simulate = True
time_start = dt.now()
list_content = []
list_excluded = []
list_extensions = []
list_renamed = []
list_skipped = []
if ";" in extension:
while (";" * 2) in extension:
extension = extension.replace((";" * 2), ";")
list_temp = extension.split(";")
for extension in list_temp:
if not extension == "":
list_extensions.append(extension)
if len(list_extensions) == 0:
raise Exception("The given extension list does not contain any " \
"extensions.")
else:
list_extensions.append(extension)
pattern = ""
for extension in list_extensions:
pattern += "(.*\." + str(extension) + "$)|"
pattern = pattern.rstrip("|")
if ignore_case:
regex = re.compile(pattern, re.IGNORECASE)
else:
regex = re.compile(pattern)
list_content, list_excluded = \
common.get_files(directory, recursive, ignore_case, regex, False,
ignore_symlinks)
for item in list_content:
list_files = item[1]
list_renamed, list_skipped = \
__rename_extensions(list_files, list_extensions, list_renamed,
list_skipped, conflict_mode, extension_target)
if simulate:
list_header = []
list_header.append("Nomen Extension Renamer simulation report")
list_header.append(["Report file name:", report_file])
list_header.append(["Directory:", directory])
list_header.append(["Recursive:", recursive])
list_header.append(["Ignore symlinks:", ignore_symlinks])
list_header.append(["Conflict mode:", conflict_mode.capitalize()])
list_header.append(["Extensions:", extension])
list_header.append(["Target extension:", extension_target])
list_header.append(["Ignore case:", ignore_case])
common.report(report_file, list_header, list_renamed, list_excluded,
list_skipped, time_start)
else:
common.rename(list_renamed)
def find_repeated_substrings(test_files):
"""Return the longest substring(s) s that is repeated in <test_files>
according to rule:
For each x in test_files:
s occurs at least x['repeats'] times in x['text']
test_files[name] = {'text':text, 'repeats':repeats}
"""
common.note_time('start searching strings')
common.report('find_repeated_substrings(%d,%d,%d)' % (len(test_files.keys()), _MIN_K, _MAX_K))
if not test_files:
print 'no test files'
return
# Find the substrings that are repeated >= k times in files with k repeats
# It is important to test shorter files first
file_names = [x for x in test_files.keys()]
file_names.sort(key = lambda x: len(test_files[x]['text']))
common.report('file_names:\n%s' % '\n'.join(['%8d:%3d: %s' %
(len(test_files[name]['text']),test_files[name]['repeats'],name) for name in file_names]))
# Start by finding all substrings of length _MIN_K which is typically 4
k = _MIN_K
if False:
print 'Pure Python'
pattern_offsets_list = get_offsets_from_texts(file_names, test_files, k)
else:
print 'Cython rolling hash'
text_list, repeats_list = test_files_to_text_repeats(file_names, test_files)
# Get rid of expensive references to big strings
for name in file_names:
test_files[name]['text'] = None
pattern_offsets_list = rolling_hash.get_offsets_from_texts(text_list, repeats_list, k,
_JUNK_KEY_THRESHOLD)
text_repeats_to_test_files(file_names, test_files, text_list, repeats_list)
text_list = None
min_repeats_list = None
if False:
# Does not work. !@#$ Find out why
while k >= _MIN_K:
pattern_offsets_list = rolling_hash.get_offsets_from_texts(text_list, repeats_list, k,
_JUNK_KEY_THRESHOLD)
if pattern_offsets_list[0]:
break
print 'reducing k %d=>%d' % (k, k // 2)
k = k // 2
common.note_time('got substrings')
offsets_dict = dict(zip(file_names, pattern_offsets_list))
# Work in increasing length of substrings, +1 per round
offsets_dict_dict = {}
k_list = []
for k in range(_MIN_K, _MAX_K):
#sparsify the text
for name in file_names:
test_files[name]['text'] = common.sparsify_by_offsets(test_files[name]['text'],
offsets_dict[name], k)
offsets_dict_dict[k] = offsets_dict
k_list.append(k)
common.note_time('found %3d substrings of length >= %3d' % (len(offsets_dict[file_names[0]]), k))
child_offsets_dict = get_child_offsets(file_names, test_files, offsets_dict, k)
if not child_offsets_dict:
break
offsets_dict = child_offsets_dict
# The offsets dict may have too many repeats
# Walk back through the offsets list to find the first one without excess repeats
k_list.reverse()
print '$' * 60
print 'k_list', k_list
for k in k_list:
print '-' * 60
offsets_dict = offsets_dict_dict[k]
for key in sorted(offsets_dict[file_names[0]].keys()):
print '%s:' % H(key),
for name in file_names:
if len(offsets_dict[name][key]) != test_files[name]['repeats']:
print '"%s":%d,%d' % (name, len(offsets_dict[name][key]), test_files[name]['repeats']),
print
for name in file_names:
for key, ofs_set in offsets_dict[name].items():
if len(ofs_set) > test_files[name]['repeats']:
for n in file_names:
del(offsets_dict[n][key])
#for name in file_names:
# if not offsets_dict[name]:
# del(offsets_dict[name])
print 'k=%d, offsets_dict=' % (k)
for name in file_names:
print ' ', name, ['"%s":%d'%(key, len(val)) for (key,val) in offsets_dict[name].items()]
if all(offsets_dict.values()):
break
exit()
# return last non-empty dict of offsets
return offsets_dict
def get_child_offsets(file_names, test_files, offsets_dict, k):
""" Given a set of substrings of length <k> defined by offsets in a set of
test_files, return a dict of substrings of length k+1
where
offsets_dict[<filename>][<substring>] is the set of offsets of <substring>
in test_files[<filename>]
<file_names> is keys of test_files in the desired sort order (shorter first)
Performance
-----------
This is the inner loop of the program.
The returned dict will no longer than offsets_dict and string searches are on existing
substrings + 1 character to left or right so there is not that much text to search.
"""
common.report('get_child_offsets(file_names=%d,test_files=%d,%d,substrings=%d,k=%d)' %
(len(file_names), len(test_files), len(offsets_dict), len(offsets_dict.values()[0]), k))
parent_substrings = offsets_dict[file_names[0]].keys()
child_offsets_dict = {}
allowed_substrings = None
for name in file_names:
x = test_files[name]
child_offsets_dict[name] = {}
for key, ofs_set in offsets_dict[name].items():
# Use a list which unlike a set can be indexed and sorted
ofs_list = sorted(ofs_set)
# Remove parent offsets that would truncate substrings of length k+1
if ofs_list[0] == 0:
del(ofs_list[0])
if ofs_list[-1]+k+1 == len(x['text']):
del(ofs_list[-1])
# Create the child length k+1 substrings and add them to the child offsets dict
# ofs1 is the offset of the k+1 substring key1
for ofs in ofs_list:
for ofs1 in [ofs-1, ofs]:
key1 = x['text'][ofs1:ofs1+k+1]
#if len(key1) != k+1:
# print 'key="%s", key1="%s"' % (key, key1)
assert(len(key1) == k+1)
if allowed_substrings:
if not key1 in allowed_substrings:
continue
# Only allow keys with valid parents
if not key1[1:] in parent_substrings or not key1[:-1] in parent_substrings:
continue
# Get rid of the junk too
if common.is_junk(key1):
continue
# Got through all the filters. Add the new offset to the child dict
if not key1 in child_offsets_dict[name].keys():
child_offsets_dict[name][key1] = set([])
child_offsets_dict[name][key1].add(ofs1)
# Prune the entries with insufficient repeats
unpruned_len = len(child_offsets_dict[name].keys())
for key, ofs_set in child_offsets_dict[name].items():
if len(ofs_set) < x['repeats']:
del(child_offsets_dict[name][key])
# allowed_substrings is used as a filter in all but first pass through this loop
allowed_substrings = child_offsets_dict[name].keys()
common.report(' allowed_substrings=%3d,%3d,size=%7d' %
(unpruned_len, len(allowed_substrings), len(x['text'])))
# Need to go back and trim the substrings lists to allowed_substrings
# If this results in a zero length list for any file then returns
for name in file_names:
for key in child_offsets_dict[name].keys():
if not key in allowed_substrings:
del(child_offsets_dict[name][key])
if len(child_offsets_dict[name]) == 0:
return None
common.dump_dict('dumpfile_%03d' % (k+1), file_names, test_files, child_offsets_dict)
if common.is_validate():
if not validate_child_offsets(file_names, offsets_dict, child_offsets_dict, k):
raise ValueError
for name in file_names:
common.report('before=%3d,after=%3d,file=%s' % (len(offsets_dict[name]),
len(child_offsets_dict[name]),name))
return child_offsets_dict
