def handle(self, pass_nr, segment):
if segment.name != "0":
return
prefixes = None
for type in self.filetypes.keys():
if re_search(type, segment.filename):
prefixes = self.filetypes[type]
break
if not prefixes:
logger.info("File %s is not processed" % segment.filename)
return
if len(self.text) == 0:
self.license("LICENSE.txt")
logger.info("file %s is processed from %d with %s %s %s" %
(segment.filename, prefixes['line_nr'], prefixes['start'],
prefixes['middle'], prefixes['end']))
first_license_line = prefixes['line_nr']
while first_license_line < len(segment.text) and \
not re_search(prefixes['matcher'], segment.text[first_license_line]) and \
not re_search(prefixes['stopper'], segment.text[first_license_line]):
first_license_line += 1
license_found = first_license_line != len(segment.text) and \
not re_search(prefixes['stopper'], segment.text[first_license_line])
if license_found:
logging.info("old license was found starting on line %d" %
first_license_line)
last_license_line = first_license_line
while last_license_line < len(segment.text) and re_search(
prefixes['matcher'], segment.text[last_license_line]):
last_license_line += 1
if last_license_line == len(segment.text):
last_license_line -= 1
text = [prefixes['start'] +
"\n"] if prefixes['start'] is not None else []
for line in self.text:
text.append(prefixes['middle'] + line)
if prefixes['end'] is not None:
text.append(prefixes['end'] + "\n")
if license_found:
logger.info("replacing license text")
segment.text = segment.text[:
first_license_line] + text + segment.text[
last_license_line:]
else:
logger.info("inserting license text")
segment.text = segment.text[:prefixes[
'line_nr']] + text + segment.text[prefixes['line_nr']:]
segment.modified = True
def collect(self):
files = set()
for pattern in self.patterns:
for file in glob.glob(pattern, recursive=True):
for configuration in self.configurations:
if any([
re_search(regex, file)
for regex in configuration.filename_regexes
]) and not any([
re_search(regex, file)
for regex in configuration.filename_excludes
]):
files.add(file)
return files
def fetch_values(self, text, regex):
"""
Fetch macro values from the lines of TEXT based on regex
:param text: the lines that may hold keys and values fr macros
:param regex: if that matches a line then group(1) is treated as key and group(2) is treated as value and
stored into the macroset to be references in snippets
:return:
"""
for line in text:
match = re_search(regex, line)
if match and match.lastindex >= 2:
self.macroset(match.group(1), match.group(2))
def handle(self, pass_nr, segment):
if not re_search(ShellSnippet.start_line, segment.text[0]):
return
exec = self.init_exec()
for line in segment.text[1:-1]:
exec.append(line.rstrip())
text = self.execute(exec)
if text:
store_snippet(segment.filename, segment.name,
segment.text[0:1] + text + [segment.text[-1]])
def postprocess(self, segment):
""" postprocess the lines in case there is any REPLACE command on the first line"""
match = re.search(r"REPLACE\s+(.*)$", segment.text[0])
if match:
replaces = self.get_replace_tuples(match.group(1))
else:
replaces = None
match = re.search(r"KILL (.*)", segment.text[0])
if match:
kills = self.get_kill_regexes(match.group(1))
else:
kills = None
trim = re.search(r"TRIM", segment.text[0])
trim_start = 2 if trim and segment.text[1].startswith('```') else 1
if replaces is None and kills is None and not trim:
return segment.text
# process the intermediate lines, not the first and the last
i = 1
while i < len(segment.text) - 1:
killed = False
if kills is not None:
for kill in kills:
if re_search(kill, segment.text[i]):
segment.text = segment.text[:i] + segment.text[i + 1:]
killed = True
break
if killed:
continue
if replaces is not None:
for replace in replaces:
segment.text[i] = re_sub(replace[0], replace[1],
segment.text[i])
i += 1
trim_size = 0
if trim:
trim_size = self.max_line_length(segment, trim_size, trim_start)
trim_size = self.calculate_trim_size(segment, trim_size,
trim_start)
self.trim_lines(segment, trim_size, trim_start)
# save the user from shooting the foot
# may accidentally remove the new-line characters, but even then the last line has to have a new line
if len(segment.text[-2]) == 0 or segment.text[-2][-1] != "\n":
segment.text[-2] = segment.text[-2] + "\n"
return segment.text
def handle(self, pass_nr, segment):
if segment.name == "0":
# at the start of a file there is no matching defined
self.regex = None
for line in segment.text[1:-1]:
if re.search(r"NO\s+MATCH\W", line):
self.regex = None
continue
startline = re.search(r"MATCH\s+(.*)", line)
if startline:
self.regex = startline.group(1)
continue
if self.regex:
match = re_search(self.regex, line)
if match and match.lastindex >= 2:
self.set(match.group(1), match.group(2))
def handle(self, pass_nr, segment):
match = re_search(PySnippet.start_line, segment.text[0])
if not match:
return
globals_name = segment.parameter("GLOBALS")
if not globals_name:
globals_name = "_"
if globals_name:
if globals_name == "*":
snippet_globals = globals()
else:
if globals_name not in snippet_globals_dict:
snippet_globals_dict[globals_name] = dict()
snippet_globals = snippet_globals_dict[globals_name]
code_out = StringIO()
code_err = StringIO()
code = ''
for line in segment.text[1:-1]:
code = code + line
sysstdout = sys.stdout
sysstderr = sys.stderr
# capture output and errors
sys.stdout = code_out
sys.stderr = code_err
try:
exec(code, snippet_globals)
except Exception as e:
logger.error("Python snippet threw up")
logger.error(e)
# restore stdout and stderr
sys.stdout = sysstdout
sys.stderr = sysstderr
s = code_err.getvalue()
if len(s) > 0:
logger.error("Python snippet is erroneous")
logger.error(s)
text = code_out.getvalue()
store_snippet(segment.filename, segment.name,
segment.text[0:1] + [text] + [segment.text[-1]])
code_out.close()
code_err.close()
def read(self):
"""
Read the file and split up into segments
:return: the File object that contains the segments of the file
"""
segments = []
segment = None
end_regex = None
with open(self.filename, 'r') as f:
for line in f:
is_start, name, end_regex_ = self._startsegment(line)
if is_start:
segment = Segment(name, self.filename)
segments.append(segment)
end_regex = end_regex_
segment.add(line)
self.analyze_parameters(line, segment)
continue
if end_regex and re_search(end_regex, line):
logger.debug("line '%s' matches '%s' segment ending" %
(line, end_regex))
segment.add(line)
segment = None
end_regex = None
continue
if segment is None:
name = self.next_segment()
logger.debug("Creating new unnamed segment '%s'" % name)
segment = Segment(name, self.filename)
segments.append(segment)
segment.add(line)
self.chain_segments(segments)
return File(self.filename, segments)
def _startsegment(self, line):
"""
Check the line against the regular expressions.
:param line: the line to check against the regular expressions
:param regexes: the iterable of the regular expressions
:return: A tulpe (match, name, regex) where
match is False when there is no start regex mathing the line and then the returned value is (False, None,None)
When there is a match then the returned tuple is (True,name,regex), where
name is the name of the segment, aither from the first capture group or the next generated number
regex is the segment end matching regular expression
"""
for regex in self.regexes:
match = re_search(regex[0], line)
if match:
if match.lastindex and len(match.group(1)) > 0:
logger.debug("line '%s' matches '%s' and name is '%s'" %
(line, regex, match.group(1)))
return True, match.group(1), regex[1]
else:
name = self.next_segment()
logger.debug("line '%s' matches '%s' and name is '%s'" %
(line, regex, name))
return True, name, regex[1]
return False, None, None
def reader(self, configurations, filename):
for configuration in configurations:
for regex in configuration.filename_regexes:
if re_search(regex, filename):
return FileReader(filename, regexes=configuration.regexes)