def file_diff(a,b,extend_name=""):
"""
Compare two files for differences, print only differences to console and
put in a file in logs directory.
"""
#a = "/home/RunFromHere/logs/10.38.36.67.txt"
#b = "/home/RunFromHere/logs/10.38.36.167.txt"
#difference = ("/home/RunFromHere/logs/difference_%s.txt" % c)
difference = ("logs/difference_%s.txt" % extend_name)
#filecmp = difflib.Differ()
z = filecmp.cmp(a,b)
if z == True:
print("\n\nThe files are the same")
return(True)
else:
with open (a) as File1:
c = File1.readlines()
with open (b) as File2:
d = File2.readlines()
print('\n')
for line in difflib.context_diff(c,d, fromfile=(a), tofile=(b), n=0):
print((line))
with open (difference, 'w') as differ:
for line in difflib.context_diff(c,d, fromfile=(a), tofile=(b), n=0):
differ.write(line)
return(False) ## false would mean that there are differences
def test(self, ctx):
self.perform(ctx)
#displayed = True
# launch the command
if self.out:
elfmake.mkdir(str(self.out.parent()))
out_stream = open(str(self.out), "w")
else:
out_stream = NULL
if self.err:
err_stream = open(str(self.err), "w")
else:
err_stream = NULL
if self.input:
in_stream = open(str(self.input), "r")
else:
in_stream = NULL
cmd = action.make_line(self.cmd)
if elfmake.verbose:
ctx.print_info("running %s" % cmd)
rc = subprocess.call(cmd, stdin = in_stream, stdout = out_stream, stderr = err_stream, shell = True)
if rc <> 0:
self.failure(ctx, "return code = %d, command = %s" % (rc, cmd))
return
# compare output if any
if self.out:
if not self.out_ref.exists():
self.info(ctx, "no reference file for output, creating it!")
elfmake.mkdir(str(self.out_ref.parent()))
shutil.copyfile(str(self.out), str(self.out_ref))
else:
c = 0
for l in difflib.context_diff(open(str(self.out), "r").readlines(), open(str(self.out_ref), "r").readlines()):
c += 1
if c:
self.failure(ctx, "different output stream")
return
# compare error if any
if self.err:
if not os.path.exists(self.err_ref):
self.info(ctx, "no reference file for error, creating it!")
shutil.copyfile(self.err, self.err_ref)
else:
c = 0
for l in difflib.context_diff(open(self.err, "r").readlines(), open(self.err_ref, "r").readlines()):
c += 1
if c:
self.failure(ctx, "different error stream")
return
# display result
self.success(ctx)
def main():
parser = argparse.ArgumentParser(description="Compare two flattened UW XML files.")
parser.add_argument("xml_file_1",
help="The first XML file.")
parser.add_argument("xml_file_2",
help="The second XML file.")
args = parser.parse_args()
if not os.path.isfile(args.xml_file_1):
sys.exit("ERROR - Can't find first XML file: {}".format(args.xml_file_1))
if not os.path.isfile(args.xml_file_2):
sys.exit("ERROR - Can't find second XML file: {}".format(args.xml_file_2))
tree1 = ElementTree.parse(args.xml_file_1)
tree2 = ElementTree.parse(args.xml_file_2)
dict1 = _elementToDict(tree1.getroot())
dict2 = _elementToDict(tree2.getroot())
diff = dict_diff(dict1, dict2)
if(len(diff)):
print "Differences:"
for key, value in diff.items():
first = str(value[0]).split('),')
second = str(value[1]).split('),')
result = list(difflib.context_diff(first, second, lineterm=""))
pprint(result)
else:
print "No Differences"
def func(args):
# Get modification times
fromdate = time.ctime(os.stat(args.FILE1).st_mtime)
todate = time.ctime(os.stat(args.FILE2).st_mtime)
# Open fromfile
try:
with open(args.FILE1, 'U') as fd:
fromlines = fd.readlines()
except IOError:
print("Error opening file " + args.FILE1, file=sys.stderr)
# Open tofile
try:
with open(args.FILE2, 'U') as fd:
tolines = fd.readlines()
except IOError:
print("Error opening file " + args.FILE2, file=sys.stderr)
# Create diff
if args.unified:
diff = difflib.unified_diff(fromlines, tolines, args.FILE1, args.FILE2,
fromdate, todate, n=args.lines)
elif args.ndiff:
diff = difflib.ndiff(fromlines, tolines)
elif args.html:
diff = difflib.HtmlDiff().make_file(fromlines, tolines, args.FILE1,
args.FILE2, context=args.context,
numlines=args.lines)
else:
diff = difflib.context_diff(fromlines, tolines, args.FILE1, args.FILE2,
fromdate, todate, n=args.lines)
# we're using writelines because diff is a generator
sys.stdout.writelines(diff)
def xml_diff(xml_1, xml_2, ignore_order=True):
"""This function will generate string showing the differences
between two json data (similar to the Linux 'diff' command output)
ignore_order - Flag denotes whether to ignore the
order of dictionary keys and list items
Arguments:
xml_1, xml_1 - XML contents (strings) to be compared
Return:
String describing the differences (similar to the
Linux 'diff' command output)
"""
if ignore_order:
xml_1 = ordered_xml(xml_1)
xml_2 = ordered_xml(xml_2)
pp1 = pretty_xml(xml_1)
pp2 = pretty_xml(xml_2)
pp_lst1 = [item for item in pp1.split('\n') if item.strip()]
pp_lst2 = [item for item in pp2.split('\n') if item.strip()]
diff_lines = [line for line in difflib.context_diff(pp_lst1, pp_lst2)]
return '\n'.join(diff_lines)
def test_rebuilding_does_not_generate_new_content(self):
with resource("__buckconfig_common.soy") as buckconfig_common, resource(
"files-and-dirs/buckconfig.soy"
) as buckconfig, resource(
"generate_buckconfig_aliases", suffix=".pex"
) as script, named_temp_file() as temp_out:
subprocess.check_call(
[
sys.executable,
script,
"--buckconfig",
buckconfig,
"--buckconfig-aliases",
buckconfig_common,
"--buckconfig-aliases-dest",
temp_out,
]
)
with open(temp_out, "r") as actual_in, open(
buckconfig_common, "r"
) as expected_in:
actual = actual_in.read()
expected = expected_in.read()
if actual != expected:
diff = "\n".join(
difflib.context_diff(expected.splitlines(), actual.splitlines())
)
raise AssertionError(
"Expected unchanged buckconfig_common. Got\n{}".format(diff)
)
def test_fixture(url, expected_html):
print('url=%s' % url)
actual_html = readable(url)
# Write the HTML for later diagnosis
_, actual_html_fn = tempfile.mkstemp('readabilitytest')
os.close(_)
_, expected_html_fn = tempfile.mkstemp('readabilitytest')
os.close(_)
with codecs.open(actual_html_fn, 'w', DEFAULT_ENCODING) as f:
f.write(actual_html)
with codecs.open(expected_html_fn, 'w', DEFAULT_ENCODING) as f:
f.write(expected_html)
# Verify that there is no 'diff' between the two versions
diff = list(difflib.context_diff(
actual_html.splitlines(), expected_html.splitlines()))
diff_string = '\n'.join(diff)
assert not diff, (
'readable version differs; diff:\n{0}\n'
'expected: {1}\n'
'actual: {2}').format(
diff_string,
expected_html_fn,
actual_html_fn,
)
def calculate_diff(self, new, old, zone):
return "\n".join(context_diff(
a=old.splitlines(),
b=new.splitlines(),
fromfile=str(zone.domain_active_serial) + '-' + zone.domain_name,
tofile=str(zone.domain_serial) + '-' + zone.domain_name,
lineterm=''))
def check():
mech = mechanize.Browser()
url = settings.watch_url
soup = BeautifulSoup(mech.open(url).read())
links = [x['href'] for x in soup.findAll('a')]
links = filter(is_magnet_link, links)
seen_links_loc = '%s/seen.list' % settings.working_dir
if os.path.exists(seen_links_loc):
with open(seen_links_loc, 'r') as seen:
seen_links = [x.strip() for x in seen.readlines()]
diff = difflib.context_diff(seen_links, links)
if links != seen_links:
for d in diff:
if d[0:1] == '+':
magnet_link = d[2:]
print("adding new magnet link: " + magnet_link)
subprocess.call(
[
'bash',
'%s/convert.sh' % settings.working_dir,
settings.torrent_watch_dir,
magnet_link])
write_new_seen(links, seen_links_loc)
else:
write_new_seen(links, seen_links_loc)
def _check_register_format(reg, build_monitor):
"""
Check the register format.
We do this by generating a perfectly formatted version and
then comparing it to the version on disc.
Any discrepancy will be reported as a design nonconformity.
"""
register_text = reg['text']
reference_text = _format_register(reg['data'])
if reference_text != register_text:
reference_lines = reference_text.splitlines()
register_lines = register_text.splitlines()
iline = 0
for iline, lines in enumerate(zip(reference_lines, register_lines)):
(ref_line, reg_line) = lines
if ref_line != reg_line:
break
diff_text = '\n'.join((line for line in difflib.context_diff(
reference_lines,
register_lines,
'reference_format',
'current_register')))
msg = 'Dependencies register format error '\
'on line {line}:\n{diff}'.format(line = iline,
diff = diff_text)
build_monitor.report_nonconformity(tool = 'da.check.dependencies',
msg_id = 'E6005',
msg = msg,
file = reg['filepath'])
def main():
usage = "usage: %prog [options] fromfile tofile"
parser = optparse.OptionParser(usage)
parser.add_option("-c", action="store_true", default=False, help='Produce a context format diff (default)')
parser.add_option("-u", action="store_true", default=False, help='Produce a unified format diff')
parser.add_option("-m", action="store_true", default=False, help='Produce HTML side by side diff (can use -c and -l in conjunction)')
parser.add_option("-n", action="store_true", default=False, help='Produce a ndiff format diff')
parser.add_option("-l", "--lines", type="int", default=3, help='Set number of context lines (default 3)')
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
sys.exit(1)
if len(args) != 2:
parser.error("need to specify both a fromfile and tofile")
n = options.lines
fromfile, tofile = args
fromdate = time.ctime(os.stat(fromfile).st_mtime)
todate = time.ctime(os.stat(tofile).st_mtime)
fromlines = open(fromfile, 'U').readlines()
tolines = open(tofile, 'U').readlines()
if options.u:
diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)
elif options.n:
diff = difflib.ndiff(fromlines, tolines)
elif options.m:
diff = difflib.HtmlDiff().make_file(fromlines,tolines,fromfile,tofile,context=options.c,numlines=n)
else:
diff = difflib.context_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)
sys.stdout.writelines(diff)
def main():
print 'Getting initial reference file...'
refHTML = getHTML( remoteURL )
while( refHTML == False ):
print '...retrying...'
refHTML = getHTML( remoteURL )
writeHTML( localFile, refHTML )
while True:
print 'Getting latest page...', time.strftime('%a, %d %b %Y %H:%M:%S',time.localtime())
newHTML = getHTML( remoteURL )
while( newHTML == False ):
print '...retrying...'
newHTML = getHTML( remoteURL )
refHTML = getHTML( localURL )
diffList = difflib.context_diff(
newHTML.splitlines(),
refHTML.splitlines() )
listOfDiffs = list(diffList)
numDiffs = len( listOfDiffs )
if numDiffs == 0:
print "No change..."
time.sleep(secondsDelay)
else:
print "Page changed: sending email"
emailChanges(refHTML, newHTML, listOfDiffs)
print "Updating reference page..."
writeHTML( localFile, newHTML )
def runDiff(a, b, outFile, acceptable_diffs=()):
fa = open(a, "rU")
fb = open(b, "rU")
all_diffs = list(difflib.context_diff(fa.readlines(), fb.readlines(), n=0))
print "\n".join(all_diffs)
if all_diffs:
all_diffs = parse_diffs(all_diffs)
for k in acceptable_diffs:
try:
del all_diffs[k]
except:
pass
if all_diffs:
l = []
for k, vi in all_diffs.iteritems():
t = []
for v in vi:
t.extend(v)
l.append((k, t))
l.sort()
nd = []
for k, lines in l:
nd.extend(lines)
s = "".join(nd)
fout = open(outFile, "a")
fout.write("Diffs in file %s:\n" % a)
fout.write(s)
if len(s) < 5000:
t = ":\n%s\n(diffs also stored at the end of %s)." % (s, outFile)
else:
t = ". See the end of the %s." % (outFile)
fout.close()
sys.exit("Script aborted because of failed example (%s).\nOutput differed from the expected output%s\n" % (last_test,t))
fa.close()
fb.close()
def calculate_diff(plugins,options):
od_plugins = collections.OrderedDict(sorted(plugins.items()))
diff = ""
for plugin_name, plugin_code in od_plugins.iteritems():
ks = list(plugin_code)
fromlines = StringIO.StringIO(plugin_code[ks[0]]).readlines()
tolines = StringIO.StringIO(plugin_code[ks[1]]).readlines()
now = time.time()
fromdate = now
todate = now
fromfile = ks[0] + ": " + plugin_name
tofile = ks[1] + ": " + plugin_name
if options.u:
diff += ''.join(difflib.unified_diff(fromlines, tolines, fromfile, tofile,
fromdate, todate, n=options.lines))
elif options.n:
diff += ''.join(difflib.ndiff(fromlines, tolines))
elif options.m:
diff += difflib.HtmlDiff().make_table(
fromlines,
tolines,
" " + fromfile + " ",
" " + tofile + " ",
context=options.c,
numlines=options.lines) + "<br /> <p />"
else:
diff += ''.join(difflib.context_diff(fromlines, tolines, fromfile, tofile,
fromdate, todate, n=options.lines))
return diff
def test_no_trailing_tab_on_empty_filedate(self):
args = ['one', 'two', 'Original', 'Current']
ud = difflib.unified_diff(*args, lineterm='')
self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"])
cd = difflib.context_diff(*args, lineterm='')
self.assertEqual(list(cd)[0:2], ["*** Original", "--- Current"])
def _diff(self, baselinedir, outputdir, dc=None):
if dc is None:
dc = filecmp.dircmp(baselinedir, outputdir, ['.svn'])
if dc.left_only:
self.fail("Files or subdirectories missing from output: "
+str(dc.left_only))
if dc.right_only:
self.fail("Files or subdirectories missing from baseline: "
+str(dc.right_only))
for name in dc.diff_files:
fromfile = join(dc.left, name)
tofile = join(dc.right, name)
with open(fromfile, 'r') as f_from:
fromlines = f_from.readlines()
with open(tofile, 'r') as f_to:
tolines = f_to.readlines()
diff = difflib.context_diff(fromlines, tolines,
fromfile+" (baseline)",
tofile+" (output)")
out = StringIO()
out.write("Output file does not match baseline:\n")
for line in diff:
out.write(line)
self.fail(out.getvalue())
for subdir in dc.subdirs:
self._diff(join(baselinedir, subdir),
join(outputdir, subdir),
dc=dc.subdirs[subdir])
shutil.rmtree(outputdir, ignore_errors=True)
def generate_diff(trial,ref,what):
diff_name = ".".join([trial[what],'diff'])
with open(trial[what], 'rb') as of:
ref[what].seek(0)
diff = difflib.context_diff([line for line in of], [line for line in ref[what]], fromfile=of.name, tofile='reference')
#sys.stderr.write("Comparing {} to reference {}\n".format(of.name,ref[what].name))
return (diff),
def get_diff(self, s1, s2): # pylint: disable=C0103
diff = difflib.context_diff(s1, s2)
rope = ""
for line in diff:
rope += line
return rope
def calculate_diff(self, config, new, old):
return "\n".join(context_diff(
a=old.splitlines(),
b=new.splitlines(),
fromfile=str(config.active_serial),
tofile=str(config.serial),
lineterm=''))
def show_diff(self, newdata, langfile):
with open(langfile) as f:
olddata = f.read().split('\n')
diff = difflib.context_diff(olddata, newdata.split('\n'),
fromfile='before', tofile='after')
for line in diff:
print(line.rstrip())
def checkRules(self, source, key, ufwi_ruleset):
# bart.xml => bart.iptables_ipv4
correct = source[:-3] + key
prefix = "%s:%s" % (basename(source)[:-4], key)
if self.options.regenerate:
if ufwi_ruleset:
print "%s: file regenerated" % prefix
with open(correct, "w") as fp:
for line in ufwi_ruleset:
print >>fp, line
elif exists(correct):
print "%s: remove file (no rule)" % prefix
unlink(correct)
return True
text1 = ufwi_ruleset
if exists(correct):
with open(correct) as fp:
text2 = [line.rstrip() for line in fp]
else:
text2 = tuple()
diff = list(context_diff(text1, text2))
if diff:
print "%s: FAILURE!" % prefix
for line in diff:
print line.rstrip()
return False
print "%s: ok" % prefix
return True
def test_app_iptables_rules(SystemInfo, Command, Sudo):
# Build a dict of variables to pass to jinja for iptables comparison
kwargs = dict(
mon_ip=os.environ.get('MON_IP', securedrop_test_vars.mon_ip),
default_interface=Command.check_output("ip r | head -n 1 | "
"awk '{ print $5 }'"),
tor_user_id=Command.check_output("id -u debian-tor"),
securedrop_user_id=Command.check_output("id -u www-data"),
ssh_group_gid=Command.check_output("getent group ssh | cut -d: -f3"),
dns_server=securedrop_test_vars.dns_server)
# Build iptables scrape cmd, purge comments + counters
iptables = "iptables-save | sed 's/ \[[0-9]*\:[0-9]*\]//g' | egrep -v '^#'"
environment = os.environ.get("CI_SD_ENV", "staging")
iptables_file = "{}/iptables-app-{}.j2".format(
os.path.dirname(os.path.abspath(__file__)),
environment)
# template out a local iptables jinja file
jinja_iptables = Template(open(iptables_file, 'r').read())
iptables_expected = jinja_iptables.render(**kwargs)
with Sudo():
# Actually run the iptables scrape command
iptables = Command.check_output(iptables)
# print diff comparison (only shows up in pytests if test fails or
# verbosity turned way up)
for iptablesdiff in difflib.context_diff(iptables_expected.split('\n'),
iptables.split('\n')):
print(iptablesdiff)
# Conduct the string comparison of the expected and actual iptables
# ruleset
assert iptables_expected == iptables
def generateContextDiff(self):
self.clear()
a = self.snapShot.text().splitlines()
b = self.editor.text().splitlines()
QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
lines = 0
for line in difflib.context_diff(a, b, "Original", "Current"):
if line.startswith('+'):
styleType = 1
elif line.startswith('-'):
styleType = 2
elif line.startswith('!'):
styleType = 3
elif line.startswith('*** '):
styleType = 4
else:
styleType = 0
self.appendText(line, styleType)
lines += 1
if lines == 0:
self.appendText('Nothing has changed.', 0)
QtGui.QApplication.restoreOverrideCursor()
def compareJava(self, oldPath, newPath, myClass, oldClass, newClass):
self.reporter.info("Custom class is being used:" + myClass)
self.reporter.info("Checking java for:" + oldClass + " " + newClass)
oldSource = ""
for c in self.locateClass(oldClass, oldPath):
oldSource = c
newSource = ""
for c in self.locateClass(newClass, newPath):
newSource = c
if oldSource == "":
self.reporter.error("Cannot find java for class:" + oldClass)
if newSource == "":
self.reporter.error("Cannot find java for class:" + newClass)
try:
if filecmp.cmp(oldSource, newSource):
self.reporter.info("Same java:" + oldSource + " " + newSource)
else:
mySource = ""
for c in self.locateClass(myClass, self.customPath):
mySource = c
if mySource == "":
self.reporter.error("Cannot find java for class:" + myClass)
self.reporter.actionRequired("Different java between versions", mySource, oldSource, newSource)
fromfile = oldSource
tofile = newSource
with open(fromfile) as fromf, open(tofile) as tof:
fromlines, tolines = list(fromf), list(tof)
diff = difflib.context_diff(fromlines, tolines, fromfile=oldSource, tofile=newSource)
sys.stdout.writelines(diff)
except OSError as ose:
print(ose)
def testSimplify(self):
inputFile = os.path.join(wikidataDir, "data/1000.json")
linesOut = open(os.path.join(wikidataDir, "test/1000.out")).read().split()
linesNew = subprocess.check_output([wdExtractPy, "-f", "-F", "-l", "en", "-s", "", "-t", "all", inputFile]).split()
diff = list(difflib.context_diff(linesOut, linesNew))
self.assertEqual(len(diff), 0, "Unexpected differences in output of wd-extract.py -f -F -l en -s '' -t all data/1000.json\n"
+ "".join(diff))
def save_if_diffs(config_lines, old_file):
"""
"""
do_write = False
if os.path.exists(old_file):
# read the existing
bc_config_orig = open(old_file, "r")
bc_config_lines_orig = bc_config_orig.readlines()
bc_config_orig.close()
for idx in range(len(bc_config_lines_orig)): bc_config_lines_orig[idx] = bc_config_lines_orig[idx].strip()
print "Checking if different from previous file [%s]" % old_file
different = False
for line in difflib.context_diff(bc_config_lines_orig, bc_config_lines, fromfile=old_file, tofile='BC_config_new.csv'):
print line
different = True
if different:
print
print
print "Differences found => "
print " Moving previous file [%s] to [%s.bak]" % (old_file, old_file)
shutil.move(old_file, "%s.bak" % old_file)
do_write = True
else:
# if it doesn't exist, write this for sure
do_write = True
if do_write:
print " Writing new file %s" % old_file
# write it to [project_folder]\BC_config_new.csv
bc_config_new = open(old_file, "wb")
for line in bc_config_lines: bc_config_new.write("%s\r\n" % line)
bc_config_new.close()
def do_unit_test(unit_test):
host_srctext = get_contents("./"+unit_test+"/host_"+unit_test+".c")
host_srctext = expand_pid_pattern(host_srctext)
host_expected_outputs = re.findall(EXPECT_PATTERN, host_srctext)
e_srctext = get_contents("./"+unit_test+"/e_"+unit_test+".c")
e_srctext = expand_pid_pattern(e_srctext)
e_expected_outputs = re.findall(EXPECT_PATTERN, e_srctext)
expected_output = "\n".join(e_expected_outputs)
if len(host_expected_outputs) != 0:
expected_output += "\n" + "\n".join(host_expected_outputs);
if not os.path.isfile("bin/host_"+unit_test):
print("WARNING: bin/host_"+unit_test+" not found! Did you run make?")
return False
actual_output = run_unit_test(unit_test)
succes = (actual_output.replace("\n","") == expected_output.replace("\n",""))
if not succes:
print('WARNING: Unit test "' + unit_test + '" failed with output:')
if actual_output == "TIMEOUT":
print("A timeout occurred")
else:
for line in difflib.context_diff(actual_output.split('\n'), expected_output.split('\n')):
print(line)
return succes
def generateContextDiff(self):
self.clear()
a = self.snapShot.text().splitlines()
b = self.editor.text().splitlines()
QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
lines = 0
for line in difflib.context_diff(a, b, "Deleted", "Added"):
if line.startswith('+ '):
styleType = 1
elif line.startswith('- '):
styleType = 2
elif line.startswith('! '):
styleType = 3
elif (line.startswith('*** ') or line.startswith('--- ')) and lines > 1:
styleType = 4
else:
styleType = 0
self.appendText(line, styleType)
lines += 1
if lines == 0:
self.appendText(self.trUtf8(
'Nothing has changed.'), self.cNormalFormat)
QtGui.QApplication.restoreOverrideCursor()
def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
verbose=False):
artifact_in_disk_str = json.dumps(
artifact_in_disk.__json__(), sort_keys=True,
indent=4, separators=(',', ': ')
)
artifact_in_db_str = json.dumps(
artifact_in_db.__json__(), sort_keys=True,
indent=4, separators=(',', ': ')
)
diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
artifact_in_disk_str.splitlines(),
fromfile='DB contents', tofile='Disk contents')
printed = False
for diff in diffs:
if not printed:
identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
identifier))
printed = True
print(diff)
if verbose:
print('\n\nOriginal contents:')
print('===================\n')
print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)
def get_diff(self, s1, s2):
diff = difflib.context_diff(s1, s2)
str = ''
for line in diff:
str += line
return str
def main():
usage = "usage: %prog [options] fromfile tofile"
parser = optparse.OptionParser(usage)
parser.add_option("-c",
action="store_true",
default=False,
help='Produce a context format diff (default)')
parser.add_option("-u",
action="store_true",
default=False,
help='Produce a unified format diff')
parser.add_option(
"-m",
action="store_true",
default=False,
help='Produce HTML side by side diff (can use -c and -l in conjunction)'
)
parser.add_option("-n",
action="store_true",
default=False,
help='Produce a ndiff format diff')
parser.add_option("-l",
"--lines",
type="int",
default=3,
help='Set number of context lines (default 3)')
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
sys.exit(1)
if len(args) != 2:
parser.error("need to specify both a fromfile and tofile")
n = options.lines
fromfile, tofile = args
fromdate = time.ctime(os.stat(fromfile).st_mtime)
todate = time.ctime(os.stat(tofile).st_mtime)
fromlines = open(fromfile, 'U').readlines()
tolines = open(tofile, 'U').readlines()
if options.u:
diff = difflib.unified_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
elif options.n:
diff = difflib.ndiff(fromlines, tolines)
elif options.m:
diff = difflib.HtmlDiff().make_file(fromlines,
tolines,
fromfile,
tofile,
context=options.c,
numlines=n)
else:
diff = difflib.context_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
sys.stdout.writelines(diff)
def handle(self, *args, **options):
import os, random, string
deploy_hash = ''.join(
random.choice(string.lowercase) for x in range(32))
host = None
hostname = 'unknown'
hosts = []
if len(args):
hostname = args[0]
for HostSettingsClass in HostSettings.__subclasses__():
name = HostSettingsClass.__name__.replace('HostSettings',
'').lower()
hosts.append(name)
if hostname == name:
host = HostSettingsClass
if host is None:
print 'host should be one of:', hosts
raise Exception("Unknown host %s" % (hostname, ))
print colors.red("DEPLOYING TO: %s" % hostname, bold=True)
print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True)
#if
env.host_string = host.HOST_STRING
print colors.red("TEST COMMAND:", bold=True)
run("ls")
#run("source virtualenv/bin/activate");
virtpath = host.PYTHON_PREFIX
if options['initial'] and not files.exists(
host.SRCROOT) and not files.exists(host.APPROOT):
print colors.red(
"initial=true and SRCROOT/APPROOT does not exist. will install now"
)
run('mkdir %s' % host.SRCROOT)
run('mkdir %s' % host.APPROOT)
run('mkdir %s/logs' % host.APPROOT)
with cd(host.SRCROOT):
run("git init")
run("git remote add origin %s" % host.GIT_REMOTE)
run("git pull origin master")
run(host.PYTHON_INSTALL)
run("%spip install -r requirements.txt" % virtpath)
sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % (
host.DATABASE['NAME'], host.DATABASE['NAME'],
host.DATABASE['USER'], host.DATABASE['PASSWORD'])
print colors.red(sql)
#print sql
run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root'))
run("%spython manage.py install" % virtpath)
#return
with cd(host.SRCROOT):
if host.CLASS == 'PROD' and not options['nobackup']:
print colors.red("PROD BACKUP:", bold=True)
run("%spython manage.py backup" % (virtpath, ))
print colors.red("REVERTING SETTINGS:", bold=True)
run("git checkout -- %s/settings.py" %
(django_settings.CADO_PROJECT))
print colors.red("UPDATING CODEBASE:", bold=True)
run("git pull origin %s" % options['branch'])
run("git checkout %s" % options['branch'])
print colors.red("INSTALLING REQUIREMENTS:", bold=True)
run("%spip install -q -r requirements.txt" % virtpath)
print colors.red("INSERTING HASH:", bold=True)
run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py"
% (deploy_hash, django_settings.CADO_PROJECT,
django_settings.CADO_PROJECT))
run("mv %s/settings.tmp.py %s/settings.py" %
(django_settings.CADO_PROJECT, django_settings.CADO_PROJECT))
#sed 's/foo/bar/' mydump.sql > fixeddump.sql
print colors.red("REGENERATIN CONFIG FILES:", bold=True)
run("%spython manage.py regenerate_config" % virtpath)
if django_settings.MULTISITE:
for site in django_settings.SITES:
#run("mkdir config/solr/%s" % site.SOLR_CORE_NAME)
run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml"
% (virtpath, site.CADO_PROJECT, site.SOLR_CORE_NAME))
else:
run("%spython manage.py build_solr_schema > config/solr/schema.xml"
% virtpath)
for name, getter, setter, combined in host.CONFIGS:
diff = False
current = run(getter, quiet=True, warn_only=True).splitlines()
current = [line + "\n" for line in current]
new = run("cat config/" + name, quiet=True).splitlines()
new = [line + "\n" for line in new]
if combined:
combined = []
hash = hashlib.md5(host.APPROOT).hexdigest()
start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash
end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash
if start_line not in current:
current.append(start_line)
if end_line not in current:
current.append(end_line)
in_chunk = False
for line in current:
if line == start_line:
in_chunk = True
combined.append(start_line)
combined = combined + new
combined.append(end_line)
if not in_chunk:
combined.append(line)
if line == end_line:
in_chunk = False
tf = tempfile.NamedTemporaryFile()
tfName = tf.name
tf.seek(0)
#print current, new, combined
for line in combined:
tf.write(line)
tf.flush()
put(tfName, 'config/%s.combined' % name)
new = combined
name = name + '.combined'
for line in context_diff(current,
new,
fromfile='CURRENT',
tofile='NEW'):
diff = True
choice = 'd'
if diff:
while choice == 'd':
choice = console.prompt(
'%s config file differs. [d]=show diff, [r]=replace, [i]=ignore'
% (name, ),
default='i',
validate='d|r|i')
if (choice == 'd'):
for line in context_diff(current,
new,
fromfile='CURRENT',
tofile='NEW'):
sys.stdout.write(line)
if (choice == 'r'):
run("cat config/" + name + " " + setter)
for site in django_settings.SITES:
print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT,
bold=True)
arguments = ''
if django_settings.MULTISITE:
arguments = site.CADO_PROJECT
#run("git submodule init")
#run("git submodule update")
run("%spython manage.py syncdb %s" % (virtpath, arguments))
run("%spython manage.py migrate %s" %
(virtpath, arguments)) #--traceback
run("%spython manage.py collectstatic %s --noinput" %
(virtpath, arguments))
run("%spython manage.py restyle_tinymce %s" %
(virtpath, arguments))
print colors.yellow("RESTARTING FASTCGI:", bold=True)
with settings(warn_only=True):
run("kill -9 `cat %s/%s.pid`" %
(host.APPROOT, site.CADO_PROJECT))
run("find . -name '*.pyc' -delete")
maxchildren = 3
if host.CLASS == 'TEST':
maxchildren = 1
run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid"
% (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT,
site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT))
#run("sleep 3")
print colors.yellow("NOT CLEARING CACHE:)", bold=True)
#run("%spython manage.py clear_cache" % (virtpath,))
#run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT))
run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT))
run("%spython manage.py warmup %s" % (virtpath, arguments))
print colors.green("DONE!", bold=True)
def bctest(testDir, testObj, buildenv):
"""Runs a single test, comparing output and RC to expected output and RC.
Raises an error if input can't be read, executable fails, or output/RC
are not as expected. Error is caught by bctester() and reported.
"""
# Get the exec names and arguments
execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"])
execargs = testObj['args']
execrun = [execprog] + execargs
# Read the input data (if there is any)
stdinCfg = None
inputData = None
if "input" in testObj:
filename = os.path.join(testDir, testObj["input"])
inputData = open(filename, encoding="utf8").read()
stdinCfg = subprocess.PIPE
# Read the expected output data (if there is any)
outputFn = None
outputData = None
outputType = None
if "output_cmp" in testObj:
outputFn = testObj['output_cmp']
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
try:
outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read()
except:
logging.error("Output file " + outputFn + " can not be opened")
raise
if not outputData:
logging.error("Output data missing for " + outputFn)
raise Exception
if not outputType:
logging.error("Output file %s does not have a file extension" % outputFn)
raise Exception
# Run the test
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
try:
outs = proc.communicate(input=inputData)
except OSError:
logging.error("OSError, Failed to execute " + execprog)
raise
if outputData:
data_mismatch, formatting_mismatch = False, False
# Parse command output and expected output
try:
a_parsed = parse_output(outs[0], outputType)
except Exception as e:
logging.error('Error parsing command output as %s: %s' % (outputType, e))
raise
try:
b_parsed = parse_output(outputData, outputType)
except Exception as e:
logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
raise
# Compare data
if a_parsed != b_parsed:
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
data_mismatch = True
# Compare formatting
if outs[0] != outputData:
error_message = "Output formatting mismatch for " + outputFn + ":\n"
error_message += "".join(difflib.context_diff(outputData.splitlines(True),
outs[0].splitlines(True),
fromfile=outputFn,
tofile="returned"))
logging.error(error_message)
formatting_mismatch = True
assert not data_mismatch and not formatting_mismatch
# Compare the return code to the expected return code
wantRC = 0
if "return_code" in testObj:
wantRC = testObj['return_code']
if proc.returncode != wantRC:
logging.error("Return code mismatch for " + outputFn)
raise Exception
if "error_txt" in testObj:
want_error = testObj["error_txt"]
# Compare error text
# TODO: ideally, we'd compare the strings exactly and also assert
# That stderr is empty if no errors are expected. However, vircles-tx
# emits DISPLAY errors when running as a windows application on
# linux through wine. Just assert that the expected error text appears
# somewhere in stderr.
if want_error not in outs[1]:
logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
raise Exception
def diff_dumps(initial_dump, migrated_dump):
""" Perform a diff of two database dumps """
file_initial = open(initial_dump)
file_migrated = open(migrated_dump)
return difflib.context_diff(file_initial.readlines(),
file_migrated.readlines())
def diff_bsps(bsp1, bsp2, full=False) -> str:
"""WARNING: full diffs can be incredibly large!"""
out = []
if bsp1.folder == bsp2.folder:
out.append(f"Comparing {bsp1} -> {bsp2}...")
else:
out.append(
f"Comparing {bsp1.folder}/{bsp1} -> {bsp2.folder}/{bsp2}...")
# NOTE: comparing lumps by index, same number of lumps expected
for lump1, lump2 in zip(bsp1.branch.LUMP, bsp2.branch.LUMP):
lump1 = lump1.name
lump2 = lump2.name
# diff headers
if lump1 not in bsp1.headers or lump2 not in bsp2.headers:
continue # lazy fix for rbsp externals
# TODO: note absent headers (not just for respawn.ExternalLumpManager!)
bsp1_header = bsp1.headers[lump1]
bsp2_header = bsp2.headers[lump2]
lump_name = lump1 if lump1 == lump2 else f"{lump1} -> {lump2}"
# NOTE: fourCC (decompressed size) vs length is not calculated
# -- in fact, no check to check opposing compressed state (one compressed, one uncompressed)
# -- however, LZMA compressed lump contents are always decompressed before comparison
header_diff = "".join([
"Y" if bsp1_header.offset == bsp2_header.offset else "N",
"Y" if bsp1_header.length == bsp2_header.length else "N",
"Y" if bsp1_header.version == bsp2_header.version else "N",
"Y" if bsp1_header.fourCC == bsp2_header.fourCC else "N"
])
# diff lump contents
try:
lump_1_contents = bsp1.lump_as_bytes(lump1)
lump_2_contents = bsp2.lump_as_bytes(lump2)
except Exception as exc:
out.append(f"{lump_name} {header_diff} ???? {exc}")
continue # skip this lump
lumps_match = bool(lump_1_contents == lump_2_contents)
contents_diff = "YES!" if lumps_match else "NOPE"
out.append(f"{lump_name} {header_diff} {contents_diff}")
# was a lump removed / added?
if (len(lump_1_contents) == 0
or len(lump_2_contents) == 0) and not lumps_match:
out.append(" ".join([
"+" if hasattr(bsp1, lump1) else "-",
f"{bsp1.filename}.{lump1}"
]))
out.append(" ".join([
"+" if hasattr(bsp2, lump2) else "-",
f"{bsp2.filename}.{lump2}"
]))
# detailed comparisons
elif full:
if not lumps_match:
# TODO: measure the scale of the differences
if lump1 in bsp1.branch.LUMP_CLASSES and lump2 in bsp2.branch.LUMP_CLASSES:
diff = difflib.unified_diff(
[lc.__repr__() for lc in getattr(bsp1, lump1)],
[lc.__repr__() for lc in getattr(bsp2, lump2)],
f"{bsp1.filename}.{lump1}", f"{bsp1.filename}.{lump1}")
out.extend(diff)
# SPECIAL_LUMP_CLASSES
elif all([ln == "ENTITIES" for ln in (lump1, lump2)]):
out.append(diff_entities(bsp1.ENTITIES, bsp2.ENTITIES))
elif all([ln == "PAKFILE" for ln in (lump1, lump2)]):
# NOTE: this will fail on nexon.cso2 bsps, as their pakfiles are unmapped
out.append(diff_pakfiles(bsp1.PAKFILE, bsp2.PAKFILE))
# TODO: GAME_LUMP diff model_names
else: # BASIC_LUMP_CLASSES / general raw bytes
# NOTE: xxd line numbers prevent accurately tracing insertions
diff = difflib.context_diff(
[*xxd(lump_1_contents, 32)
], # 32 bytes per line looks nice
[*xxd(lump_2_contents, 32)],
f"{bsp1.filename}.{lump1}",
f"{bsp2.filename}.{lump2}")
# TODO: run xxd without creating line numbers
# -- then, generate line numbers from diff & update diff with these line numbers
out.extend(diff)
else:
out.extend([str(bsp1_header), str(bsp2_header)])
return "\n".join(out)
_file = file.replace("\\", "/")
if _file not in files:
print("Missing: %s" % file)
todo = True
continue
tfile = os.path.join(tdir, file)
if os.path.isdir(file):
continue
targ_diff = 0 # no difference
if not cmp(tfile, file, shallow=False):
if ".gpr.py" in file:
with open(file) as sfil:
with open(tfile) as tfil:
diff = list(
difflib.context_diff(sfil.readlines(),
tfil.readlines(),
n=0))
for line in diff:
if 'gramps_target_version' in line:
print("gpr differs: %s %s" %
(addon, line), end='')
targ_diff = 1 # Potential problem
continue
if(line.startswith('---') or
line.startswith('***') or
'version' in line.lower()):
continue
targ_diff = 2 # just different
else:
targ_diff = 2 # just different
if targ_diff == 0:
def process_files(repair_files, check_all_files, display_diff,
colorize_output):
if check_all_files:
process = subprocess.run(['git', 'ls-files', '-z'],
stdout=subprocess.PIPE,
universal_newlines=True)
else:
process = subprocess.run([
'git', 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD',
'-z'
],
stdout=subprocess.PIPE,
universal_newlines=True)
wrong_license_files = []
wrong_extensions = [
'.map', '.txt', '.gitignore', '.clang-format', '.clang-format-ignore',
'.gitmodules', '.md', '.json', '.xml', '.log', '.csv', '.html', '.css',
'.pdf', '.rtf'
]
tmp_files = set()
for file in process.stdout.split('\0'):
authors = set()
wrong_ext = False
for we in wrong_extensions:
if file.endswith(we):
wrong_ext = True
if (not os.path.isfile(file)) or wrong_ext:
continue
process = subprocess.run([
'git', 'log', '--format=%H Author: %aN <%aE>', '--follow', '--',
file
],
stdout=subprocess.PIPE,
universal_newlines=True)
author_lines = process.stdout.split('\n')
for author_line in author_lines:
commit_hash = author_line.split(' ')[0]
author_line = author_line.lstrip(commit_hash + ' ')
if author_line.startswith('Author: '):
author = author_line.lstrip('Author: ')
process = subprocess.run([
'git', 'diff-tree', '--no-commit-id', '--name-only', '-r',
'--exit-code', commit_hash, '--', file
],
stdout=subprocess.PIPE,
universal_newlines=True)
# git returns non zero exit code if file was changed in this commit
if process.returncode != 0 and not '<not.committed.yet>' in author:
authors.add(author)
# find commit date
process = subprocess.run(['git', 'show', '-s', '--format=%ai', 'HEAD'],
stdout=subprocess.PIPE,
universal_newlines=True)
commit_year = process.stdout[:4]
process = subprocess.run(
['git', 'rev-list', '--max-parents=0', 'HEAD'],
stdout=subprocess.PIPE,
universal_newlines=True)
first_commit_id = process.stdout.split('\n')[0]
process = subprocess.run(
['git', 'show', '-s', '--format=%ai', first_commit_id],
stdout=subprocess.PIPE,
universal_newlines=True)
first_commit_year = process.stdout[:4]
if file.endswith('.c') or file.endswith('.h'):
license_type = 'c_style'
else:
license_type = 'hash'
license = generate_license(authors, commit_year, first_commit_year,
license_type)
with open(file) as f:
lines = f.readlines()
shebang = ''
if file.endswith('.sh') or file.endswith(
'.py') and lines[0].startswith('#!'): #files with shebang
shebang = lines[0]
lines = lines[1:]
license_len = len(license)
file_len = len(lines)
if license_len > file_len:
print("license length missmatch, want >= {}, got {}".format(
license_len, file_len))
COPYRIGHT_LINE_REG = r'^[#| ]*Copyright \\(c\\) [0-9]{4}.*All rights reserved\.'
comp = re.compile(COPYRIGHT_LINE_REG)
copyright_pos = 0
if license_type == 'c_style':
copyright_pos += 1
if comp.match(lines[copyright_pos]):
print("file without license: {}".format(file))
wrong_license_files.append(file)
else:
for it in range(0, license_len):
if lines[it] != license[it]:
wrong_license_files.append(file)
break
if file in wrong_license_files:
end_point = 0
if license_type == 'c_style':
if lines[0].startswith('/*'):
for it in range(0, len(lines)):
if lines[it].endswith('*/\n'):
end_point = it + 1
break
else:
for it in range(0, len(lines)):
if not (lines[it].isspace()
or lines[it].startswith('#')):
end_point = it
break
tmp_output = file + '.tmp'
with open(tmp_output, 'w') as fw:
if shebang != '':
fw.write(shebang)
for line in license:
fw.write(line)
for it in range(end_point, len(lines)):
fw.write(lines[it])
tmp_files.add(tmp_output)
diff = list(
difflib.context_diff(lines[:end_point],
license,
fromfile=file,
tofile=tmp_output))
# do not count files with more authors as file with wrong license
# it may have more authors because file could be e.g. renamed, merged etc
only_more_authors = True
CHANGED_LINE_RE = r'^[-+?!] .*'
AUTHOR_LINE_RE = r'- # Author: .*<.*>.*$'
comp_changed = re.compile(CHANGED_LINE_RE)
comp_author = re.compile(AUTHOR_LINE_RE)
diff_len = len(
diff) - 1 # last line contains diff lines numbers
for it in range(0, diff_len):
if comp_changed.match(diff[it]) and (
not comp_author.match(diff[it])):
# does not treat last empty line as error
if it == diff_len - 1 and (len(diff[it]) <= 2
or diff[it] == '- \n'):
break
only_more_authors = False
break
if only_more_authors:
wrong_license_files.remove(file)
else:
if display_diff:
print_diff(diff, colorize_output)
if repair_files:
os.rename(tmp_output, file)
got_error = False
for wlf in wrong_license_files:
print("File with wrong license: {}".format(wlf))
got_error = True
# when repairing files temporary files are moved to original ones
if not repair_files:
for tf in tmp_files:
os.remove(tf)
if got_error:
exit(1)
def query_interactive(src_fn, dest_fn, src_content, dest_content,
simulate, out_=sys.stdout):
def out(msg):
out_.write(msg)
out_.write('\n')
out_.flush()
global all_answer
from difflib import unified_diff, context_diff
u_diff = list(unified_diff(
dest_content.splitlines(),
src_content.splitlines(),
dest_fn, src_fn))
c_diff = list(context_diff(
dest_content.splitlines(),
src_content.splitlines(),
dest_fn, src_fn))
added = len([l for l in u_diff if l.startswith('+')
and not l.startswith('+++')])
removed = len([l for l in u_diff if l.startswith('-')
and not l.startswith('---')])
if added > removed:
msg = '; %i lines added' % (added-removed)
elif removed > added:
msg = '; %i lines removed' % (removed-added)
else:
msg = ''
out('Replace %i bytes with %i bytes (%i/%i lines changed%s)' % (
len(dest_content), len(src_content),
removed, len(dest_content.splitlines()), msg))
prompt = 'Overwrite %s [y/n/d/B/?] ' % dest_fn
while 1:
if all_answer is None:
response = input_(prompt).strip().lower()
else:
response = all_answer
if not response or response[0] == 'b':
import shutil
new_dest_fn = dest_fn + '.bak'
n = 0
while os.path.exists(new_dest_fn):
n += 1
new_dest_fn = dest_fn + '.bak' + str(n)
out('Backing up %s to %s' % (dest_fn, new_dest_fn))
if not simulate:
shutil.copyfile(dest_fn, new_dest_fn)
return True
elif response.startswith('all '):
rest = response[4:].strip()
if not rest or rest[0] not in ('y', 'n', 'b'):
out(query_usage)
continue
response = all_answer = rest[0]
if response[0] == 'y':
return True
elif response[0] == 'n':
return False
elif response == 'dc':
out('\n'.join(c_diff))
elif response[0] == 'd':
out('\n'.join(u_diff))
else:
out(query_usage)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-c',
action='store_true',
default=False,
help='Produce a context format diff (default)')
parser.add_argument('-u',
action='store_true',
default=False,
help='Produce a unified format diff')
parser.add_argument('-m',
action='store_true',
default=False,
help='Produce HTML side by side diff '
'(can use -c and -l in conjunction)')
parser.add_argument('-n',
action='store_true',
default=False,
help='Produce a ndiff format diff')
parser.add_argument('-l',
'--lines',
type=int,
default=3,
help='Set number of context lines (default 3)')
parser.add_argument('fromfile')
parser.add_argument('tofile')
options = parser.parse_args()
n = options.lines
fromfile = options.fromfile
tofile = options.tofile
fromdate = file_mtime(fromfile)
todate = file_mtime(tofile)
with open(fromfile) as ff:
fromlines = ff.readlines()
with open(tofile) as tf:
tolines = tf.readlines()
if options.u:
diff = difflib.unified_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
elif options.n:
diff = difflib.ndiff(fromlines, tolines)
elif options.m:
diff = difflib.HtmlDiff().make_file(fromlines,
tolines,
fromfile,
tofile,
context=options.c,
numlines=n)
else:
diff = difflib.context_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
sys.stdout.writelines(diff)
def output_difference(self, runstate=None, colored=True):
"""
Return a string describing the differences between the expected output
for a given example (`example`) and the actual output (`got`).
The `runstate` contains option flags used to compare `want` and `got`.
Notes:
This does not check if got matches want, it only outputs the raw
differences. Got/Want normalization may make the differences appear
more exagerated than they are.
"""
got = self.got
want = self.want
if runstate is None:
runstate = directive.RuntimeState()
# Don't normalize because it usually removes the newlines
runstate_ = runstate.to_dict()
# Don't normalize whitespaces in report for better visibility
runstate_['NORMALIZE_WHITESPACE'] = False
runstate_['IGNORE_WHITESPACE'] = False
got, want = normalize(got, want, runstate_)
# If <BLANKLINE>s are being used, then replace blank lines
# with <BLANKLINE> in the actual output string.
# if not runstate['DONT_ACCEPT_BLANKLINE']:
# got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)
got = utils.ensure_unicode(got)
# Check if we should use diff.
if self._do_a_fancy_diff(runstate):
# Split want & got into lines.
want_lines = want.splitlines(True)
got_lines = got.splitlines(True)
# Use difflib to find their differences.
if runstate['REPORT_UDIFF']:
diff = difflib.unified_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'unified diff with -expected +actual'
elif runstate['REPORT_CDIFF']:
diff = difflib.context_diff(want_lines, got_lines, n=2)
diff = list(diff)[2:] # strip the diff header
kind = 'context diff with expected followed by actual'
elif runstate['REPORT_NDIFF']:
# TODO: Is there a way to make Differ ignore whitespace if that
# runtime directive is specified?
engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
diff = list(engine.compare(want_lines, got_lines))
kind = 'ndiff with -expected +actual'
else:
raise ValueError('Invalid difflib option')
# Remove trailing whitespace on diff output.
diff = [line.rstrip() + '\n' for line in diff]
diff_text = ''.join(diff)
if colored:
diff_text = utils.highlight_code(diff_text, lexer_name='diff')
text = 'Differences (%s):\n' % kind + utils.indent(diff_text)
else:
# If we're not using diff, then simply list the expected
# output followed by the actual output.
if want and got:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n%s\nGot:\n%s' % (utils.indent(want),
utils.indent(got))
elif want:
if colored:
got = utils.color_text(got, 'red')
want = utils.color_text(want, 'red')
text = 'Expected:\n%s\nGot nothing\n' % utils.indent(want)
elif got: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot:\n%s' % utils.indent(got)
else: # nocover
raise AssertionError('impossible state')
text = 'Expected nothing\nGot nothing\n'
return text
#!/usr/bin/env python3 # -*- coding: UTF-8 -*- import difflib from difflib_data import * diff = difflib.context_diff(text1_lines, text2_lines, lineterm='') print '\n'.join(list(diff))
def erdiff(expect, result):
''' Expects large strings that have newlines in both expect and result '''
cd = context_diff(expect.splitlines(True), result.splitlines(True))
for line in cd:
sys.stdout.write(line)
assert expect == result
fp = FilePath(path)
children = [x for x in fp.walk() if x.splitext()[1] == ".py"]
with click.progressbar(children) as c:
for f in c:
inp = f.getContent().decode('utf8')
out = process_from_epydoc_to_sphinx(f, inp)
f.setContent(u'\n'.join(out).encode('utf8'))
if __name__ == "__main__":
run()
from difflib import context_diff
import sys
from transdocrify.tests import in_epydoc, out_rst
with open(in_epydoc.__file__.replace(".pyc", ".py"), 'r') as f:
inp = str(f.read())
with open(out_rst.__file__.replace(".pyc", ".py"), 'r') as f:
outp = f.readlines()
out3 = process_from_epydoc_to_sphinx(inp)
out2 = list(map(lambda x: x + "\n", out3))[:-1]
print(''.join(context_diff(outp, out2)))
import difflib
from pprint import pprint
import nltk
text1 = "text1.txt"
text2 = "text2.txt"
html = difflib.HtmlDiff()
diff = difflib.Differ()
f = open("result.html", "w+")
# use nltk to tokenize text into word list first before the comparison
content = html.make_file(nltk.word_tokenize(open(text1, "r").read()),
nltk.word_tokenize(open(text2, "r").read()),
context=True)
f.write(content)
f.close()
# use nltk to tokenize text into word list first before the comparison
pprint(
list(
difflib.context_diff(nltk.word_tokenize(open(text1, "r").read()),
nltk.word_tokenize(open(text2, "r").read()),
fromfile='text1.txt',
tofile='text2.txt')))
def difflib_contextDiff():
diff = difflib.context_diff(text1_lines, text2_lines, lineterm='')
print('\n'.join(diff))
def main():
args = sys.argv
if ("--help" in args) or (len(sys.argv) < 3):
if len(sys.argv) < 3:
print("ERROR: analyze_mutants requires at least two arguments\n")
print(
"USAGE: analyze_mutants <sourcefile> <cmd> [--mutantDir <dir>] [--fromFile <mutantfile>]"
)
print(
" <cmd> is command to execute to run tests; non-zero return indicates mutant killed"
)
print(
" --mutantDir: directory with all mutants; defaults to current directory"
)
print(
" --fromFile: file containing list of mutants to process; others ignored"
)
print(" --timeout <val>: change the timeout setting")
print(" --show: show mutants")
print(" --verbose: show mutants and output of analysis")
print(" --seed: random seed for shuffling of mutants")
print(" --noShuffle: do not randomize order of mutants")
print(
" --resume: use existing killed.txt and notkilled.txt, resume mutation analysis"
)
print(" --prefix: add a prefix to killed.txt and notkilled.txt")
sys.exit(0)
verbose = "--verbose" in sys.argv
if verbose:
args.remove("--verbose")
showM = "--show" in sys.argv
if showM:
args.remove("--show")
resume = "--resume" in sys.argv
if resume:
args.remove("--resume")
noShuffle = "--noShuffle" in sys.argv
if noShuffle:
args.remove("--noShuffle")
prefix = None
try:
prefixpos = args.index("--prefix")
except ValueError:
prefixpos = -1
if prefixpos != -1:
prefix = args[prefixpos + 1]
args.remove("--prefix")
args.remove(prefix)
fromFile = None
try:
filepos = args.index("--fromFile")
except ValueError:
filepos = -1
if filepos != -1:
fromFile = args[filepos + 1]
args.remove("--fromFile")
args.remove(fromFile)
seed = None
try:
seedpos = args.index("--seed")
except ValueError:
seedpos = -1
if seedpos != -1:
seed = args[seedpos + 1]
args.remove("--seed")
args.remove(seed)
seed = int(seed)
timeout = 30
try:
topos = args.index("--timeout")
except ValueError:
topos = -1
if topos != -1:
timeout = args[topos + 1]
args.remove("--timeout")
args.remove(timeout)
timeout = float(timeout)
onlyMutants = None
if fromFile is not None:
with open(fromFile, 'r') as file:
onlyMutants = file.read().split()
mdir = "."
try:
mdirpos = args.index("--mutantDir")
except ValueError:
mdirpos = -1
if mdirpos != -1:
mdir = args[mdirpos + 1]
args.remove("--mutantDir")
args.remove(mdir)
if mdir[-1] != "/":
mdir += "/"
src = args[1]
tstCmd = [args[2]]
ignore = []
if len(args) > 3:
with open(sys.argv[3]) as file:
for l in file:
ignore.append(l.split()[0])
srcBase = src.split("/")[-1]
srcEnd = "." + ((src.split(".")[-1]).split("/")[-1])
count = 0.0
killCount = 0.0
killFileName = "killed.txt"
notkillFileName = "notkilled.txt"
if prefix is not None:
killFileName = prefix + "." + killFileName
notkillFileName = prefix + "." + notkillFileName
print("ANALYZING", src)
print("COMMAND: **", tstCmd, "**")
allTheMutants = glob.glob(mdir +
srcBase.replace(srcEnd, ".mutant*" + srcEnd))
if onlyMutants is not None:
newMutants = []
for f1 in onlyMutants:
for f2 in allTheMutants:
if f2.split("/")[-1] == f1:
newMutants.append(f2)
allTheMutants = newMutants
if seed is not None:
random.seed(seed)
if not noShuffle:
random.shuffle(allTheMutants)
allStart = time.time()
if resume:
alreadyKilled = []
alreadyNotKilled = []
if not (os.path.exists(killFileName)
and os.path.exists(notkillFileName)):
print("ATTEMPTING TO RESUME, BUT NO PREVIOUS RESULTS FOUND")
else:
with open(killFileName, 'r') as killed:
with open(notkillFileName, 'r') as notkilled:
for line in killed:
if line == "\n":
continue
alreadyKilled.append(line[:-1])
count += 1
killCount += 1
for line in notkilled:
if line == "\n":
continue
alreadyNotKilled.append(line[:-1])
count += 1
print("RESUMING FROM EXISTING RUN, WITH", int(killCount),
"KILLED MUTANTS OUT OF", int(count))
with open(os.devnull, 'w') as dnull:
with open(killFileName, 'w') as killed:
with open(notkillFileName, 'w') as notkilled:
if resume:
for line in alreadyKilled:
killed.write(line + "\n")
killed.flush()
for line in alreadyNotKilled:
notkilled.write(line + "\n")
notkilled.flush()
for f in allTheMutants:
if resume:
if (f.split("/")[-1]
in alreadyKilled) or (f.split("/")[-1]
in alreadyNotKilled):
continue
if f in ignore:
print(f, "SKIPPED")
print("=" * 80)
print("#" + str(int(count) + 1) + ":", end=" ")
print("[" + str(round(time.time() - allStart, 2)) + "s",
end=" ")
print(
str(round(count / len(allTheMutants) * 100.0, 2)) +
"% DONE]")
if verbose or showM:
print("MUTANT:", f)
with open(src, 'r') as ff:
fromLines = ff.readlines()
with open(f, 'r') as tf:
toLines = tf.readlines()
diff = difflib.context_diff(fromLines, toLines,
"Original", "Mutant")
print(''.join(diff))
print()
sys.stdout.flush()
print("RUNNING", f + "...")
sys.stdout.flush()
try:
shutil.copy(src, src + ".um.backup")
shutil.copy(f, src)
if srcEnd == ".py":
py_compile.compile(src)
start = time.time()
if not verbose:
P = subprocess.Popen(tstCmd,
shell=True,
stderr=dnull,
stdout=dnull,
preexec_fn=os.setsid)
else:
P = subprocess.Popen(tstCmd,
shell=True,
preexec_fn=os.setsid)
try:
while P.poll() is None and (time.time() -
start) < timeout:
time.sleep(0.05)
finally:
if P.poll() is None:
print()
print(
"HAD TO TERMINATE ANALYSIS (TIMEOUT OR EXCEPTION)"
)
os.killpg(os.getpgid(P.pid), signal.SIGTERM)
# Avoid any weird race conditions from grabbing the return code
time.sleep(0.05)
r = P.returncode
runtime = time.time() - start
count += 1
if r == 0:
print(f, "NOT KILLED")
notkilled.write(f.split("/")[-1] + "\n")
notkilled.flush()
else:
killCount += 1
print(f, "KILLED IN", runtime, "(RETURN CODE",
str(r) + ")")
killed.write(f.split("/")[-1] + "\n")
killed.flush()
print(" RUNNING SCORE:", killCount / count)
sys.stdout.flush()
finally:
shutil.copy(src + ".um.backup", src)
os.remove(src + ".um.backup")
print("=" * 80)
print("MUTATION SCORE:", killCount / count)
def bctest(testDir, testObj, exeext):
"""Runs a single test, comparing output and RC to expected output and RC.
Raises an error if input can't be read, executable fails, or output/RC
are not as expected. Error is caught by bctester() and reported.
"""
# Get the exec names and arguments
execprog = testObj['exec'] + exeext
execargs = testObj['args']
execrun = [execprog] + execargs
# Read the input data (if there is any)
stdinCfg = None
inputData = None
if "input" in testObj:
filename = testDir + "/" + testObj['input']
inputData = open(filename).read()
stdinCfg = subprocess.PIPE
# Read the expected output data (if there is any)
outputFn = None
outputData = None
if "output_cmp" in testObj:
outputFn = testObj['output_cmp']
outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
try:
outputData = open(testDir + "/" + outputFn).read()
except:
logging.error("Output file " + outputFn + " can not be opened")
raise
if not outputData:
logging.error("Output data missing for " + outputFn)
raise Exception
# Run the test
proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
try:
outs = proc.communicate(input=inputData)
except OSError:
logging.error("OSError, Failed to execute " + execprog)
raise
if outputData:
data_mismatch, formatting_mismatch = False, False
# Parse command output and expected output
try:
a_parsed = parse_output(outs[0], outputType)
except Exception as e:
logging.error('Error parsing command output as %s: %s' % (outputType,e))
raise
try:
b_parsed = parse_output(outputData, outputType)
except Exception as e:
logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
raise
# Compare data
if a_parsed != b_parsed:
logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
data_mismatch = True
# Compare formatting
if outs[0] != outputData:
error_message = "Output formatting mismatch for " + outputFn + ":\n"
error_message += "".join(difflib.context_diff(outputData.splitlines(True),
outs[0].splitlines(True),
fromfile=outputFn,
tofile="returned"))
logging.error(error_message)
formatting_mismatch = True
assert not data_mismatch and not formatting_mismatch
# Compare the return code to the expected return code
wantRC = 0
if "return_code" in testObj:
wantRC = testObj['return_code']
if proc.returncode != wantRC:
logging.error("Return code mismatch for " + outputFn)
raise Exception
def get_test_files(maxn=10,
input_template="input%i.txt",
expected_template="expected%i.txt"):
for i in range(maxn):
input_filename = input_template % i
output_filename = expected_template % i
if os.path.exists(input_filename) and os.path.exists(output_filename):
yield i, input_filename, output_filename
if __name__ == "__main__":
test_files = list(get_test_files())
if test_files:
# If local files exist, run, and then diff actual vs expected.
for i, input_filename, expected_filename in test_files:
with open(input_filename) as inf, open(expected_filename) as exf:
result = list(main(inf))
expected_text = list(l.strip() for l in exf)
differences = list(
difflib.context_diff(result, expected_text, input_filename,
expected_filename))
if differences:
for diff in differences:
print(diff)
else:
print("Test Case %i passed" % i)
else:
# Or just run and report the output back to stdout
sys.stdout.write("\n".join(main(list(sys.stdin))))
except urllib2.URLError as e:
print '%s: URL Error: %s' % (domain, str(e.reason))
m = hashlib.md5()
m.update(newJson)
newHash = m.hexdigest()
oldDump = '%s.dump' % domain
if os.path.isfile(oldDump):
try:
oldJson = open(oldDump).read()
oldHash = hashlib.md5(oldJson).hexdigest()
except IOError as e:
print 'Cannot read %s.dump: %s (%s)' % (domain, e.strerror,
e.errno)
continue
if newHash != oldHash:
print "New credentials leaked for %s:" % domain
diff = difflib.context_diff(oldJson.splitlines(1),
newJson.splitlines(1),
fromfile='Old Hash',
tofile='New Hash')
print ''.join(diff)
try:
w = open(oldDump, 'w')
w.write(newJson)
w.close()
except IOError as e:
print "Cannot save %s.dump: %s (%s)" % (e.strerror, e.errno)
def diff(self, other):
return list(
line.rstrip()
for line in difflib.context_diff(self.lines, other.lines))[2:]
if l.upper() != l:
print("error:", l)
ATRs.append(l.strip())
# if l.startswith("\t"):
# ATRs.append([atr, l.strip()])
# else:
# atr = l.strip()
# pp.pprint(ATRs)
sorted_ATRs = list(ATRs)
sorted_ATRs.sort()
# pp.pprint(sorted_ATRs)
for l in difflib.context_diff(ATRs, sorted_ATRs):
print(l)
# compte le nombre de nouveau ATR
from subprocess import Popen, PIPE
p1 = Popen(["git", "diff"], stdout=PIPE)
p2 = Popen(["grep", "+3[B,F]"], stdin=p1.stdout, stdout=PIPE)
p1.stdout.close()
output = p2.communicate()[0]
output = output.decode("utf-8")
size = len(output.split("\n")) - 1
if size >= 10:
print()
def diff_ast(a1, a2):
for line in difflib.context_diff(repr(a1), repr(a2)):
sys.stdout.write(line)
def processFile(pathname):
print(pathname)
walktree('C:\\Users\Abridge\Desktop\PythonLearning', processFile)
### filecmp
import filecmp
filecmp.cmp('./dataFile.txt', './Day1/datafile.txt', shallow=False)
# returns the file names as a tuple of three lists - match, mismatch, error
result = filecmp.cmpfiles('.', './Day1/', ['datafile.txt'], shallow=False)
import difflib
d = difflib.context_diff(
open('./dataFile.txt').readlines(),
open('./Day1/datafile.txt').readlines())
for l in d:
print(l)
### dircmp
d = filecmp.dircmp('.', './Day1/')
d.report()
d.left
d.common
d.common_dirs
d.left_only
### zipfile and tarfile
import zipfile
z = zipfile.ZipFile('test.zip', 'w')
def dumb_diff(d1, d2):
json1 = json.dumps(d1, indent=2, sort_keys=True).splitlines()
json2 = json.dumps(d2, indent=2, sort_keys=True).splitlines()
print("\n".join(
difflib.context_diff(json1, json2, fromfile="old", tofile="new", n=2)))
def handle_new(self, feed):
self.handle_possible_rename(feed)
#check user whether rename.
cursor = self.database.cursor()
cursor.execute(
"""SELECT COUNT(*), `deleted`, `content`, `modified`, `edited_list` FROM `feeds` WHERE `id` = %s""",
(feed['id']))
info = cursor.fetchone()
num_previous = info[0]
if info[1] is not None:
was_deleted = (int(info[1]) == 1)
else:
was_deleted = False
if num_previous > 0: #feed exist.
if feed.has_key('message'):
if list(difflib.context_diff(info[2], feed.get('message'))):
msg = {"message": info[2], "updated_time": str(info[3])}
if "null" in str(info[-1]):
edited_list = list()
edited_list.append(msg)
else:
edited_list = anyjson.deserialize(info[-1])
edited_list.append(msg)
cursor.execute(
"""UPDATE `feeds` SET `user_name`=%s, `politician_id`=%s,`content`=%s, `modified`=%s, `edited_list`=%s WHERE id=%s""",
(feed['from']['name'], self.users[feed['from']['id']],
feed['message'], feed.get('updated_time').replace(
'+0000',
''), anyjson.serialize(edited_list), feed['id']))
log.notice(u"Updated {0}'s feed {0}",
feed.get('from', {}).get('name'),
feed.get('id'))
else:
log.info(u"{0}'s feed hasn't message key.",
feed['from']['name'])
else:
url = ""
if feed.has_key('actions'):
url = feed.get('actions')[0].get('link')
if self.images:
log.notice("Queued feed {0} for entity archiving.",
feed['id'])
self.beanstalk.put(anyjson.serialize(feed))
else:
url = "is a activity."
cursor.execute(
"""INSERT INTO `feeds` (`id`, `user_name`, `politician_id`, `content`, `created`, `modified`, `feed`, `feed_type`, url, edited_list)
VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
(feed.get('id'), feed.get('from', {}).get('name'),
self.users[feed['from']['id']], feed.get('message', ''),
feed.get('created_time').replace('+0000', ''),
feed.get('updated_time').replace('+0000', ''),
anyjson.serialize(feed), feed.get('type'), url, "[]"))
log.notice(u"Inserted {1}'s new feed {0}", feed.get('id'),
feed.get('from', {}).get('name'))
if was_deleted:
log.warn("feed deleted {0} before it came!", feed.get('id'))
#!/usr/bin/env python
import difflib, sys
with open("file1.txt", "r") as file1:
dataset1 = file1.readlines()
with open("file2.txt", "r") as file2:
dataset1 = file2.readlines()
for line in difflib.context_diff(file1, file2):
sys.stdout.write(line)
def release(session):
package_name = "packaging"
version_file = Path(f"{package_name}/__about__.py")
changelog_file = Path("CHANGELOG.rst")
try:
release_version = _get_version_from_arguments(session.posargs)
except ValueError as e:
session.error(f"Invalid arguments: {e}")
return
# Check state of working directory and git.
_check_working_directory_state(session)
_check_git_state(session, release_version)
# Prepare for release.
_changelog_update_unreleased_title(release_version, file=changelog_file)
session.run("git", "add", str(changelog_file), external=True)
_bump(session, version=release_version, file=version_file, kind="release")
# Tag the release commit.
# fmt: off
session.run(
"git",
"tag",
"-s",
release_version,
"-m",
f"Release {release_version}",
external=True,
)
# fmt: on
# Prepare for development.
_changelog_add_unreleased_title(file=changelog_file)
session.run("git", "add", str(changelog_file), external=True)
major, minor = map(int, release_version.split("."))
next_version = f"{major}.{minor + 1}.dev0"
_bump(session, version=next_version, file=version_file, kind="development")
# Checkout the git tag.
session.run("git", "checkout", "-q", release_version, external=True)
session.install("build", "twine")
# Build the distribution.
session.run("python", "-m", "build")
# Check what files are in dist/ for upload.
files = sorted(glob.glob("dist/*"))
expected = [
f"dist/{package_name}-{release_version}-py3-none-any.whl",
f"dist/{package_name}-{release_version}.tar.gz",
]
if files != expected:
diff_generator = difflib.context_diff(expected,
files,
fromfile="expected",
tofile="got",
lineterm="")
diff = "\n".join(diff_generator)
session.error(f"Got the wrong files:\n{diff}")
# Get back out into main.
session.run("git", "checkout", "-q", "main", external=True)
# Check and upload distribution files.
session.run("twine", "check", *files)
# Push the commits and tag.
# NOTE: The following fails if pushing to the branch is not allowed. This can
# happen on GitHub, if the main branch is protected, there are required
# CI checks and "Include administrators" is enabled on the protection.
session.run("git",
"push",
"upstream",
"main",
release_version,
external=True)
# Upload the distribution.
session.run("twine", "upload", *files)
# Open up the GitHub release page.
webbrowser.open("https://github.com/pypa/packaging/releases")
def run_regression(proof, dump, wrapper, cvc4_binary, benchmark_path, timeout):
"""Determines the expected output for a benchmark, runs CVC4 on it and then
checks whether the output corresponds to the expected output. Optionally
uses a wrapper `wrapper`, tests proof generation (if proof is true), or
dumps a benchmark and uses that as the input (if dump is true)."""
if not os.access(cvc4_binary, os.X_OK):
sys.exit(
'"{}" does not exist or is not executable'.format(cvc4_binary))
if not os.path.isfile(benchmark_path):
sys.exit('"{}" does not exist or is not a file'.format(benchmark_path))
basic_command_line_args = []
benchmark_basename = os.path.basename(benchmark_path)
benchmark_filename, benchmark_ext = os.path.splitext(benchmark_basename)
benchmark_dir = os.path.dirname(benchmark_path)
comment_char = '%'
status_regex = None
status_to_output = lambda s: s
if benchmark_ext == '.smt':
status_regex = r':status\s*(sat|unsat)'
comment_char = ';'
elif benchmark_ext == '.smt2':
status_regex = r'set-info\s*:status\s*(sat|unsat)'
comment_char = ';'
elif benchmark_ext == '.cvc':
pass
elif benchmark_ext == '.p':
basic_command_line_args.append('--finite-model-find')
status_regex = r'% Status\s*:\s*(Theorem|Unsatisfiable|CounterSatisfiable|Satisfiable)'
status_to_output = lambda s: '% SZS status {} for {}'.format(
s, benchmark_filename)
elif benchmark_ext == '.sy':
comment_char = ';'
# Do not use proofs/unsat-cores with .sy files
proof = False
else:
sys.exit('"{}" must be *.cvc or *.smt or *.smt2 or *.p or *.sy'.format(
benchmark_basename))
# If there is an ".expect" file for the benchmark, read the metadata
# from there, otherwise from the benchmark file.
metadata_filename = benchmark_path + '.expect'
if os.path.isfile(metadata_filename):
comment_char = '%'
else:
metadata_filename = benchmark_path
metadata_lines = None
with open(metadata_filename, 'r') as metadata_file:
metadata_lines = metadata_file.readlines()
benchmark_content = None
if metadata_filename == benchmark_path:
benchmark_content = ''.join(metadata_lines)
else:
with open(benchmark_path, 'r') as benchmark_file:
benchmark_content = benchmark_file.read()
# Extract the metadata for the benchmark.
scrubber = None
error_scrubber = None
expected_output = ''
expected_error = ''
expected_exit_status = None
command_line = ''
for line in metadata_lines:
# Skip lines that do not start with "%"
if line[0] != comment_char:
continue
line = line[2:]
if line.startswith(SCRUBBER):
scrubber = line[len(SCRUBBER):]
elif line.startswith(ERROR_SCRUBBER):
error_scrubber = line[len(ERROR_SCRUBBER):]
elif line.startswith(EXPECT):
expected_output += line[len(EXPECT):]
elif line.startswith(EXPECT_ERROR):
expected_error += line[len(EXPECT_ERROR):]
elif line.startswith(EXIT):
expected_exit_status = int(line[len(EXIT):])
elif line.startswith(COMMAND_LINE):
command_line += line[len(COMMAND_LINE):]
expected_output = expected_output.strip()
expected_error = expected_error.strip()
# Expected output/expected error has not been defined in the metadata for
# the benchmark. Try to extract the information from the benchmark itself.
if expected_output == '' and expected_error == '':
match = None
if status_regex:
match = re.search(status_regex, benchmark_content)
if match:
expected_output = status_to_output(match.group(1))
elif expected_exit_status is None:
# If there is no expected output/error and the exit status has not
# been set explicitly, the benchmark is invalid.
sys.exit('Cannot determine status of "{}"'.format(benchmark_path))
if not proof and ('(get-unsat-core)' in benchmark_content
or '(get-unsat-assumptions)' in benchmark_content):
print(
'1..0 # Skipped: unsat cores not supported without proof support')
return
if expected_exit_status is None:
expected_exit_status = 0
if 'CVC4_REGRESSION_ARGS' in os.environ:
basic_command_line_args += shlex.split(
os.environ['CVC4_REGRESSION_ARGS'])
basic_command_line_args += shlex.split(command_line)
command_line_args_configs = [basic_command_line_args]
extra_command_line_args = []
if benchmark_ext == '.sy' and \
'--no-check-synth-sol' not in basic_command_line_args and \
'--check-synth-sol' not in basic_command_line_args:
extra_command_line_args = ['--check-synth-sol']
if re.search(r'^(sat|invalid|unknown)$', expected_output) and \
'--no-check-models' not in basic_command_line_args:
extra_command_line_args = ['--check-models']
if proof and re.search(r'^(unsat|valid)$', expected_output):
if '--no-check-proofs' not in basic_command_line_args and \
'--incremental' not in basic_command_line_args and \
'--unconstrained-simp' not in basic_command_line_args and \
not cvc4_binary.endswith('pcvc4'):
extra_command_line_args = [
'--check-proofs', '--no-bv-eq', '--no-bv-ineq',
'--no-bv-algebraic'
]
if '--no-check-unsat-cores' not in basic_command_line_args and \
'--incremental' not in basic_command_line_args and \
'--unconstrained-simp' not in basic_command_line_args and \
not cvc4_binary.endswith('pcvc4'):
extra_command_line_args += ['--check-unsat-cores']
if extra_command_line_args:
command_line_args_configs.append(basic_command_line_args +
extra_command_line_args)
# Run CVC4 on the benchmark with the different option sets and check
# whether the exit status, stdout output, stderr output are as expected.
print('1..{}'.format(len(command_line_args_configs)))
print('# Starting')
for command_line_args in command_line_args_configs:
output, error, exit_status = run_benchmark(dump, wrapper, scrubber,
error_scrubber, cvc4_binary,
command_line_args,
benchmark_dir,
benchmark_basename, timeout)
if output != expected_output:
print(
'not ok - Differences between expected and actual output on stdout - Flags: {}'
.format(command_line_args))
for line in difflib.context_diff(output.splitlines(),
expected_output.splitlines()):
print(line)
elif error != expected_error:
print(
'not ok - Differences between expected and actual output on stderr - Flags: {}'
.format(command_line_args))
for line in difflib.context_diff(error.splitlines(),
expected_error.splitlines()):
print(line)
elif expected_exit_status != exit_status:
print(
'not ok - Expected exit status "{}" but got "{}" - Flags: {}'.
format(expected_exit_status, exit_status, command_line_args))
else:
print('ok - Flags: {}'.format(command_line_args))
def diff_main():
""" Command line interface to difflib.py providing diffs in four formats:
* ndiff: lists every line and highlights interline changes.
* context: highlights clusters of changes in a before/after format.
* unified: highlights clusters of changes in an inline format.
* html: generates side by side comparison with change highlights.
"""
# Configure the option parser
usage = "usage: %prog [options] fromfile tofile"
parser = optparse.OptionParser(usage)
parser.add_option("-c",
action="store_true",
default=False,
help='Produce a context format diff (default)')
parser.add_option("-u",
action="store_true",
default=False,
help='Produce a unified format diff')
hlp = 'Produce HTML side by side diff (can use -c and -l in conjunction)'
parser.add_option("-m", action="store_true", default=False, help=hlp)
parser.add_option("-n",
action="store_true",
default=False,
help='Produce a ndiff format diff')
parser.add_option("-l",
"--lines",
type="int",
default=3,
help='Set number of context lines (default 3)')
(options, args) = parser.parse_args()
if len(args) == 0:
parser.print_help()
sys.exit(1)
if len(args) != 2:
parser.error("need to specify both a fromfile and tofile")
n = options.lines
fromfile, tofile = args # as specified in the usage string
# we're passing these as arguments to the diff function
fromdate = time.ctime(os.stat(fromfile).st_mtime)
todate = time.ctime(os.stat(tofile).st_mtime)
fromlines = open(fromfile, 'U').readlines()
tolines = open(tofile, 'U').readlines()
if options.u:
diff = difflib.unified_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
elif options.n:
diff = difflib.ndiff(fromlines, tolines)
elif options.m:
diff = difflib.HtmlDiff().make_file(fromlines,
tolines,
fromfile,
tofile,
context=options.c,
numlines=n)
else:
diff = difflib.context_diff(fromlines,
tolines,
fromfile,
tofile,
fromdate,
todate,
n=n)
# we're using writelines because diff is a generator
sys.stdout.writelines(diff)
def create_source_map(nodes, code, filename, indices_in_code):
"""Creates a source map between an annotated AST and the code it compiles to.
Args:
nodes: Iterable[ast.AST, ...]
code: Text
filename: Optional[Text]
indices_in_code: Union[int, Iterable[int, ...]], the positions at which
nodes appear in code. The parser always returns a module when parsing
code. This argument indicates the position in that module's body at
which the corresponding of node should appear.
Returns:
Dict[CodeLocation, OriginInfo], mapping locations in code to locations
indicated by origin annotations in node.
"""
reparsed_nodes = parser.parse_str(code)
reparsed_nodes = [reparsed_nodes.body[i] for i in indices_in_code]
resolve(reparsed_nodes, code)
result = {}
try:
for before, after in ast_util.parallel_walk(nodes, reparsed_nodes):
# Note: generated code might not be mapped back to its origin.
# TODO(mdan): Generated code should always be mapped to something.
origin_info = anno.getanno(before, anno.Basic.ORIGIN, default=None)
final_info = anno.getanno(after, anno.Basic.ORIGIN, default=None)
if origin_info is None or final_info is None:
continue
line_loc = LineLocation(filename, final_info.loc.lineno)
existing_origin = result.get(line_loc)
if existing_origin is not None:
# Overlaps may exist because of child nodes, but almost never to
# different line locations. Exception make decorated functions, where
# both lines are mapped to the same line in the AST.
# Line overlaps: keep bottom node.
if existing_origin.loc.line_loc == origin_info.loc.line_loc:
if existing_origin.loc.lineno >= origin_info.loc.lineno:
continue
# In case of overlaps, keep the leftmost node.
if existing_origin.loc.col_offset <= origin_info.loc.col_offset:
continue
result[line_loc] = origin_info
except ValueError:
if logging.has_verbosity(3):
for n, rn in zip(nodes, reparsed_nodes):
nodes_str = pretty_printer.fmt(n, color=False, noanno=True)
reparsed_nodes_str = pretty_printer.fmt(rn,
color=False,
noanno=True)
diff = difflib.context_diff(nodes_str.split('\n'),
reparsed_nodes_str.split('\n'),
fromfile='Original nodes',
tofile='Reparsed nodes',
n=7)
diff = '\n'.join(diff)
logging.log(3, 'AST seems to lack integrity. Diff:\n%s', diff)
raise
return result
