=> http://www.pythonchallenge.com/pc/return/brightness.html
maybe consider deltas.gz
"""
import gzip, difflib
f = gzip.open("assets/deltas.gz")
left, right = [], []
for line in f:
left.append(line[:53].decode() + '\n')
right.append(line[56:].decode())
diff = difflib.Differ().compare(left, right)
f = open("assets/diff.png", "wb")
f1 = open("assets/diff1.png", "wb")
f2 = open("assets/diff2.png", "wb")
for line in diff:
print(line)
bs = bytes([int(o, 16) for o in line[2:].strip().split(" ") if o])
if line[0] == '+':
f1.write(bs)
elif line[0] == '-':
f2.write(bs)
else:
f.write(bs)
import difflib
from difflib_data import *
if __name__ == '__main__':
d = difflib.Differ()
diff = d.compare(text1_lines, text2_lines)
print '\n'.join(diff)
print "-------------------"
diff = difflib.ndiff(text1_lines, text2_lines)
print '\n'.join(list(diff))
print "-------------------"
diff = difflib.unified_diff(text1_lines, text2_lines, lineterm='')
print '\n'.join(list(diff))
print "-------------------"
d = difflib.HtmlDiff()
print d.make_table(text1_lines, text2_lines)
def diff(first, second):
"""Human readable differ."""
return '\n'.join(
list(difflib.Differ().compare(first.splitlines(),
second.splitlines())))
class cTest:
NOTFOUND = "file not found"
DONOTMATCH = "content does not match"
# cTest::cTest(string name, string tdir) {
def __init__(self, name, tdir):
global settings, TRUE_STRINGS, RESAVAIL, EXPECTDIR, PERFDIR, TEST_LIST, PERF_BASE
self.name = name
self.tdir = tdir
self.scm = settings["scm"]
if settings.has_key("skip-tests"): self.skip = True
else: self.skip = False
self.cfg = ConfigParser.ConfigParser(settings)
self.cfg.read([os.path.join(tdir, TEST_LIST)])
expectdir = os.path.join(tdir, EXPECTDIR)
if os.path.exists(expectdir) and os.path.isdir(expectdir):
self.has_expected = True
else:
self.has_expected = False
perfdir = os.path.join(tdir, PERFDIR)
if os.path.exists(perfdir) and os.path.isdir(
perfdir) and os.path.isfile(os.path.join(perfdir, PERF_BASE)):
self.has_perf_base = True
else:
self.has_perf_base = False
if self.has_perf_base and settings.has_key("_reset_perf_base"):
try:
rev = self.scm.getVersionString(self.tdir)
oname = "perf-%s-reset-%s" % (
time.strftime("%Y-%m-%d-%H.%M.%S"), rev)
shutil.move(os.path.join(perfdir, PERF_BASE),
os.path.join(perfdir, oname))
print "%s : performance baseline reset" % name
except (IOError, OSError, shutil.Error):
pass
# Load the App for the test and check that it exists
try:
self.app = self.cfg.get('main', 'app', False, settings)
except:
self.app = settings['default_app']
self.app = os.path.abspath(self.app)
if not os.path.exists(self.app):
print "Error: Application (%s) not found" % self.app
sys.exit(-1)
if not os.path.isfile(self.app):
print "Error: Application (%s) is not a file" % self.app
sys.exit(-1)
self.args = self.getConfig("main", "args", "")
if self.getConfig("consistency", "enabled", "yes") in TRUE_STRINGS:
self.consistency_enabled = True
else:
self.consistency_enabled = False
if self.getConfig("performance", "enabled",
"no") in TRUE_STRINGS and RESAVAIL:
self.performance_enabled = True
else:
self.performance_enabled = False
self.success = True
self.result = "passed"
self.disabled = False
self.exitcode = 0
self.errors = []
self.psuccess = True
self.pdisabled = False
self.presult = "passed"
# } // End of cTest::cTest()
# string cTest::getConfig(string sect, string opt, string default)
def getConfig(self, sect, opt, default):
global settings
try:
return self.cfg.get(sect, opt, False, settings)
except:
return default
# } // End of cTest::getConfig()
def getName(self):
return self.name
# bool cTest::isConsistencyTest() {
def isConsistencyTest(self):
return self.consistency_enabled
# } // End of isConsistencyTest()
# bool cTest::wasConsistencySkipped() {
def wasConsistencySkipped(self):
return self.disabled
# } // End of wasConsistencySkipped()
# bool cTest::isPerformanceTest() {
def isPerformanceTest(self):
return self.performance_enabled
# } // End of isPerformanceTest()
# bool cTest::wasPerformanceSkipped() {
def wasPerformanceSkipped(self):
return self.pdisabled
# } // End of wasPerformanceSkipped()
# void cTest::runConsistencyTest() {
def runConsistencyTest(self, dolongtest):
global settings, tmpdir, CONFIGDIR, EXPECTDIR
confdir = os.path.join(self.tdir, CONFIGDIR)
rundir = os.path.join(tmpdir, self.name)
expectdir = os.path.join(self.tdir, EXPECTDIR)
if not self.isConsistencyTest():
self.result = "skipped (not a consistency test)"
self.disabled = True
return
# If no expected results exist and in slave mode
if not self.has_expected and settings["mode"] == "slave":
self.result = "skipped (no expected results)"
self.disabled = True
return
if settings.has_key("_reset_expected"):
if not self.scm.removeDirectory(expectdir):
print "Error: unable to remove expected directory for reset"
self.success = False
return
self.has_expected = False
if self.has_expected and self.skip:
self.result = "skipped"
self.disabled = True
return
if self.getConfig("consistency", "long",
"no") in TRUE_STRINGS and not dolongtest:
self.result = "skipped (long)"
self.disabled = True
return
# Create test directory and populate with config
try:
shutil.copytree(confdir, rundir)
except (IOError, OSError), e:
print "Error: unable to create run dir"
print " -- root cause: %s" % e
self.success = False
return
self.scm.deleteMetadata(rundir)
# Run test app, capturing output and exitcode
p = subprocess.Popen("cd %s; %s %s" % (rundir, self.app, self.args),
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=True)
# Process output from app
# Note: must at least swallow app output so that the process output buffer does not fill and block execution
if settings.has_key("_verbose"): print
for line in p.stdout:
if settings.has_key("_verbose"):
sys.stdout.write("%s output: %s" % (self.name, line))
sys.stdout.flush()
self.exitcode = p.wait()
# Check exit code, depending on mode setting
nz = self.getConfig("main", "nonzeroexit", "disallow")
if (nz == "disallow"
and self.exitcode != 0) or (nz == "require"
and self.exitcode == 0):
self.success = False
try:
shutil.rmtree(rundir, True) # Clean up test directory
except (IOError, OSError):
pass
return
# Build dictionary of config structure
confstruct = {}
for root, dirs, files in os.walk(confdir):
self.scm.removeMetadataFromDirList(dirs)
for file in files:
path = os.path.abspath(os.path.join(root, file))
key = path[len(confdir) + 1:] # remove confdir from path
confstruct[key] = path
# If no expected results exist, defer processing of new expected results to results phase
if not self.has_expected:
self.confstruct = confstruct
return
# Build dicitonary of expected structure
expectstruct = {}
for root, dirs, files in os.walk(expectdir):
self.scm.removeMetadataFromDirList(dirs)
for ifile in EXPECTED_IGNORE:
if ifile in files:
files.remove(ifile)
for file in files:
path = os.path.abspath(os.path.join(root, file))
key = path[len(expectdir) + 1:] # remove confdir from path
expectstruct[key] = [path, False, cTest.NOTFOUND]
# Traverse results, comparing with expected
for root, dirs, files in os.walk(rundir):
for file in files:
path = os.path.abspath(os.path.join(root, file))
key = path[len(rundir) + 1:] # remove confdir from path
if expectstruct.has_key(key):
# string[] getStippedLines(string filename) {
def getStrippedLines(filename):
fp = open(filename, "U")
filelines = fp.readlines()
fp.close()
retlines = []
for line in filelines:
line = string.lstrip(line)
if len(line) != 0 and line[0] != "#":
retlines.append(line)
return retlines
# } // End of getStrippedLines()
match = True
if os.path.getsize(expectstruct[key]
[0]) < settings['diff-max-threshold']:
# Generate the diff between the two files, ignoring comments and blank lines
differ = difflib.Differ()
elines = getStrippedLines(path)
tlines = getStrippedLines(expectstruct[key][0])
diff = list(differ.compare(tlines, elines))
for line in diff:
if line[0] != ' ':
expectstruct[key][2] = cTest.DONOTMATCH
if settings.has_key("show-diff"):
expectstruct[key][2] += "\n\n"
for l in diff:
expectstruct[key][2] += l
expectstruct[key][2] += "\n"
match = False
break
else:
elines = getStrippedLines(path)
tlines = getStrippedLines(expectstruct[key][0])
for i in range(len(elines)):
if elines[i] != tlines[i]:
expectstruct[key][2] = cTest.DONOTMATCH
match = False
break
expectstruct[key][1] = match
for key in expectstruct.keys():
entry = expectstruct[key]
if not entry[1]:
self.errors.append("%s : %s" % (key, entry[2]))
self.success = False
# Clean up test directory
try:
shutil.rmtree(rundir, True)
except (IOError, OSError):
pass
def load(self, filename, raise_on_error=True):
"""
Load the parameters for this network from disk.
:param filename: Load the parameters of this network from a pickle file at the named path. If this name ends in
".gz" then the input will automatically be gunzipped; otherwise the input will be treated as a "raw" pickle.
:return: None
"""
if filename is None:
return
print 'Loading model parameters from {}'.format(filename)
opener = gzip.open if filename.lower().endswith('.gz') else open
handle = opener(filename, 'rb')
saved = cPickle.load(handle)
handle.close()
if saved['network'] != self.__str__():
print "Possibly not matching network configuration!"
differences = list(difflib.Differ().compare(
saved['network'].splitlines(),
self.__str__().splitlines()))
print "Differences are:"
print "\n".join(differences)
for layer in self.layers:
if '{}-values'.format(layer.layerNum) not in saved:
if raise_on_error:
raise ImportError("{} not in saved variables!".format(
'{}-values'.format(layer.layerNum)))
else:
print "WARNING: {} not in saved variables!".format(
'{}-values'.format(layer.layerNum))
continue
if (len(layer.params) + len(layer.params_nontrained)) != len(
saved['{}-values'.format(layer.layerNum)]):
print "Warning: Layer parameters for layer {} do not match. Trying to fit on shape!".format(
layer.layerNum)
n_assigned = 0
for p in layer.params + layer.params_nontrained:
for v in saved['{}-values'.format(layer.layerNum)]:
if p.get_value().shape == v.shape:
p.set_value(v)
n_assigned += 1
if n_assigned != (len(layer.params) +
len(layer.params_nontrained)):
if raise_on_error:
raise ImportError(
"Could not load all necessary variables!")
else:
print "WARNING: Could not load all necessary variables!"
else:
print "Found fitting parameters!"
else:
for p, v in zip(layer.params + layer.params_nontrained,
saved['{}-values'.format(layer.layerNum)]):
if p.get_value().shape == v.shape:
p.set_value(v)
else:
if raise_on_error:
raise ImportError(
"Skipping parameter for {}! Shape {} does not fit {}."
.format(p.name,
p.get_value().shape, v.shape))
else:
print "WARNING: Skipping parameter for {}! Shape {} does not fit {}.".format(
p.name,
p.get_value().shape, v.shape)
print 'Done'
def print_diff(s1, s2):
differ = difflib.Differ()
result = list(differ.compare(s1.splitlines(), s2.splitlines()))
print '\n'.join(result)
def content_diff(self):
d = difflib.Differ()
original = self.solution_file.content().replace("\r\n","\n").replace("\r","\n").splitlines(0)
anotated = self.content.replace("\r\n","\n").replace("\r","\n").splitlines(0)
result = list(d.compare(original, anotated))
return "\n".join(map(lambda l: l.strip("\n"), result))
def f_compare(file1, file2, pair_report, language_par='xx'):
"""Compares 2 RAW files and writes reports (1 per file pair + 1 with a
summary for all compared files).
Return True if no difference was found, False if a difference was found."""
compfilename = os.path.basename(file1.name)
text1 = file1.readlines()
text2 = file2.readlines()
outputfile = open(pair_report, 'wb')
filecounter1, filecounter2, linecounter, qmcounter, qmcounter2, diffcounter = 0, 0, 0, 0, 0, 0
sentence_diff, index_diff, attr_cert_diff, attr_neg_diff, attr_possent_diff, attr_negsent_diff = 0, 0, 0, 0, 0, 0
attr_time_diff, attr_freq_diff, attr_dur_diff, attr_meas_diff, attr_cert_diff, attr_ev_diff, undef_diff = 0, 0, 0, 0, 0, 0, 0
for line in text1:
if line[0] == 'D':
filecounter1 += 1
for line in text2:
if line[0] == 'D':
filecounter2 += 1
d = difflib.Differ()
result = list(d.compare(text1, text2))
result_diff = []
for elt in result:
# print(linecounter)
if elt[2] == '#':
continue # skip header of file
elif elt[0] == '-' or elt[
0] == '+': # - means the line is unique to input 1, + means the line is unique to input 2
write_ln(outputfile, elt)
result_diff.append(elt)
linecounter += 1
elif elt[0] == '?' and linecounter > 0: # characters not present in either input, skip header of file
if language_par != 'ja':
write_ln(outputfile, elt)
if '^' in elt:
qmcounter2 += 1
else:
qmcounter += 1
for elt in result_diff: # scroll through differences and count them per category
if elt[2] == 'S':
sentence_diff += 1
elif elt[2] == 'C' or elt[2] == 'R':
index_diff += 1
elif elt[2:4] == 'PR':
index_diff += 1
elif 'type=\"certainty\"' in elt:
attr_cert_diff += 1
elif 'type=\"negation\"' in elt:
attr_neg_diff += 1
elif 'type=\"sentpositive\"' in elt:
attr_possent_diff += 1
elif 'type=\"sentnegative\"' in elt:
attr_negsent_diff += 1
elif 'type=\"time\"' in elt:
attr_time_diff += 1
elif 'type=\"frequency\"' in elt:
attr_freq_diff += 1
elif 'type=\"duration\"' in elt:
attr_dur_diff += 1
elif 'type=\"measurement\"' in elt:
attr_meas_diff += 1
elif 'type=\"entity_vector\"' in elt:
attr_ev_diff += 1
else:
undef_diff += 1
# Write summary
if linecounter == 0:
print('No changes are found in ' + compfilename + '.')
write_ln(report, '\nNo changes are found in ' + compfilename + '.')
else:
write_ln(
report,
'\nSummary for the comparison of reference and new_output for ' +
compfilename + ':')
write_ln(outputfile, '\nSUMMARY:')
if filecounter1 != filecounter2:
print(
'\nWARNING: The RAW files contain output for a different number of input files!'
)
write_ln(
outputfile,
'WARNING: The RAW files contain output for a different number of input files!'
)
write_ln(
report,
'WARNING: The RAW files contain output for a different number of input files!'
)
diffcounter = int(linecounter - qmcounter - (qmcounter2 / 2))
print(
str(linecounter) +
' distinct line(s) reported, corresponding to approximately ' +
str(diffcounter) + ' difference(s).')
pair_report_path = str(outputfile)[25:-1]
pair_report_path = pair_report_path.replace('\\\\', '/')
print('See ' + pair_report_path + ' for details.')
write_ln(
outputfile,
str(linecounter) +
' lines don\'t have a(n exact) copy in the other input file, of which'
)
write_ln(
report,
str(linecounter) +
' lines don\'t have a(n exact) copy in the other input file, of which'
)
if sentence_diff > 0:
write_ln(
outputfile,
str(sentence_diff) +
' concern the sentence level (sentence boundaries or indexes),'
)
write_ln(
report,
str(sentence_diff) +
' concern the sentence level (sentence boundaries or indexes),'
)
if index_diff > 0:
write_ln(
outputfile,
str(index_diff) +
' concern indexes of Concepts, Relations and PathRelevant entities,'
)
write_ln(
report,
str(index_diff) +
' concern indexes of Concepts, Relations and PathRelevant entities,'
)
if attr_neg_diff > 0:
write_ln(outputfile,
str(attr_neg_diff) + ' concern negation attributes,')
write_ln(report,
str(attr_neg_diff) + ' concern negation attributes,')
if attr_possent_diff > 0:
write_ln(
outputfile,
str(attr_possent_diff) +
' concern positive sentiment attributes,')
write_ln(
report,
str(attr_possent_diff) +
' concern positive sentiment attributes,')
if attr_negsent_diff > 0:
write_ln(
outputfile,
str(attr_negsent_diff) +
' concern negative sentiment attributes,')
write_ln(
report,
str(attr_negsent_diff) +
' concern negative sentiment attributes,')
if attr_time_diff > 0:
write_ln(outputfile,
str(attr_time_diff) + ' concern time attributes,')
write_ln(report, str(attr_time_diff) + ' concern time attributes,')
if attr_freq_diff > 0:
write_ln(outputfile,
str(attr_freq_diff) + ' concern frequency attributes,')
write_ln(report,
str(attr_freq_diff) + ' concern frequency attributes,')
if attr_dur_diff > 0:
write_ln(outputfile,
str(attr_dur_diff) + ' concern duration attributes,')
write_ln(report,
str(attr_dur_diff) + ' concern duration attributes,')
if attr_meas_diff > 0:
write_ln(outputfile,
str(attr_meas_diff) + ' concern measurement attributes,')
write_ln(report,
str(attr_meas_diff) + ' concern measurement attributes,')
if attr_cert_diff > 0:
write_ln(outputfile,
str(attr_cert_diff) + ' concern certainty attributes,')
write_ln(report,
str(attr_cert_diff) + ' concern certainty attributes,')
if attr_ev_diff > 0:
write_ln(outputfile,
str(attr_ev_diff) + ' concern entity vectors,')
write_ln(report, str(attr_ev_diff) + ' concern entity vectors,')
if undef_diff > 0:
write_ln(outputfile,
str(undef_diff) + ' are not further specified.')
write_ln(report, str(undef_diff) + ' are not further specified.')
outputfile.close()
return linecounter == 0
}
r = requests.get(URL, proxies=proxy, stream=True)
with open("18.gz", "wb") as handle:
for data in r.iter_content():
handle.write(data)
data = gzip.open("18.gz")
d1, d2 = [], []
for line in data:
line = line.strip()
d1.append(line[:53].decode())
d2.append(line[56:].decode())
data.close()
compare = difflib.Differ().compare(d1, d2)
#f = open('18-0.png', 'wb')
#f1 = open('18-1.png', 'wb')
#f2 = open('18-2.png', 'wb')
#for line in list(compare):
# print line
# bs = bytearray([int(o, 16) for o in line[2:].split()])
# if line[0] == '+':
# f1.write(bs)
# elif line[0] == '-':
# f2.write(bs)
# else:
# f.write(bs)
def zip_inject_namespace(
zip_src,
namespace=None,
managed=None,
filename_token=None,
namespace_token=None,
namespaced_org=None,
logger=None,
):
""" Replaces %%%NAMESPACE%%% for all files and ___NAMESPACE___ in all
filenames in the zip with the either '' if no namespace is provided
or 'namespace__' if provided.
"""
# Handle namespace and filename tokens
if not filename_token:
filename_token = "___NAMESPACE___"
if not namespace_token:
namespace_token = "%%%NAMESPACE%%%"
if managed is True and namespace:
namespace_prefix = namespace + "__"
else:
namespace_prefix = ""
# Handle tokens %%%NAMESPACED_ORG%%% and ___NAMESPACED_ORG___
namespaced_org_token = "%%%NAMESPACED_ORG%%%"
namespaced_org_file_token = "___NAMESPACED_ORG___"
namespaced_org = namespace_prefix if namespaced_org else ""
# Handle token %%%NAMESPACE_OR_C%%% for lightning components
namespace_or_c_token = "%%%NAMESPACE_OR_C%%%"
namespace_or_c = namespace if managed and namespace else "c"
# Handle token %%%NAMESPACED_ORG_OR_C%%%
namespaced_org_or_c_token = "%%%NAMESPACED_ORG_OR_C%%%"
namespaced_org_or_c = namespace if namespaced_org else "c"
zip_dest = zipfile.ZipFile(io.BytesIO(), "w", zipfile.ZIP_DEFLATED)
differ = difflib.Differ()
for name in zip_src.namelist():
orig_name = str(name)
content = zip_src.read(name)
try:
content = content.decode("utf-8")
except UnicodeDecodeError:
# if we cannot decode the content, don't try and replace it.
pass
else:
prev_content = content
content = content.replace(namespace_token, namespace_prefix)
if logger and content != prev_content:
logger.info(
' {}: Replaced %%%NAMESPACE%%% with "{}"'.format(name, namespace)
)
prev_content = content
content = content.replace(namespace_or_c_token, namespace_or_c)
if logger and content != prev_content:
logger.info(
' {}: Replaced %%%NAMESPACE_OR_C%%% with "{}"'.format(
name, namespace_or_c
)
)
prev_content = content
content = content.replace(namespaced_org_token, namespaced_org)
if logger and content != prev_content:
logger.info(
' {}: Replaced %%%NAMESPACED_ORG%%% with "{}"'.format(
name, namespaced_org
)
)
prev_content = content
content = content.replace(namespaced_org_or_c_token, namespaced_org_or_c)
if logger and content != prev_content:
logger.info(
' {}: Replaced %%%NAMESPACED_ORG_OR_C%%% with "{}"'.format(
name, namespaced_org_or_c
)
)
content = content.encode("utf-8")
# Replace namespace token in file name
name = name.replace(filename_token, namespace_prefix)
name = name.replace(namespaced_org_file_token, namespaced_org)
if logger and name != orig_name:
logger.info(" {}: renamed to {}".format(orig_name, name))
zip_dest.writestr(name, content)
return zip_dest
def compare(dir1, dir2, file_pattern=None, size_within=0, left_extra=False, scrubs=None):
"""Compare files matching `file_pattern` in `dir1` and `dir2`.
`size_within` is a percentage delta for the file sizes. If non-zero,
then the file contents are not compared (since they are expected to
often be different), but the file sizes must be within this amount.
For example, size_within=10 means that the two files' sizes must be
within 10 percent of each other to compare equal.
`left_extra` true means the left directory can have extra files in it
without triggering an assertion.
`scrubs` is a list of pairs, regexes to find and literal strings to
replace them with to scrub the files of unimportant differences.
An assertion will be raised if the directories fail one of their
matches.
"""
assert os.path.exists(dir1), "Left directory missing: %s" % dir1
assert os.path.exists(dir2), "Right directory missing: %s" % dir2
dc = filecmp.dircmp(dir1, dir2)
diff_files = fnmatch_list(dc.diff_files, file_pattern)
left_only = fnmatch_list(dc.left_only, file_pattern)
right_only = fnmatch_list(dc.right_only, file_pattern)
show_diff = True
if size_within:
# The files were already compared, use the diff_files list as a
# guide for size comparison.
wrong_size = []
for f in diff_files:
with open(os.path.join(dir1, f), "rb") as fobj:
left = fobj.read()
with open(os.path.join(dir2, f), "rb") as fobj:
right = fobj.read()
size_l, size_r = len(left), len(right)
big, little = max(size_l, size_r), min(size_l, size_r)
if (big - little) / float(little) > size_within/100.0:
# print "%d %d" % (big, little)
# print "Left: ---\n%s\n-----\n%s" % (left, right)
wrong_size.append("%s (%s,%s)" % (f, size_l, size_r)) # pragma: only failure
if wrong_size:
print("File sizes differ between %s and %s: %s" % ( # pragma: only failure
dir1, dir2, ", ".join(wrong_size)
))
# We'll show the diff iff the files differed enough in size.
show_diff = bool(wrong_size)
if show_diff:
# filecmp only compares in binary mode, but we want text mode. So
# look through the list of different files, and compare them
# ourselves.
text_diff = []
for f in diff_files:
with open(os.path.join(dir1, f), READ_MODE) as fobj:
left = fobj.read()
with open(os.path.join(dir2, f), READ_MODE) as fobj:
right = fobj.read()
if scrubs:
left = scrub(left, scrubs)
right = scrub(right, scrubs)
if left != right: # pragma: only failure
text_diff.append(f)
left = left.splitlines()
right = right.splitlines()
print("\n".join(difflib.Differ().compare(left, right)))
assert not text_diff, "Files differ: %s" % text_diff
if not left_extra:
assert not left_only, "Files in %s only: %s" % (dir1, left_only)
assert not right_only, "Files in %s only: %s" % (dir2, right_only)
def _compare(text1, text2):
text1_split = text1.splitlines()
text2_split = text2.splitlines()
diff = difflib.Differ().compare(text1_split, text2_split)
return '\n'.join(diff)
def diff_check_revision(self):
create_revision_list = []
table_column_current = None
table_column_previous = None
code_current = mwparserfromhell.parse(self.current_revision_file[0],
skip_style_tags=True)
code_previous = mwparserfromhell.parse(self.previous_revision_file[0],
skip_style_tags=True)
try:
########### Current revision table data extraction
table1 = code_current.filter_tags(
matches=lambda node: node.tag == "table")
table_code_current = wtp.parse(str(table1[0])).tables[0]
table_data_current = table_code_current.data()
table_column_current = table_data_current[0]
########## previous revision table data extraction
table2 = code_previous.filter_tags(
matches=lambda node: node.tag == "table")
table_code_previous = wtp.parse(str(table2[0])).tables[0]
table_data_previous = table_code_previous.data()
table_column_previous = table_data_previous[0]
df_data = DataFrame(table_data_previous)
header = df_data.iloc[0]
new_column_list = header.tolist()
df_data = df_data[1:]
df_data.columns = header
except Exception as e:
print('Exception from table data: ', str(e))
if table_column_current and table_column_previous and len(
table_column_previous) == len(set(table_column_previous)):
self.table_count_with_error = self.table_count_with_error + 1
if len(table_column_current) == len(table_column_previous):
text1 = table_data_previous
text2 = table_data_current
if text1 and text2:
for index1, (txt1,
txt2) in enumerate(zip(text1,
text2)): #row parsing
if index1 == 0:
continue
d = difflib.Differ()
for index, (cell1, cell2) in enumerate(zip(
txt1, txt2)): # values of row parsing
create_revision_dict = {}
old_value = None
new_value = None
cell1 = remove_markup(str(cell1))
cell2 = remove_markup(str(cell2))
#print(cell1)
#print(cell2)
if cell1 and cell2:
diff1 = d.compare([''.join(cell1)], [cell2])
try:
if diff1:
for line in diff1:
#print(line)
#print('###############################################################################')
if not line.startswith(' '):
if line.startswith('-'):
old_value = line[1:]
if line.startswith('+'):
new_value = line[1:]
if old_value and new_value:
#table_column_current1=remove_markup(str(table_column_current))
txt1 = remove_markup(str(txt1))
old_value = remove_markup(
str(old_value))
new_value = remove_markup(
str(new_value))
column_name = new_column_list[
index]
column_name = str(column_name)
#print(column_name)
#print(type(column_name))
column_values = df_data[
column_name].tolist()
column_values = remove_markup(
str(column_values))
#value = html.unescape(value)
#new_value = re.sub("[\t\n ]+", " ", new_value, re.UNICODE)
#value = value.strip("\t\n ")
cleanr = re.compile('<.*?>')
all_column = list(df_data.columns)
#all_column=html.unescape(str(all_column))
#all_column=remove_markup(str(all_column))
all_column = re.sub(
cleanr, ' ', str(all_column))
all_column = remove_markup(
all_column)
column_name = re.sub(
cleanr, ' ', str(column_name))
column_name = remove_markup(
column_name)
if len(old_value) < 50 and len(
new_value) < 50:
create_revision_dict = {
"columns": all_column,
"domain": column_values,
"vicinity": txt1,
"errored_column":
column_name,
"old_value": old_value,
"new_value": new_value
}
create_revision_list.append(
create_revision_dict)
print('column: ', column_name,
'old cell: ', old_value,
'new_cell: ', new_value)
except Exception as e:
print('Exception from revised value: ',
str(e))
return create_revision_list
class ConfigFileTests(unittest.TestCase):
config_files = {
'old': fixpath('data/config_old.txt'),
'one': fixpath('data/config_1.txt'),
'two': fixpath('data/config_2.txt'),
'list': fixpath('data/config_list.txt'),
'list2': fixpath('data/config_list_2.txt'),
'error': fixpath('data/config_error_handler.txt')
}
settings = {
'old': {
u'datestamp': u'%Y-%m-%d %H:%M UTC',
u'generator': True,
u'no_random': True,
u'python_home': u'http://www.python.org',
u'source_link': True,
'stylesheet': None,
u'stylesheet_path': [u'stylesheets/pep.css'],
'template': fixpath(u'data/pep-html-template')
},
'one': {
u'datestamp': u'%Y-%m-%d %H:%M UTC',
u'generator': True,
u'no_random': True,
u'python_home': u'http://www.python.org',
u'raw_enabled': False,
'record_dependencies': utils.DependencyList(),
u'source_link': True,
'stylesheet': None,
u'stylesheet_path': [u'stylesheets/pep.css'],
u'tab_width': 8,
u'template': fixpath(u'data/pep-html-template'),
u'trim_footnote_reference_space': True,
},
'two': {
u'footnote_references': u'superscript',
u'generator': False,
'record_dependencies': utils.DependencyList(),
u'stylesheet': None,
u'stylesheet_path': [u'test.css'],
'trim_footnote_reference_space': None
},
'list': {
u'expose_internals': [u'a', u'b', u'c', u'd', u'e'],
u'strip_classes': [u'spam', u'pan', u'fun', u'parrot'],
u'strip_elements_with_classes':
[u'sugar', u'flour', u'milk', u'safran']
},
'list2': {
u'expose_internals': [u'a', u'b', u'c', u'd', u'e', u'f'],
u'strip_classes':
[u'spam', u'pan', u'fun', u'parrot', u'ham', u'eggs'],
u'strip_elements_with_classes':
[u'sugar', u'flour', u'milk', u'safran', u'eggs', u'salt']
},
'error': {
u'error_encoding': u'ascii',
u'error_encoding_error_handler': u'strict'
},
}
compare = difflib.Differ().compare
"""Comparison method shared by all tests."""
def setUp(self):
self.option_parser = frontend.OptionParser(components=(pep_html.Writer,
rst.Parser),
read_config_files=None)
def files_settings(self, *names):
settings = frontend.Values()
for name in names:
settings.update(
self.option_parser.get_config_file_settings(
self.config_files[name]), self.option_parser)
return settings.__dict__
def expected_settings(self, *names):
expected = {}
for name in names:
expected.update(self.settings[name])
return expected
def compare_output(self, result, expected):
"""`result` and `expected` should both be dicts."""
self.assertTrue('record_dependencies' in result)
if 'record_dependencies' not in expected:
# Delete it if we don't want to test it.
del result['record_dependencies']
result = pprint.pformat(result) + '\n'
expected = pprint.pformat(expected) + '\n'
try:
self.assertEqual(result, expected)
except AssertionError:
print >> sys.stderr, '\n%s\n' % (self, )
print >> sys.stderr, '-: expected\n+: result'
print >> sys.stderr, ''.join(
self.compare(expected.splitlines(1), result.splitlines(1)))
raise
def test_nofiles(self):
self.compare_output(self.files_settings(), self.expected_settings())
def test_old(self):
self.compare_output(self.files_settings('old'),
self.expected_settings('old'))
def test_one(self):
self.compare_output(self.files_settings('one'),
self.expected_settings('one'))
def test_multiple(self):
self.compare_output(self.files_settings('one', 'two'),
self.expected_settings('one', 'two'))
def test_old_and_new(self):
self.compare_output(self.files_settings('old', 'two'),
self.expected_settings('old', 'two'))
def test_list(self):
self.compare_output(self.files_settings('list'),
self.expected_settings('list'))
def test_list2(self):
self.compare_output(self.files_settings('list', 'list2'),
self.expected_settings('list2'))
def test_error_handler(self):
self.compare_output(self.files_settings('error'),
self.expected_settings('error'))
def compare_html_pages(valid_html, infected_html):
try:
if settings.print_diff == 1:
# Set info and prepare request
if settings.print_info == 1:
print Style.NORMAL + Fore.YELLOW + '[>]\n'
if settings.request_method == 1:
message = Style.NORMAL + Fore.YELLOW + '[<] Compare HTML response before ( ' + Fore.GREEN + settings.pre_url + Fore.YELLOW + ' ) \nand after ( ' + Fore.RED + settings.url + ', & ' + settings.inject_here + Fore.YELLOW + ' ) injection :'
else:
message = Style.NORMAL + Fore.YELLOW + '[<] Compare HTML response before ( ' + Fore.GREEN + settings.pre_url + Fore.YELLOW + ' ) \nand after ( ' + Fore.RED + settings.url + Fore.YELLOW + ' ) injection :'
parameter = settings.inject_here
verbosity.print_message(message, settings.print_info)
# Compare valid and malicious requests' responses
diff_lib = difflib.Differ()
diff = diff_lib.compare(list(valid_html.stripped_strings),
list(infected_html.stripped_strings))
comparison = list(diff)
counter = 0
differences_removed = []
differences_added = []
differences = []
for i in comparison:
first_char_comparison = comparison[counter][:1]
if first_char_comparison == "-":
splitted_comparison = comparison[counter].split("- ")
differences_removed.append(splitted_comparison[1])
differences.append(splitted_comparison[1])
elif first_char_comparison == "+":
splitted_comparison = comparison[counter].split("+ ")
differences_added.append(splitted_comparison[1])
differences.append(splitted_comparison[1])
counter += 1
if settings.print_diff not in []:
message = '[i] Removed content : \n' + Fore.WHITE + Style.DIM + '%s' % list(
differences_removed)
verbosity.print_message(message, settings.print_info)
message = '[i] Added Content : \n' + Fore.WHITE + Style.DIM + '%s' % list(
differences_added)
verbosity.print_message(message, settings.print_info)
# False negative error
# No changes on html page based on your payload
if not differences:
print Fore.RED + '[i] HTML content does not seem to change based on your payload.'
ask_blind = True
else:
ask_blind = False
if ask_blind == True:
blind = interfaces.change_technique(
settings.bl_prompt_message, settings.bl_options_message,
settings.bl_alter_tech_msg, settings.bl_current_tech_msg)
if blind == True:
# Re-initialize input with [INJECT_HERE]
settings.inject_here = settings.initial_inject_here
blind_technique.blind_injection()
sys.exit()
except Exception as e:
print(Fore.RED + "[!] ERROR: %s" % e)
verbosity.error_info(e)
def test(val,gold):
# get the attribute file context
val_content = open(val).read()
gf_content = open(gold).read()
if True:
val_content = val_content.replace("\r\n", "\n")
gf_content = gf_content.replace("\r\n", "\n")
# make seqerncer differ
seq = difflib.SequenceMatcher(None,val_content,gf_content,autojunk=False)
#seq = difflib.SequenceMatcher(None,gf_content,val_content)
#do we have a match
if seq.ratio() == 1.0:
#The says ratio everything matched
print("Match")
return
# if we are here we don't have a match at the moment. At this point we
# process difference to see if they
# match and special code we have and do replacements of values and diff
# again to see if we have a match
#get diffs
results = seq.get_opcodes()
newtext = ''
sub=False # true is we are doign a {} and have not had non-white space to replace
for tag, i1, i2, j1, j2 in results:
# technically we can see that we might have a real diff
# but we continue as this allow certain values to be replaced
# helping to make the
# finial diff string more readable
print("\n-----",tag)
print("gold:\n'{0}'".format(gf_content[j1:j2]))
print("val :\n'{0}'".format(val_content[i1:i2]))
print("---------------------------")
data = gf_content[j1:j2].strip()
if data.strip() == '{}' or (data == '' and sub==True):
sub=True
data='{}'
if tag != 'insert':
tag = "replace"
else:
sub=False
print("sub :'{0}'".format(sub))
if tag == "replace" :
tmp = _do_action_replace(data,val_content[i1:i2])
if tmp:
newtext+=tmp
continue
if tag == "insert" :
tmp = _do_action_add(data,val_content[i1:i2])
if tmp is not None:
newtext+=tmp
continue
newtext+=gf_content[j1:j2]
print("new text:\n",newtext)
#reset the sequence test
seq.set_seq2(newtext)
if seq.ratio() == 1.0:
#The says ratio everything matched
print("WE HAVE A MATCH !!!!!!!!!!!!!!")
return
# this makes a nice string value..
diff = difflib.Differ()
tmp_result = "\n".join(diff.compare(val_content.splitlines(),
newtext.splitlines()))
tmp_result = "\n".join(diff.compare(newtext.splitlines(),
val_content.splitlines()))
print("File differences\nGold File : {0}\nData File : {1}\n{2}".format(gold,
val,
tmp_result))
def __init__(self, path="."):
self.repo_dir = path
self.differ = difflib.Differ()
def setUp(self):
self.differ = difflib.Differ()
def vpt2(name, **kwargs):
"""Perform vibrational second-order perturbation computation through
Cfour to get anharmonic frequencies. This version uses c4 for the disp
and pt2 but gets gradients from p4.
:type c4full: :ref:`boolean <op_py_boolean>`
:param c4full: ``'on'`` || |dl| ``'off'`` |dr|
Indicates whether when *name* indicates a Cfour method and *mode*
indicates a sow/reap approach, sown files are direct ZMAT files
and FJOBARC files are expected to reap, so that Cfour only, not
Cfour-through-Psi4, is needed for distributed jobs.
.. caution:: Some features are not yet implemented. Buy a developer a coffee.
- Presently uses all gradients. Could mix in analytic 2nd-derivs.
- Collect resutls.
- Manage scratch / subdir better.
- Untangle CCSD(T) vs CCSD[T] and FJOBARC issue
- Allow CFOUR_BASIS
- Consider forcing some tighter convcrit, c4 and p4
- sow/reap
- mixed ang/bohr signals
- error by converting to ang in psi?
- Expand CURRENT DIPOLE XYZ beyond SCF
- Remember additional FJOBARC record TOTENER2 if EXCITE .ne. NONE
- S/R P4grad
- S/R C4grad
- C P4grad
- C C4grad
- switch C --> S/R with recovery using shelf
- pure C mode where only need P4 for wrapper
"""
lowername = name.lower()
kwargs = p4util.kwargs_lower(kwargs)
optstash = p4util.OptionsState(['BASIS'])
# Option mode of operation- whether vpt2 run in one job or files farmed out
if not ('vpt2_mode' in kwargs):
if ('mode' in kwargs):
kwargs['vpt2_mode'] = kwargs['mode']
del kwargs['mode']
else:
kwargs['vpt2_mode'] = 'continuous'
# Switches for route through code- S/R or continuous & Psi4 or Cfour gradients
isSowReap = True if kwargs['vpt2_mode'].lower() == 'sowreap' else False
isC4notP4 = bool(re.match('cfour', lowername)) or bool(
re.match('c4-', lowername))
isC4fully = True if ('c4full' in kwargs
and yes.match(str(kwargs['c4full'])) and isC4notP4
and isSowReap) else False
# Save submission directory and basis set
current_directory = os.getcwd()
user_basis = psi4.get_global_option('BASIS')
# Open data persistence shelf- vital for sowreap, checkpoint for continuouw
shelf = shelve.open(current_directory + '/' +
os.path.splitext(psi4.outfile_name())[0] + '.shelf',
writeback=True)
# Cfour keywords to request vpt2 analysis through findif gradients
psi4.set_local_option('CFOUR', 'CFOUR_VIBRATION', 'FINDIF')
psi4.set_local_option('CFOUR', 'CFOUR_FREQ_ALGORITHM', 'PARALLEL')
psi4.set_local_option('CFOUR', 'CFOUR_ANH_ALGORITHM', 'PARALLEL')
psi4.set_local_option('CFOUR', 'CFOUR_ANHARMONIC', 'VPT2')
psi4.set_local_option('CFOUR', 'CFOUR_FD_PROJECT', 'OFF')
# When a Psi4 method is requested for vpt2, a skeleton of
# computations in Cfour is still required to hang the gradients
# upon. The skeleton is as cheap as possible (integrals only
# & sto-3g) and set up here.
if isC4notP4:
skelname = lowername
else:
skelname = 'c4-scf'
psi4.set_global_option('BASIS', 'STO-3G')
# P4 'c4-scf'/'cfour'CALC_LEVEL lowername # temporary
# C4 lowername cfour{} # temporary
if 'status' not in shelf:
shelf['status'] = 'initialized'
shelf['linkage'] = os.getpid()
shelf['zmat'] = {
} # Cfour-generated ZMAT files with finite difference geometries
shelf['fjobarc'] = {
} # Cfour- or Psi4-generated ascii files with packaged gradient results
shelf.sync()
else:
pass
# how decide whether to use. keep precedent of intco.dat in mind
# Construct and move into directory job scratch / cfour scratch / harm
psioh = psi4.IOManager.shared_object()
psio = psi4.IO.shared_object()
os.chdir(psioh.get_default_path()) # psi_scratch
cfour_tmpdir = kwargs['path'] if 'path' in kwargs else \
'psi.' + str(os.getpid()) + '.' + psio.get_default_namespace() + \
'.cfour.' + str(random.randint(0, 99999))
if not os.path.exists(cfour_tmpdir):
os.mkdir(cfour_tmpdir)
os.chdir(cfour_tmpdir) # psi_scratch/cfour
if not os.path.exists('harm'):
os.mkdir('harm')
os.chdir('harm') # psi_scratch/cfour/harm
psioh.set_specific_retention(32, True) # temporary, to track p4 scratch
#shelf['status'] = 'anharm_jobs_sown' # temporary to force backtrack
print('STAT', shelf['status']) # temporary
# Generate the ZMAT input file in scratch
with open('ZMAT', 'w') as handle:
cfour_infile = write_zmat(skelname, 1)
handle.write(cfour_infile)
print('\n====== Begin ZMAT input for CFOUR ======')
print(open('ZMAT', 'r').read())
print('======= End ZMAT input for CFOUR =======\n')
shelf['genbas'] = open('GENBAS', 'r').read()
# Check existing shelf consistent with generated ZMAT, store
if ('000-000'
in shelf['zmat']) and (shelf['zmat']['000-000'] != cfour_infile):
diff = difflib.Differ().compare(shelf['zmat']['000-000'].splitlines(),
cfour_infile.splitlines())
raise ValidationError(
"""Input file translated to Cfour ZMAT does not match ZMAT stored in shelf.\n\n"""
+ '\n'.join(list(diff)))
shelf['zmat']['000-000'] = cfour_infile
shelf.sync()
# Reset basis after Cfour skeleton seeded
psi4.set_global_option('BASIS', user_basis)
if shelf['status'] == 'initialized':
p4util.banner(' VPT2 Setup: Harmonic ')
# Generate the displacements that will form the harmonic freq
os.chdir(psioh.get_default_path() + cfour_tmpdir +
'/harm') # psi_scratch/cfour/harm
with open('partial.out', 'w') as handle:
handle.write(run_cfour_module('xjoda'))
handle.write(run_cfour_module('xsymcor'))
# Read the displacements that will form the harmonic freq
zmats0N = ['000-' + item[-3:] for item in sorted(glob.glob('zmat*'))]
for zm12 in zmats0N:
zm1, zm2 = zm12.split('-')
with open('zmat' + zm2, 'r') as handle:
shelf['zmat'][zm12] = handle.read()
shelf.sync()
psi4.print_out(
' CFOUR scratch file %s for %s-%s has been read\n' %
('zmat' + zm2, zm1, zm2))
psi4.print_out('%s\n' % shelf['zmat'][zm12])
# S/R: Write distributed input files for harmonic freq
if isSowReap:
os.chdir(current_directory)
inputSansMol = p4util.format_currentstate_for_input(gradient,
lowername,
allButMol=True,
**kwargs)
for zm12 in zmats0N:
zm1, zm2 = zm12.split('-')
ifile = vpt2_sow_files(zm12, shelf['linkage'], isC4notP4,
isC4fully, shelf['zmat'][zm12],
inputSansMol, shelf['genbas'])
with open('VPT2-' + zm12 + '.in', 'w') as handle:
handle.write(ifile)
msg = vpt2_instructions('harmonic', current_directory, zmats0N)
psi4.print_out(msg)
print(msg)
shelf['status'] = 'harm_jobs_sown'
# S/R: Pause for distributed calculations
if isSowReap:
shelf.close()
return 0.0
if shelf['status'] == 'harm_jobs_sown':
zmats0N = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] == '000' and item[-3:] != '000')
]
# S/R: Check that distributed calcs all completed correctly
if isSowReap:
msg = vpt2_instructions('harmonic', current_directory, zmats0N)
psi4.print_out(msg)
isOk, msg = sown_jobs_status(
current_directory, 'VPT2', zmats0N, reap_job_validate,
shelf['linkage'],
['CURRENT ENERGY', 'CURRENT DIPOLE', 'CURRENT GRADIENT'])
psi4.print_out(msg)
print(msg)
if not isOk:
shelf.close()
return 0.0
# Collect all results from gradients forming the harmonic freq
for zm12 in zmats0N:
zm1, zm2 = zm12.split('-')
if zm12 not in shelf['fjobarc']:
p4util.banner(' VPT2 Computation: %s ' % (zm12))
print(' VPT2 Computation: %s ' % (zm12))
fjobarc = vpt2_reaprun_files(zm12, shelf['linkage'], isSowReap,
isC4notP4, isC4fully,
shelf['zmat'][zm12],
current_directory,
psioh.get_default_path(),
cfour_tmpdir, lowername, kwargs)
shelf['fjobarc'][zm12] = fjobarc
shelf.sync()
shelf['status'] = 'harm_jobs_reaped'
if shelf['status'] == 'harm_jobs_reaped':
zmats0N = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] == '000' and item[-3:] != '000')
]
p4util.banner(' VPT2 Results: Harmonic ')
# Process the gradients into harmonic freq
os.chdir(psioh.get_default_path() + cfour_tmpdir +
'/harm') # psi_scratch/cfour/harm
harmout = run_cfour_module('xjoda')
harmout += run_cfour_module('xsymcor')
for zm12 in zmats0N:
zm1, zm2 = zm12.split('-')
with open('FJOBARC', 'w') as handle:
handle.write(shelf['fjobarc'][zm12])
harmout += run_cfour_module('xja2fja')
harmout += run_cfour_module('xsymcor')
shutil.move('FJOBARC', 'fja.' + zm12)
try:
os.remove('zmat' + zm2)
except OSError:
pass
harmout += run_cfour_module('xjoda')
harmout += run_cfour_module('xcubic')
psi4.print_out(harmout)
with open('harm.out', 'w') as handle:
handle.write(harmout)
# Generate displacements along harmonic normal modes
zmatsN0 = [item[-3:] for item in sorted(glob.glob('zmat*'))]
os.chdir('..') # psi_scratch/cfour
for zm1 in zmatsN0:
zm12 = zm1 + '-000'
with open(
psioh.get_default_path() + cfour_tmpdir + '/harm/zmat' +
zm1, 'r') as handle:
shelf['zmat'][zm12] = handle.read()
shelf.sync()
psi4.print_out(
' CFOUR scratch file %s for %s has been read\n' %
('zmat' + zm1, zm12))
psi4.print_out('%s\n' % shelf['zmat'][zm12])
# Collect displacements along the normal coordinates generated by the harmonic freq.
# Further harmonic freqs are to be run at each of these to produce quartic force field.
# To carry these out, generate displacements for findif by gradient at each displacement.
if os.path.exists(zm1):
shutil.rmtree(zm1)
os.mkdir(zm1)
os.chdir(zm1) # psi_scratch/cfour/004
with open('ZMAT', 'w') as handle:
handle.write(shelf['zmat'][zm12])
shutil.copy2('../harm/GENBAS',
'GENBAS') # ln -s $ecpdir/ECPDATA $j/ECPDATA
with open('partial.out', 'w') as handle:
handle.write(run_cfour_module('xjoda'))
handle.write(run_cfour_module('xsymcor'))
# Read the displacements that will form the anharmonic freq
zmatsNN = [item[-3:] for item in sorted(glob.glob('zmat*'))]
for zm2 in zmatsNN:
zm12 = zm1 + '-' + zm2
with open(
psioh.get_default_path() + cfour_tmpdir + '/' + zm1 +
'/zmat' + zm2, 'r') as handle:
shelf['zmat'][zm12] = handle.read()
shelf.sync()
psi4.print_out(
' CFOUR scratch file %s for %s has been read\n' %
('zmat' + zm2, zm12))
psi4.print_out('%s\n' % shelf['zmat'][zm12])
os.chdir('..') # psi_scratch/cfour
zmatsNN = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] != '000' and item[-3:] != '000')
]
# S/R: Write distributed input files for anharmonic freq
if isSowReap:
os.chdir(current_directory)
inputSansMol = p4util.format_currentstate_for_input(gradient,
lowername,
allButMol=True,
**kwargs)
for zm12 in zmatsNN:
zm1, zm2 = zm12.split('-')
ifile = vpt2_sow_files(zm12, shelf['linkage'], isC4notP4,
isC4fully, shelf['zmat'][zm12],
inputSansMol, shelf['genbas'])
# GENBAS needed here
with open('VPT2-' + zm12 + '.in', 'w') as handle:
handle.write(ifile)
msg = vpt2_instructions('anharmonic', current_directory, zmatsNN)
psi4.print_out(msg)
print(msg)
shelf['status'] = 'anharm_jobs_sown'
# S/R: Pause for distributed calculations
if isSowReap:
shelf.close()
return 0.0
if shelf['status'] == 'anharm_jobs_sown':
zmatsNN = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] != '000' and item[-3:] != '000')
]
# S/R: Check that distributed calcs all completed correctly
if isSowReap:
msg = vpt2_instructions('anharmonic', current_directory, zmatsNN)
psi4.print_out(msg)
isOk, msg = sown_jobs_status(
current_directory, 'VPT2', zmatsNN, reap_job_validate,
shelf['linkage'],
['CURRENT ENERGY', 'CURRENT DIPOLE', 'CURRENT GRADIENT'])
psi4.print_out(msg)
print(msg)
if not isOk:
shelf.close()
return 0.0
# Collect all results from gradients forming the anharmonic freq
for zm12 in zmatsNN:
zm1, zm2 = zm12.split('-')
if zm12 not in shelf['fjobarc']:
p4util.banner(' VPT2 Computation: %s ' % (zm12))
print(' VPT2 Computation: %s ' % (zm12))
fjobarc = vpt2_reaprun_files(zm12, shelf['linkage'], isSowReap,
isC4notP4, isC4fully,
shelf['zmat'][zm12],
current_directory,
psioh.get_default_path(),
cfour_tmpdir, lowername, kwargs)
shelf['fjobarc'][zm12] = fjobarc
shelf.sync()
shelf['status'] = 'anharm_jobs_reaped'
if shelf['status'] == 'anharm_jobs_reaped':
zmats0N = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] == '000' and item[-3:] != '000')
]
zmatsN0 = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] != '000' and item[-3:] == '000')
]
zmatsNN = [
item for item in sorted(shelf['zmat'].keys())
if (item[:3] != '000' and item[-3:] != '000')
]
p4util.banner(' VPT2 Results: Harmonic ')
# Process the gradients into harmonic freq
os.chdir(psioh.get_default_path() + cfour_tmpdir) # psi_scratch/cfour
if os.path.exists('anharm'):
shutil.rmtree('anharm')
os.mkdir('anharm')
os.chdir('harm') # psi_scratch/cfour/harm
run_cfour_module('xclean')
anharmout = run_cfour_module('xjoda')
anharmout += run_cfour_module('xsymcor')
for zm12 in zmats0N:
zm1, zm2 = zm12.split('-')
with open('FJOBARC', 'w') as handle:
handle.write(shelf['fjobarc'][zm12])
anharmout += run_cfour_module('xja2fja')
anharmout += run_cfour_module('xsymcor')
shutil.move('FJOBARC', 'fja.' + zm12)
anharmout += run_cfour_module('xjoda')
anharmout += run_cfour_module('xcubic')
psi4.print_out(anharmout)
with open('harm.out', 'w') as handle:
handle.write(anharmout)
# Process the gradients into harmonic freq at each normco displaced point
os.chdir('..') # psi_scratch/cfour
for zm11 in zmatsN0:
zm1 = zm11[:3]
if os.path.exists(zm1):
shutil.rmtree(zm1)
os.mkdir(zm1)
os.chdir(zm1) # psi_scratch/cfour/004
run_cfour_module('xclean')
with open('ZMAT', 'w') as handle:
handle.write(shelf['zmat'][zm11])
shutil.copy2('../harm/GENBAS', 'GENBAS')
anharmout = run_cfour_module('xjoda')
anharmout += run_cfour_module('xsymcor')
for zm22 in [
item for item in zmatsNN
if (item[:3] == zm1 and item[-3:] != '000')
]:
zm2 = zm22[-3:]
zm12 = zm1 + '-' + zm2
print(zm12)
with open('FJOBARC', 'w') as handle:
handle.write(shelf['fjobarc'][zm12])
anharmout += run_cfour_module('xja2fja')
anharmout += run_cfour_module('xsymcor')
shutil.move('FJOBARC', 'fja.' + zm12)
anharmout += run_cfour_module('xjoda')
anharmout += run_cfour_module('xja2fja')
with open('FJOBARC', 'r') as handle:
shelf['fjobarc'][zm11] = handle.read()
shelf.sync()
psi4.print_out(anharmout)
with open('partial.out', 'w') as handle:
handle.write(anharmout)
os.chdir('..') # psi_scratch/cfour
# Process the harmonic freqs at normco displacements into anharmonic freq
p4util.banner(' VPT2 Results: Anharmonic ')
os.chdir('anharm') # psi_scratch/cfour/anharm
shutil.copy2('../harm/JOBARC', 'JOBARC')
shutil.copy2('../harm/JAINDX', 'JAINDX')
for zm12 in zmatsN0:
with open('FJOBARC', 'w') as handle:
handle.write(shelf['fjobarc'][zm12])
anharmout = run_cfour_module('xja2fja')
anharmout += run_cfour_module('xcubic')
shutil.move('FJOBARC', 'fja.' + zm12)
psi4.print_out(anharmout)
with open('anharm.out', 'w') as handle:
handle.write(anharmout)
shelf['status'] = 'vpt2_completed'
# Finish up
os.chdir(current_directory)
shelf.close()
optstash.restore()
def check_solution(in_file, out_file, solution_file):
"""
Runs the given solution file, passing it the input of in_file
and asserting its output to the contents of out_file. Each file
should include the full path to the file
:param in_file: Program input file for solution file (Prob(xx).in.txt)
:param out_file: Expected output for solution file (Prob(xx).out.txt)
:param solution_file: Solution to Code Quest problem (Prob(xx).py)
:return: No return: AssertionError raised if the expected output is
incorrect, otherwise None.
"""
# get logger and log arguments
logger = logging.getLogger()
logger.info('Solution file: "{}"'.format(solution_file))
expected_out = ""
for file_name, file in (("Input", in_file), ("Expected out", out_file)):
with open(file, 'r') as f:
# reading the input in as lines makes it easier to work with
# the out files have \r in them at the end of every newline
# remove them because they are annoying and break correct
# solutions
f_contents = f.read().replace('\r', '').splitlines(True)
logger.debug('{} file contents:\n{!r}'.format(
file_name, f_contents))
if file == out_file: # we want to save this for later
expected_out = f_contents
# in_file and solution_file must be in the same directory, so if they
# aren't then create a temporary directory and move them both there
if os.path.dirname(in_file) != os.path.dirname(solution_file):
# define tmpdir outside of manager, so the directory isn't destroyed
tmpdir = tempfile.TemporaryDirectory()
with tempfile.TemporaryDirectory() as tmpdirname:
for file in (in_file, out_file):
shutil.copyfile(file, tmpdirname)
in_file = "{}/{}".format(os.path.basename(in_file), tmpdirname)
solution_file = "{}/{}".format(os.path.basename(solution_file),
tmpdirname)
# run the solution file using subprocess and then catch its output
path_to_python = sys.executable
logger.info("Using python interpreter at: '{}'".format(path_to_python))
logger.info("Executing solution file '{}'".format(
os.path.basename(solution_file)))
p = subprocess.Popen('"{}" {}'.format(path_to_python, solution_file),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
# returned from the read is a bytes string so decode it.
out, err = (x.decode('utf-8') for x in p.communicate())
if len(err) != 0: # if an error was raised
raise Exception("An error occurred while executing '{}': '{}'".format(
os.path.basename(solution_file), err))
else:
out = out.replace('\r', '').splitlines(True)
logger.debug('Got output:\n{!r}'.format(out))
# find differences between retrieved output and expected output
logger.info("Comparing output to expected output.")
diffs = difflib.Differ().compare(out, expected_out)
for line_no, diff in enumerate(diffs,
1): # starts counting line number at 1
if not diff.startswith(" "): # if the lines differ in some way:
error_file = in_file.replace("in", "error")
logger.error(
"Difference found at line {}, writing output to disk " \
"at: {}".format(line_no, error_file)
)
with open(error_file, 'w') as f:
f.write(''.join(out))
"""
it's worth mentioning that whenever a difference is found
using difflib this format is used:
' - (output from solution)'
' ? (location of difference)'
' + (expected output)'
NOTE: (sometimes the ? and + lines are switched)
In the below error, all spaces in the + and ? lines are
replaced with ^s for readability
"""
raise AssertionError(
"Solution '{}' failed at line {}: {!r} (difference: " \
" {!r}, expected: {!r}). Wrote incorrect output " \
"to {}".format(
os.path.basename(solution_file), line_no, diff,
next(diffs).replace(" ", "^"),
next(diffs).replace(" ", "^"),
error_file
)
)
logger.info("Success! {} passed".format(
os.path.basename(solution_file)))
def main():
"""Process and parse the common files."""
opts = parse_opts(sys.argv)
organization = opts.organization
files_to_compare = opts.files_to_check
repos_to_exclude = opts.exclude_repos
gh = Github()
temp_repositories = gh.search_repositories(query='user:'******'pystol/pystol')
repositories = []
for repo in temp_repositories:
if repo.full_name not in repos_to_exclude:
repositories.append(repo)
errors = False
for file in files_to_compare:
for repo in repositories:
url = ('https://raw.githubusercontent.com/%s/master/%s' %
(repo.full_name, file))
out = repo.full_name.split("/")[1] + "_" + file
try:
r = requests.get(url, stream=True)
r.raise_for_status()
open(out, 'wb').write(r.content)
except Exception as e:
print("-----")
print("Error:")
print("WGET fails to fetch some of the"
"common files across the repositories."
"Some of the files to download are not"
"present in all repos.")
print(url)
print(out)
raise e
with open(file) as f:
flines = f.readlines()
for repo in repositories:
with open(repo.full_name.split("/")[1] + "_" + file) as g:
glines = g.readlines()
d = difflib.Differ()
diffs = [
x for x in d.compare(flines, glines)
if x[0] in ('+', '-')
]
if diffs:
print(file + " is different than " +
repo.full_name.split("/")[1] + "_" + file)
errors = True
else:
print(file + " is the same than " +
repo.full_name.split("/")[1] + "_" + file)
if errors:
print("-----")
print("Error:")
print("Some of the files are not synchronized.")
raise
class CustomTestCase(StandardTestCase):
"""
Helper class, providing extended functionality over unittest.TestCase.
The methods assertEqual and assertNotEqual have been overwritten
to provide better support for multi-line strings. Furthermore,
see the compare_output method and the parameter list of __init__.
"""
compare = difflib.Differ().compare
"""Comparison method shared by all subclasses."""
def __init__(self,
method_name,
input,
expected,
id,
run_in_debugger=True,
suite_settings=None):
"""
Initialise the CustomTestCase.
Arguments:
method_name -- name of test method to run.
input -- input to the parser.
expected -- expected output from the parser.
id -- unique test identifier, used by the test framework.
run_in_debugger -- if true, run this test under the pdb debugger.
suite_settings -- settings overrides for this test suite.
"""
self.id = id
self.input = input
self.expected = expected
self.run_in_debugger = run_in_debugger
self.suite_settings = suite_settings.copy() or {}
# Ring your mother.
unittest.TestCase.__init__(self, method_name)
def __str__(self):
"""
Return string conversion. Overridden to give test id, in addition to
method name.
"""
return '%s; %s' % (self.id, unittest.TestCase.__str__(self))
def __repr__(self):
return "<%s %s>" % (self.id, unittest.TestCase.__repr__(self))
def clear_roles(self):
# Language-specific roles and roles added by the
# "default-role" and "role" directives are currently stored
# globally in the roles._roles dictionary. This workaround
# empties that dictionary.
roles._roles = {}
def setUp(self):
StandardTestCase.setUp(self)
self.clear_roles()
def compare_output(self, input, output, expected):
"""`input`, `output`, and `expected` should all be strings."""
if isinstance(input, unicode):
input = input.encode('raw_unicode_escape')
if sys.version_info > (3, 0):
# API difference: Python 3's node.__str__ doesn't escape
#assert expected is None or isinstance(expected, unicode)
if isinstance(expected, bytes):
expected = expected.decode('utf-8')
if isinstance(output, bytes):
output = output.decode('utf-8')
else:
if isinstance(expected, unicode):
expected = expected.encode('raw_unicode_escape')
if isinstance(output, unicode):
output = output.encode('raw_unicode_escape')
# Normalize line endings:
if expected:
expected = '\n'.join(expected.splitlines())
if output:
output = '\n'.join(output.splitlines())
try:
self.assertEqual(output, expected)
except AssertionError as error:
print('\n%s\ninput:' % (self, ), file=sys.stderr)
print(input, file=sys.stderr)
try:
comparison = ''.join(
self.compare(expected.splitlines(1), output.splitlines(1)))
print('-: expected\n+: output', file=sys.stderr)
print(comparison, file=sys.stderr)
except AttributeError: # expected or output not a string
# alternative output for non-strings:
print('expected: %r' % expected, file=sys.stderr)
print('output: %r' % output, file=sys.stderr)
raise error
def raw_unified_diff(self) -> Iterator[str]:
differ = difflib.Differ()
lines_lhs = self.lhs.splitlines()
lines_rhs = self.rhs.splitlines()
return differ.compare(lines_lhs, lines_rhs)
def calculate_checksum_and_validate(self, last_loaded_key, iteration=0):
diff_keys = {}
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_1_node_without_dr')
with PiClient(self.clusters[0].grid, client_config,
new_instance=True) as piclient:
checksum_master = create_distributed_checksum_operation().evaluate(
)
if COLLECT_KEYS:
cache_names = piclient.get_ignite().cacheNames().toArray()
diff_keys['master'] = {}
for cache in cache_names:
diff_keys['master'][cache] = []
iterator = piclient.get_ignite().cache(cache).iterator()
while iterator.hasNext():
diff_keys['master'][cache].append(
iterator.next().getKey())
diff_keys['master'][cache].sort()
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_2_node_without_dr')
with PiClient(self.clusters[1].grid, client_config,
new_instance=True) as piclient:
checksum_slave = create_distributed_checksum_operation().evaluate()
if COLLECT_KEYS:
cache_names = piclient.get_ignite().cacheNames().toArray()
diff_keys['replica'] = {}
for cache in cache_names:
diff_keys['replica'][cache] = []
iterator = piclient.get_ignite().cache(cache).iterator()
while iterator.hasNext():
diff_keys['replica'][cache].append(
iterator.next().getKey())
diff_keys['replica'][cache].sort()
if checksum_master != checksum_slave:
print(f"Checksum master\n{checksum_master}")
print(f"Checksum slave\n{checksum_slave}")
# if COLLECT_KEYS:
# pprint(diff_keys, width=240)
comparison = ''.join(difflib.Differ().compare(
checksum_master.splitlines(True),
checksum_slave.splitlines(True)))
if iteration > 3:
print(comparison)
# log_print('Unable to get checksum equality on both clusters after FST', color='blue')
#
# return
tiden_assert(
False,
'Unable to get checksum equality on both clusters after FST'
)
if iteration > 2:
if DO_FST_ON_DIFFERENCE:
caches_with_diff = []
for line in comparison.split('\n'):
val = None
if line.startswith('-'):
m = search('Cache.*\'(.*)\'.rows', line)
if m:
val = m.group(1)
if val:
caches_with_diff.append(val)
log_print(f'Difference detected in some caches.',
color='red')
log_print(f'Starting clear() caches: {caches_with_diff}.',
color='red')
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_2_node_without_dr')
with PiClient(self.clusters[1].grid,
client_config,
new_instance=True) as piclient:
ignite = piclient.get_ignite()
for cache in caches_with_diff:
ignite.cache(cache).clear()
# cache_names = piclient.get_ignite().cacheNames().toArray()
#
# print(list(
# create_checksum_operation(cache_name, 1, last_loaded_key).evaluate() for cache_name in
# cache_names))
util_sleep_for_a_while(RECHECK_CHECKSUM_TIMEOUT)
log_print(
f'Starting full state transfer on caches: {caches_with_diff}.',
color='red')
client_config = Ignite.config_builder.get_config(
'client', config_set_name='cluster_1_node_without_dr')
with PiClient(self.clusters[0].grid,
client_config,
new_instance=True) as piclient:
ignite = piclient.get_ignite()
try:
futs = []
for cache in caches_with_diff:
# TODO https://ggsystems.atlassian.net/browse/GG-22669
ignite.cache(cache)
futs.append(
ignite.plugin(
'GridGain').dr().stateTransfer(
cache, bytes([2])))
for fut in futs:
fut.get()
except Exception as e:
log_print(
'Exception caught on on FST\n{}'.format(e),
color='red')
log_print('Going to restart replication with FST',
color='yellow')
futs = []
for cache in caches_with_diff:
ignite.cache(cache)
ignite.plugin(
"GridGain").dr().startReplication(cache)
futs.append(
ignite.plugin(
'GridGain').dr().stateTransfer(
cache, bytes([2])))
for fut in futs:
fut.get()
util_sleep_for_a_while(FST_TIMEOUT)
log_print(
f'Going to collect checksum again after timeout - {RECHECK_CHECKSUM_TIMEOUT} seconds',
color='red')
util_sleep_for_a_while(RECHECK_CHECKSUM_TIMEOUT)
return self.calculate_checksum_and_validate(
last_loaded_key, iteration + 1)
return checksum_master, checksum_slave
def test_csv(self, request_dhis2, request_ocl):
d = difflib.Differ()
return d.compare(request_dhis2.text.splitlines(1),
request_ocl.text.splitlines(1))
def merge_recombine(self, recombination):
shell('git fetch replica')
shell('git fetch original')
removed_commits = list()
pick_revision = recombination.main_source.revision
merge_revision = recombination.patches_source.revision
patches_branch = recombination.patches_source.branch
target_replacement_branch = recombination.target_replacement_branch
# Branch prep
# local patches branch
shell('git checkout -B recomb_attempt-%s-base %s' %
(patches_branch, merge_revision))
# local recomb branch
cmd = shell('git rev-parse %s~1' % pick_revision)
starting_revision = cmd.output[0]
shell('git checkout -B %s %s' %
(recombination.branch, starting_revision))
log.info("Creating remote disposable branch on replica")
cmd = shell('git push replica HEAD:%s' % recombination.branch)
if cmd.returncode != 0:
raise PushError
patches_removal_queue = list()
merge = shell("git merge --stat --squash --no-commit %s %s" %
(pick_revision, merge_revision))
if merge.returncode != 0:
attempt_number = 0
patches_base = dict()
log.error("first attempt at merge failed")
cmd = shell('git status --porcelain')
prev_conflict_status = cmd.output
cmd = shell('git merge-base %s %s' %
(pick_revision, merge_revision))
ancestor = cmd.output[0]
cmd = shell(
'git rev-list --reverse --first-parent %s..remotes/replica/%s'
% (ancestor, patches_branch))
patches_removal_queue = cmd.output
for commit in cmd.output:
cmd = shell('git show --pretty=format:"" %s' % commit,
show_stdout=False)
diff = '\n'.join(cmd.output)
hash_object = hashlib.sha1(diff)
patches_base[hash_object.hexdigest()] = commit
if patches_removal_queue:
log.warning("attempting automatic resolution")
else:
log.error("automatic resolution impossible")
else:
log.info("Merge successful")
retry_branch = None
while merge.returncode != 0 and patches_removal_queue:
attempt_number += 1
shell('git reset --hard %s' % recombination.branch)
shell('git checkout recomb_attempt-%s-base' % patches_branch)
retry_branch = 'recomb_attempt-%s-retry_%s' % (patches_branch,
attempt_number)
shell('git checkout -b %s' % (retry_branch))
# Rebasing changes all the commits hashes after "commit"
next_patch_toremove = patches_removal_queue.pop(0)
shell('git rebase -p --onto %s^ %s' %
(next_patch_toremove, next_patch_toremove))
cmd = shell('git rev-parse %s' % retry_branch)
retry_merge_revision = cmd.output[0]
shell('git checkout %s' % recombination.branch)
merge = shell("git merge --stat --squash --no-commit %s %s" %
(pick_revision, retry_merge_revision))
if merge.returncode != 0:
log.warning("automatic resolution attempt %d failed" %
attempt_number)
cmd = shell('git status --porcelain')
conflict_status = cmd.output
diff = difflib.Differ()
same_status = list(
diff.compare(prev_conflict_status, conflict_status))
if not same_status:
removed_commits.append(next_patch_toremove)
# removing this patch did not solve everything, but did
# something nonetheless, keep it removed and try to remove
# something else too
# change the base, recalculate every patch commit hash since
# rebase changed everything
shell('git branch -D recomb_attempt-%s-base' %
patches_branch)
shell(
'git checkout -B recomb_attempt-%s-base %s' %
patches_branch, retry_merge_revision)
cmd = shell(
'git rev-list --reverse --first-parent %s..%s' %
(ancestor, retry_branch))
patches_removal_queue = cmd.output
prev_conflict_status = conflict_status
shell('git branch -D %s' % retry_branch)
else:
logsummary.warning(
"automatic resolution attempt %d succeeded" %
attempt_number)
merge_revision = retry_merge_revision
removed_commits.append(next_patch_toremove)
logsummary.info("removed commits")
logsummary.info(removed_commits)
logsummary.info(
"removed commits (commit hash relative to starting patches branch)"
)
logsummary.info('removed_commits_deashed')
if merge.returncode != 0:
logsummary.error("automatic resolution failed")
shell('git push replica :%s' % recombination.branch)
else:
logsummary.info("Recombination successful")
# create new patches-branch
# TODO: understand if this can be a automatic task or we just notify someone
if retry_branch:
shell('git push replica :%s' % patches_branch)
shell('git push replica %s:refs/heads/%s' %
(retry_branch, patches_branch))
shell('git branch -D %s' % retry_branch)
recombination.removed_patches_commits = removed_commits
recombination.status = "SUCCESSFUL"
self.commit_recomb(recombination)
# Create target branch replacement for this recombination
shell('git checkout -B %s %s' %
(target_replacement_branch, starting_revision))
cmd = shell("git merge --log --no-edit %s %s" %
(pick_revision, merge_revision))
if cmd.returncode == 0:
shell('git push replica HEAD:%s' % target_replacement_branch)
shell('git checkout parking')
#shell('git branch -D %s' % recombination_branch)
shell('git branch -D recomb_attempt-%s-base' % patches_branch)
shell('git branch -D %s' % target_replacement_branch)
def test_Type1Font():
filename = os.path.join(os.path.dirname(__file__), 'cmr10.pfb')
font = t1f.Type1Font(filename)
slanted = font.transform({'slant': 1})
condensed = font.transform({'extend': 0.5})
with open(filename, 'rb') as fd:
rawdata = fd.read()
assert font.parts[0] == rawdata[0x0006:0x10c5]
assert font.parts[1] == rawdata[0x10cb:0x897f]
assert font.parts[2] == rawdata[0x8985:0x8ba6]
assert font.decrypted.startswith(b'dup\n/Private 18 dict dup begin')
assert font.decrypted.endswith(b'mark currentfile closefile\n')
assert slanted.decrypted.startswith(b'dup\n/Private 18 dict dup begin')
assert slanted.decrypted.endswith(b'mark currentfile closefile\n')
assert b'UniqueID 5000793' in font.parts[0]
assert b'UniqueID 5000793' in font.decrypted
assert font._pos['UniqueID'] == [(797, 818), (4483, 4504)]
len0 = len(font.parts[0])
for key in font._pos.keys():
for pos0, pos1 in font._pos[key]:
if pos0 < len0:
data = font.parts[0][pos0:pos1]
else:
data = font.decrypted[pos0 - len0:pos1 - len0]
assert data.startswith(f'/{key}'.encode('ascii'))
assert {'FontType', 'FontMatrix', 'PaintType', 'ItalicAngle', 'RD'} < set(
font._pos.keys())
assert b'UniqueID 5000793' not in slanted.parts[0]
assert b'UniqueID 5000793' not in slanted.decrypted
assert 'UniqueID' not in slanted._pos
assert font.prop['Weight'] == 'Medium'
assert not font.prop['isFixedPitch']
assert font.prop['ItalicAngle'] == 0
assert slanted.prop['ItalicAngle'] == -45
assert font.prop['Encoding'][5] == 'Pi'
assert isinstance(font.prop['CharStrings']['Pi'], bytes)
assert font._abbr['ND'] == 'ND'
differ = difflib.Differ()
diff = list(
differ.compare(font.parts[0].decode('latin-1').splitlines(),
slanted.parts[0].decode('latin-1').splitlines()))
for line in (
# Removes UniqueID
'- /UniqueID 5000793 def',
# Changes the font name
'- /FontName /CMR10 def',
'+ /FontName/CMR10_Slant_1000 def',
# Alters FontMatrix
'- /FontMatrix [0.001 0 0 0.001 0 0 ]readonly def',
'+ /FontMatrix [0.001 0 0.001 0.001 0 0] readonly def',
# Alters ItalicAngle
'- /ItalicAngle 0 def',
'+ /ItalicAngle -45.0 def'):
assert line in diff, 'diff to slanted font must contain %s' % line
diff = list(
differ.compare(font.parts[0].decode('latin-1').splitlines(),
condensed.parts[0].decode('latin-1').splitlines()))
for line in (
# Removes UniqueID
'- /UniqueID 5000793 def',
# Changes the font name
'- /FontName /CMR10 def',
'+ /FontName/CMR10_Extend_500 def',
# Alters FontMatrix
'- /FontMatrix [0.001 0 0 0.001 0 0 ]readonly def',
'+ /FontMatrix [0.0005 0 0 0.001 0 0] readonly def'):
assert line in diff, 'diff to condensed font must contain %s' % line
def __make_diff(self):
differ = difflib.Differ()
diff = differ.compare(self.original_data, self.changed_data)
return diff
text_correct = """
Bloomberg, who will be on the ballot Tuesday for the first time this primary season, made the
comments amid new turmoil in the presidential contest. With rivals Pete Buttigieg and Amy Klobuchar
now abruptly out, Joe Biden has consolidated support among establishment Democrats hoping to
present a more centrist alternative to democratic socialist Bernie Sanders.
""".splitlines()
text_wrong = """
Bloomberg, who will be on the ballot tuesday for the first time this primary season, mdae the
comments amid enw tuomiol in the presidential contest. With rivals Pete Buttigieg and amy Klobuchar
now abruptly out, Joe Biden has consolidated support among establishment Democrats hoping to
present a more centrist altenative to democratic socialist bernie sanders.
""".splitlines()
checker = difflib.Differ()
diff = checker.compare(text_correct, text_wrong)
print("\n".join(diff))
# - Bloomberg, who will be on the ballot Tuesday for the first time this primary season, made the
# ? ^ -
# + Bloomberg, who will be on the ballot tuesday for the first time this primary season, mdae the
# ? ^ +
# - comments amid new turmoil in the presidential contest. With rivals Pete Buttigieg and Amy Klobuchar
# ? - ^ - ^ -
# + comments amid enw tuomiol in the presidential contest. With rivals Pete Buttigieg and amy Klobuchar
# ? + + ^ + ^
def compare(
expected_dir,
actual_dir,
file_pattern=None,
actual_extra=False,
scrubs=None,
):
"""Compare files matching `file_pattern` in `expected_dir` and `actual_dir`.
A version-specific subdirectory of `expected_dir` will be used if
it exists.
`actual_extra` true means `actual_dir` can have extra files in it
without triggering an assertion.
`scrubs` is a list of pairs: regexes to find and replace to scrub the
files of unimportant differences.
An assertion will be raised if the directories fail one of their
matches.
"""
expected_dir = versioned_directory(expected_dir)
dc = filecmp.dircmp(expected_dir, actual_dir)
diff_files = fnmatch_list(dc.diff_files, file_pattern)
expected_only = fnmatch_list(dc.left_only, file_pattern)
actual_only = fnmatch_list(dc.right_only, file_pattern)
# filecmp only compares in binary mode, but we want text mode. So
# look through the list of different files, and compare them
# ourselves.
text_diff = []
for f in diff_files:
expected_file = os.path.join(expected_dir, f)
with open(expected_file, READ_MODE) as fobj:
expected = fobj.read()
if expected_file.endswith(".xml"):
expected = canonicalize_xml(expected)
actual_file = os.path.join(actual_dir, f)
with open(actual_file, READ_MODE) as fobj:
actual = fobj.read()
if actual_file.endswith(".xml"):
actual = canonicalize_xml(actual)
if scrubs:
expected = scrub(expected, scrubs)
actual = scrub(actual, scrubs)
if expected != actual: # pragma: only failure
text_diff.append('%s != %s' % (expected_file, actual_file))
expected = expected.splitlines()
actual = actual.splitlines()
print(":::: diff {!r} and {!r}".format(expected_file, actual_file))
print("\n".join(difflib.Differ().compare(expected, actual)))
print(":::: end diff {!r} and {!r}".format(expected_file,
actual_file))
assert not text_diff, "Files differ: %s" % '\n'.join(text_diff)
assert not expected_only, "Files in %s only: %s" % (expected_dir,
expected_only)
if not actual_extra:
assert not actual_only, "Files in %s only: %s" % (actual_dir,
actual_only)
