Example #1
0
def file_diff(a,b,extend_name=""):
    """
    Compare two files for differences, print only differences to console and
    put in a file in logs directory.
    """

    #a = "/home/RunFromHere/logs/10.38.36.67.txt"
    #b = "/home/RunFromHere/logs/10.38.36.167.txt"
    #difference = ("/home/RunFromHere/logs/difference_%s.txt" % c)
    difference = ("logs/difference_%s.txt" % extend_name)
    #filecmp = difflib.Differ()
    
    z = filecmp.cmp(a,b)
    if z == True:
        print("\n\nThe files are the same")
        return(True)
    else:
        with open (a) as File1:
            c = File1.readlines() 
        with open (b) as File2:
            d = File2.readlines()
    print('\n')
    for line in difflib.context_diff(c,d, fromfile=(a), tofile=(b), n=0):
        print((line))
    with open (difference, 'w') as differ:
        for line in difflib.context_diff(c,d, fromfile=(a), tofile=(b), n=0):
            differ.write(line)    
    return(False) ## false would mean that there are differences 
Example #2
0
	def test(self, ctx):
		self.perform(ctx)
		#displayed = True
		
		# launch the command
		if self.out:
			elfmake.mkdir(str(self.out.parent()))
			out_stream = open(str(self.out), "w")
		else:
			out_stream = NULL
		if self.err:
			err_stream = open(str(self.err), "w")
		else:
			err_stream = NULL
		if self.input:
			in_stream = open(str(self.input), "r")
		else:
			in_stream = NULL
		cmd = action.make_line(self.cmd)
		if elfmake.verbose:
			ctx.print_info("running %s" % cmd) 
		rc = subprocess.call(cmd, stdin = in_stream, stdout = out_stream, stderr = err_stream, shell = True)
		if rc <> 0:
			self.failure(ctx, "return code = %d, command = %s" % (rc, cmd))
			return
			
		# compare output if any
		if self.out:
			if not self.out_ref.exists():
				self.info(ctx, "no reference file for output, creating it!")
				elfmake.mkdir(str(self.out_ref.parent()))
				shutil.copyfile(str(self.out), str(self.out_ref))
			else:
				c = 0
				for l in difflib.context_diff(open(str(self.out), "r").readlines(), open(str(self.out_ref), "r").readlines()):
					c += 1
				if c:
					self.failure(ctx, "different output stream")
					return
		
		# compare error if any
		if self.err:
			if not os.path.exists(self.err_ref):
				self.info(ctx, "no reference file for error, creating it!")
				shutil.copyfile(self.err, self.err_ref)
			else:
				c = 0
				for l in difflib.context_diff(open(self.err, "r").readlines(), open(self.err_ref, "r").readlines()):
					c += 1
				if c:
					self.failure(ctx, "different error stream")
					return
			
		# display result
		self.success(ctx)
def main():
    parser = argparse.ArgumentParser(description="Compare two flattened UW XML files.")
    parser.add_argument("xml_file_1",
                        help="The first XML file.")
    parser.add_argument("xml_file_2",
                        help="The second XML file.")
    args = parser.parse_args()

    if not os.path.isfile(args.xml_file_1):
        sys.exit("ERROR - Can't find first XML file: {}".format(args.xml_file_1))
    if not os.path.isfile(args.xml_file_2):
        sys.exit("ERROR - Can't find second XML file: {}".format(args.xml_file_2))

    tree1 = ElementTree.parse(args.xml_file_1)
    tree2 = ElementTree.parse(args.xml_file_2)
    dict1 = _elementToDict(tree1.getroot())
    dict2 = _elementToDict(tree2.getroot())
    diff = dict_diff(dict1, dict2)
    if(len(diff)):
        print "Differences:"
        for key, value in diff.items():
            first = str(value[0]).split('),')
            second = str(value[1]).split('),')
            result = list(difflib.context_diff(first, second, lineterm=""))
            pprint(result)
    else:
        print "No Differences"
Example #4
0
def func(args):
    # Get modification times
    fromdate = time.ctime(os.stat(args.FILE1).st_mtime)
    todate = time.ctime(os.stat(args.FILE2).st_mtime)

    # Open fromfile
    try:
        with open(args.FILE1, 'U') as fd:
            fromlines = fd.readlines()
    except IOError:
        print("Error opening file " + args.FILE1, file=sys.stderr)

    # Open tofile
    try:
        with open(args.FILE2, 'U') as fd:
            tolines = fd.readlines()
    except IOError:
        print("Error opening file " + args.FILE2, file=sys.stderr)

    # Create diff
    if args.unified:
        diff = difflib.unified_diff(fromlines, tolines, args.FILE1, args.FILE2,
                                    fromdate, todate, n=args.lines)
    elif args.ndiff:
        diff = difflib.ndiff(fromlines, tolines)
    elif args.html:
        diff = difflib.HtmlDiff().make_file(fromlines, tolines, args.FILE1,
                                            args.FILE2, context=args.context,
                                            numlines=args.lines)
    else:
        diff = difflib.context_diff(fromlines, tolines, args.FILE1, args.FILE2,
                                    fromdate, todate, n=args.lines)

    # we're using writelines because diff is a generator
    sys.stdout.writelines(diff)
Example #5
0
def xml_diff(xml_1, xml_2, ignore_order=True):
    """This function will generate string showing the differences
       between two json data (similar to the Linux 'diff' command output)
       ignore_order - Flag denotes whether to ignore the
                      order of dictionary keys and list items

    Arguments:
        xml_1, xml_1 - XML contents (strings) to be compared

    Return:
        String describing the differences (similar to the
        Linux 'diff' command output)
    """
    if ignore_order:
        xml_1 = ordered_xml(xml_1)
        xml_2 = ordered_xml(xml_2)

    pp1 = pretty_xml(xml_1)
    pp2 = pretty_xml(xml_2)

    pp_lst1 = [item for item in pp1.split('\n') if item.strip()]
    pp_lst2 = [item for item in pp2.split('\n') if item.strip()]

    diff_lines = [line for line in difflib.context_diff(pp_lst1, pp_lst2)]
    return '\n'.join(diff_lines)
    def test_rebuilding_does_not_generate_new_content(self):

        with resource("__buckconfig_common.soy") as buckconfig_common, resource(
            "files-and-dirs/buckconfig.soy"
        ) as buckconfig, resource(
            "generate_buckconfig_aliases", suffix=".pex"
        ) as script, named_temp_file() as temp_out:
            subprocess.check_call(
                [
                    sys.executable,
                    script,
                    "--buckconfig",
                    buckconfig,
                    "--buckconfig-aliases",
                    buckconfig_common,
                    "--buckconfig-aliases-dest",
                    temp_out,
                ]
            )
            with open(temp_out, "r") as actual_in, open(
                buckconfig_common, "r"
            ) as expected_in:
                actual = actual_in.read()
                expected = expected_in.read()

                if actual != expected:
                    diff = "\n".join(
                        difflib.context_diff(expected.splitlines(), actual.splitlines())
                    )
                    raise AssertionError(
                        "Expected unchanged buckconfig_common. Got\n{}".format(diff)
                    )
Example #7
0
 def test_fixture(url, expected_html):
     print('url=%s' % url)
     
     actual_html = readable(url)
     
     # Write the HTML for later diagnosis
     _, actual_html_fn = tempfile.mkstemp('readabilitytest')
     os.close(_)
     _, expected_html_fn = tempfile.mkstemp('readabilitytest')
     os.close(_)
     with codecs.open(actual_html_fn, 'w', DEFAULT_ENCODING) as f:
         f.write(actual_html)
     with codecs.open(expected_html_fn, 'w', DEFAULT_ENCODING) as f:
         f.write(expected_html)
     
     # Verify that there is no 'diff' between the two versions
     diff = list(difflib.context_diff(
         actual_html.splitlines(), expected_html.splitlines()))
     diff_string = '\n'.join(diff)
     assert not diff, (
         'readable version differs; diff:\n{0}\n'
         'expected: {1}\n'
         'actual: {2}').format(
         diff_string,
         expected_html_fn,
         actual_html_fn,
     )
Example #8
0
 def calculate_diff(self, new, old, zone):
     return "\n".join(context_diff(
         a=old.splitlines(),
         b=new.splitlines(),
         fromfile=str(zone.domain_active_serial) + '-' + zone.domain_name,
         tofile=str(zone.domain_serial) + '-' + zone.domain_name,
         lineterm=''))
Example #9
0
def check():
    mech = mechanize.Browser()
    url = settings.watch_url
    soup = BeautifulSoup(mech.open(url).read())
    links = [x['href'] for x in soup.findAll('a')]
    links = filter(is_magnet_link, links)
    seen_links_loc = '%s/seen.list' % settings.working_dir

    if os.path.exists(seen_links_loc):
        with open(seen_links_loc, 'r') as seen:
            seen_links = [x.strip() for x in seen.readlines()]
        diff = difflib.context_diff(seen_links, links)
        if links != seen_links:
            for d in diff:
                if d[0:1] == '+':
                    magnet_link = d[2:]
                    print("adding new magnet link: " + magnet_link)
                    subprocess.call(
                            [
                            'bash',
                            '%s/convert.sh' % settings.working_dir,
                            settings.torrent_watch_dir,
                            magnet_link])
            write_new_seen(links, seen_links_loc)
    else:
        write_new_seen(links, seen_links_loc)
Example #10
0
def _check_register_format(reg, build_monitor):
    """
    Check the register format.

    We do this by generating a perfectly formatted version and
    then comparing it to the version on disc.

    Any discrepancy will be reported as a design nonconformity.

    """
    register_text  = reg['text']
    reference_text = _format_register(reg['data'])
    if reference_text != register_text:

        reference_lines = reference_text.splitlines()
        register_lines  = register_text.splitlines()
        iline = 0
        for iline, lines in enumerate(zip(reference_lines, register_lines)):
            (ref_line, reg_line) = lines
            if ref_line != reg_line:
                break

        diff_text = '\n'.join((line for line in difflib.context_diff(
                                                        reference_lines,
                                                        register_lines,
                                                        'reference_format',
                                                        'current_register')))
        msg = 'Dependencies register format error '\
              'on line {line}:\n{diff}'.format(line = iline,
                                               diff = diff_text)

        build_monitor.report_nonconformity(tool    = 'da.check.dependencies',
                                           msg_id  = 'E6005',
                                           msg     = msg,
                                           file    = reg['filepath'])
Example #11
0
def main():

    usage = "usage: %prog [options] fromfile tofile"
    parser = optparse.OptionParser(usage)
    parser.add_option("-c", action="store_true", default=False, help='Produce a context format diff (default)')
    parser.add_option("-u", action="store_true", default=False, help='Produce a unified format diff')
    parser.add_option("-m", action="store_true", default=False, help='Produce HTML side by side diff (can use -c and -l in conjunction)')
    parser.add_option("-n", action="store_true", default=False, help='Produce a ndiff format diff')
    parser.add_option("-l", "--lines", type="int", default=3, help='Set number of context lines (default 3)')
    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.print_help()
        sys.exit(1)
    if len(args) != 2:
        parser.error("need to specify both a fromfile and tofile")

    n = options.lines
    fromfile, tofile = args

    fromdate = time.ctime(os.stat(fromfile).st_mtime)
    todate = time.ctime(os.stat(tofile).st_mtime)
    fromlines = open(fromfile, 'U').readlines()
    tolines = open(tofile, 'U').readlines()

    if options.u:
        diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)
    elif options.n:
        diff = difflib.ndiff(fromlines, tolines)
    elif options.m:
        diff = difflib.HtmlDiff().make_file(fromlines,tolines,fromfile,tofile,context=options.c,numlines=n)
    else:
        diff = difflib.context_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n)

    sys.stdout.writelines(diff)
def main():
    print 'Getting initial reference file...'
    refHTML = getHTML( remoteURL )
    while( refHTML == False ):
        print '...retrying...'
        refHTML = getHTML( remoteURL )
    writeHTML( localFile, refHTML )
    
    while True:
            print 'Getting latest page...', time.strftime('%a, %d %b %Y %H:%M:%S',time.localtime())
            newHTML = getHTML( remoteURL )
            while( newHTML == False ):
                print '...retrying...'
                newHTML = getHTML( remoteURL )
    
            refHTML = getHTML( localURL )
        
            diffList = difflib.context_diff(
                            newHTML.splitlines(),
                            refHTML.splitlines() )
    
            listOfDiffs = list(diffList)
            numDiffs =  len( listOfDiffs )
    
            if numDiffs == 0:
                print "No change..."
                time.sleep(secondsDelay)
            else:
                print "Page changed: sending email"
                emailChanges(refHTML, newHTML, listOfDiffs)
    
                print "Updating reference page..."
                writeHTML( localFile, newHTML )
Example #13
0
def runDiff(a, b, outFile, acceptable_diffs=()):
    fa = open(a, "rU")
    fb = open(b, "rU")
    all_diffs = list(difflib.context_diff(fa.readlines(), fb.readlines(), n=0))
    print "\n".join(all_diffs)
    if all_diffs:
        all_diffs = parse_diffs(all_diffs)
        for k in acceptable_diffs:
            try:
                del all_diffs[k]
            except: 
                pass
    if all_diffs:
        l = []
        for k, vi in all_diffs.iteritems():
            t = []
            for v in vi:
                t.extend(v)
            l.append((k, t))
        l.sort()
        nd = []
        for k, lines in l:
            nd.extend(lines)
        s = "".join(nd)
        fout = open(outFile, "a")
        fout.write("Diffs in file %s:\n" % a)
        fout.write(s)
        if len(s) < 5000:
            t = ":\n%s\n(diffs also stored at the end of %s)." % (s, outFile)
        else:
            t = ". See the end of the %s." % (outFile)
        fout.close()
        sys.exit("Script aborted because of failed example (%s).\nOutput differed from the expected output%s\n" % (last_test,t))
    fa.close()
    fb.close()
def calculate_diff(plugins,options):
    od_plugins = collections.OrderedDict(sorted(plugins.items()))
    diff = ""
    for plugin_name, plugin_code in od_plugins.iteritems():
        ks = list(plugin_code)
        fromlines = StringIO.StringIO(plugin_code[ks[0]]).readlines()
        tolines = StringIO.StringIO(plugin_code[ks[1]]).readlines()
        now = time.time()
        fromdate = now
        todate = now
        fromfile = ks[0] + ": " + plugin_name
        tofile = ks[1] + ": " + plugin_name
        
        if options.u:
            diff += ''.join(difflib.unified_diff(fromlines, tolines, fromfile, tofile,
                                    fromdate, todate, n=options.lines))
        elif options.n:
            diff += ''.join(difflib.ndiff(fromlines, tolines))
        elif options.m:
            diff += difflib.HtmlDiff().make_table(
                                    fromlines,
                                    tolines,
                                    " " + fromfile + " ",
                                    " " + tofile + " ",
                                    context=options.c,
                                    numlines=options.lines) + "<br /> <p />"
        else:
            diff += ''.join(difflib.context_diff(fromlines, tolines, fromfile, tofile,
                                        fromdate, todate, n=options.lines))
        
    return diff
Example #15
0
    def test_no_trailing_tab_on_empty_filedate(self):
        args = ['one', 'two', 'Original', 'Current']
        ud = difflib.unified_diff(*args, lineterm='')
        self.assertEqual(list(ud)[0:2], ["--- Original", "+++ Current"])

        cd = difflib.context_diff(*args, lineterm='')
        self.assertEqual(list(cd)[0:2], ["*** Original", "--- Current"])
Example #16
0
 def _diff(self, baselinedir, outputdir, dc=None):
     if dc is None:
         dc = filecmp.dircmp(baselinedir, outputdir, ['.svn'])
     if dc.left_only:
         self.fail("Files or subdirectories missing from output: "
                   +str(dc.left_only))
     if dc.right_only:
         self.fail("Files or subdirectories missing from baseline: "
                   +str(dc.right_only))
     for name in dc.diff_files:
         fromfile = join(dc.left, name)
         tofile = join(dc.right, name)
         with open(fromfile, 'r') as f_from:
             fromlines = f_from.readlines()
             with open(tofile, 'r') as f_to:
                 tolines = f_to.readlines()
                 diff = difflib.context_diff(fromlines, tolines,
                                             fromfile+" (baseline)",
                                             tofile+" (output)")
                 out = StringIO()
                 out.write("Output file does not match baseline:\n")
                 for line in diff:
                     out.write(line)
                 self.fail(out.getvalue())
     for subdir in dc.subdirs:
         self._diff(join(baselinedir, subdir),
                    join(outputdir, subdir),
                    dc=dc.subdirs[subdir])
     shutil.rmtree(outputdir, ignore_errors=True)
Example #17
0
def generate_diff(trial,ref,what): 
    diff_name = ".".join([trial[what],'diff'])
    with open(trial[what], 'rb') as of:
        ref[what].seek(0)
        diff = difflib.context_diff([line for line in of], [line for line in ref[what]], fromfile=of.name, tofile='reference')
        #sys.stderr.write("Comparing {} to reference {}\n".format(of.name,ref[what].name))
        return (diff),
Example #18
0
    def get_diff(self, s1, s2):  # pylint: disable=C0103
        diff = difflib.context_diff(s1, s2)
        rope = ""
        for line in diff:
            rope += line

        return rope
Example #19
0
 def calculate_diff(self, config, new, old):
     return "\n".join(context_diff(
         a=old.splitlines(),
         b=new.splitlines(),
         fromfile=str(config.active_serial),
         tofile=str(config.serial),
         lineterm=''))
Example #20
0
 def show_diff(self, newdata, langfile):
     with open(langfile) as f:
         olddata = f.read().split('\n')
     diff = difflib.context_diff(olddata, newdata.split('\n'),
                                 fromfile='before', tofile='after')
     for line in diff:
         print(line.rstrip())
Example #21
0
    def checkRules(self, source, key, ufwi_ruleset):
        # bart.xml => bart.iptables_ipv4
        correct = source[:-3] + key
        prefix = "%s:%s" % (basename(source)[:-4], key)

        if self.options.regenerate:
            if ufwi_ruleset:
                print "%s: file regenerated" % prefix
                with open(correct, "w") as fp:
                    for line in ufwi_ruleset:
                        print >>fp, line
            elif exists(correct):
                print "%s: remove file (no rule)" % prefix
                unlink(correct)
            return True

        text1 = ufwi_ruleset
        if exists(correct):
            with open(correct) as fp:
                text2 = [line.rstrip() for line in fp]
        else:
            text2 = tuple()
        diff = list(context_diff(text1, text2))
        if diff:
            print "%s: FAILURE!" % prefix
            for line in diff:
                print line.rstrip()
            return False
        print "%s: ok" % prefix
        return True
Example #22
0
def test_app_iptables_rules(SystemInfo, Command, Sudo):

    # Build a dict of variables to pass to jinja for iptables comparison
    kwargs = dict(
        mon_ip=os.environ.get('MON_IP', securedrop_test_vars.mon_ip),
        default_interface=Command.check_output("ip r | head -n 1 | "
                                               "awk '{ print $5 }'"),
        tor_user_id=Command.check_output("id -u debian-tor"),
        securedrop_user_id=Command.check_output("id -u www-data"),
        ssh_group_gid=Command.check_output("getent group ssh | cut -d: -f3"),
        dns_server=securedrop_test_vars.dns_server)

    # Build iptables scrape cmd, purge comments + counters
    iptables = "iptables-save | sed 's/ \[[0-9]*\:[0-9]*\]//g' | egrep -v '^#'"
    environment = os.environ.get("CI_SD_ENV", "staging")
    iptables_file = "{}/iptables-app-{}.j2".format(
                          os.path.dirname(os.path.abspath(__file__)),
                          environment)

    # template out a local iptables jinja file
    jinja_iptables = Template(open(iptables_file, 'r').read())
    iptables_expected = jinja_iptables.render(**kwargs)

    with Sudo():
        # Actually run the iptables scrape command
        iptables = Command.check_output(iptables)
        # print diff comparison (only shows up in pytests if test fails or
        # verbosity turned way up)
        for iptablesdiff in difflib.context_diff(iptables_expected.split('\n'),
                                                 iptables.split('\n')):
            print(iptablesdiff)
        # Conduct the string comparison of the expected and actual iptables
        # ruleset
        assert iptables_expected == iptables
Example #23
0
    def generateContextDiff(self):
        self.clear()

        a = self.snapShot.text().splitlines()
        b = self.editor.text().splitlines()

        QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
        lines = 0
        for line in difflib.context_diff(a, b, "Original", "Current"):
            if line.startswith('+'):
                styleType = 1
            elif line.startswith('-'):
                styleType = 2
            elif line.startswith('!'):
                styleType = 3
            elif line.startswith('*** '):
                styleType = 4
            else:
                styleType = 0
            self.appendText(line, styleType)
            lines += 1

        if lines == 0:
            self.appendText('Nothing has changed.', 0)

        QtGui.QApplication.restoreOverrideCursor()
Example #24
0
    def compareJava(self, oldPath, newPath, myClass, oldClass, newClass):
        self.reporter.info("Custom class is being used:" + myClass)
        self.reporter.info("Checking java for:" + oldClass + " " + newClass)
        oldSource = ""
        for c in self.locateClass(oldClass, oldPath):
            oldSource = c

        newSource = ""
        for c in self.locateClass(newClass, newPath):
            newSource = c

        if oldSource == "":
            self.reporter.error("Cannot find java for class:" + oldClass)
        if newSource == "":
            self.reporter.error("Cannot find java for class:" + newClass)
        try:
            if filecmp.cmp(oldSource, newSource):
                self.reporter.info("Same java:" + oldSource + " " + newSource)
            else:
                mySource = ""
                for c in self.locateClass(myClass, self.customPath):
                    mySource = c

                if mySource == "":
                    self.reporter.error("Cannot find java for class:" + myClass)
                self.reporter.actionRequired("Different java between versions", mySource, oldSource, newSource)
                fromfile = oldSource
                tofile = newSource
                with open(fromfile) as fromf, open(tofile) as tof:
                    fromlines, tolines = list(fromf), list(tof)
                diff = difflib.context_diff(fromlines, tolines, fromfile=oldSource, tofile=newSource)

                sys.stdout.writelines(diff)
        except OSError as ose:
            print(ose)
Example #25
0
 def testSimplify(self):
     inputFile = os.path.join(wikidataDir, "data/1000.json")
     linesOut  = open(os.path.join(wikidataDir, "test/1000.out")).read().split()
     linesNew  = subprocess.check_output([wdExtractPy, "-f", "-F", "-l", "en", "-s", "", "-t", "all", inputFile]).split()
     diff      = list(difflib.context_diff(linesOut, linesNew))
     self.assertEqual(len(diff), 0, "Unexpected differences in output of wd-extract.py -f -F -l en -s '' -t all data/1000.json\n"
                                    + "".join(diff))
def save_if_diffs(config_lines, old_file):
    """
    """
    do_write = False
    if os.path.exists(old_file):
        # read the existing
        bc_config_orig = open(old_file, "r")
        bc_config_lines_orig = bc_config_orig.readlines()
        bc_config_orig.close()
        for idx in range(len(bc_config_lines_orig)): bc_config_lines_orig[idx] = bc_config_lines_orig[idx].strip()

        print "Checking if different from previous file [%s]" % old_file
        different = False

        for line in difflib.context_diff(bc_config_lines_orig, bc_config_lines, fromfile=old_file, tofile='BC_config_new.csv'):
            print line
            different = True

        if different:
            print
            print
            print "Differences found => "
            print "  Moving previous file [%s] to [%s.bak]" % (old_file, old_file)
            shutil.move(old_file, "%s.bak" % old_file)
            do_write = True
    else:
        # if it doesn't exist, write this for sure
        do_write = True

    if do_write:
        print "  Writing new file %s" % old_file
        # write it to [project_folder]\BC_config_new.csv
        bc_config_new = open(old_file, "wb")
        for line in bc_config_lines: bc_config_new.write("%s\r\n" % line)
        bc_config_new.close()
Example #27
0
def do_unit_test(unit_test):
    host_srctext = get_contents("./"+unit_test+"/host_"+unit_test+".c")
    host_srctext = expand_pid_pattern(host_srctext)
    host_expected_outputs = re.findall(EXPECT_PATTERN, host_srctext)

    e_srctext = get_contents("./"+unit_test+"/e_"+unit_test+".c")
    e_srctext = expand_pid_pattern(e_srctext)
    e_expected_outputs = re.findall(EXPECT_PATTERN, e_srctext)

    expected_output = "\n".join(e_expected_outputs)
    if len(host_expected_outputs) != 0:
        expected_output += "\n" + "\n".join(host_expected_outputs);
        
    if not os.path.isfile("bin/host_"+unit_test):
        print("WARNING: bin/host_"+unit_test+" not found! Did you run make?")
        return False
    
    actual_output = run_unit_test(unit_test)

    succes = (actual_output.replace("\n","") == expected_output.replace("\n",""))
    
    if not succes:
        print('WARNING: Unit test "' + unit_test + '" failed with output:')
        if actual_output == "TIMEOUT":
            print("A timeout occurred")
        else:
            for line in difflib.context_diff(actual_output.split('\n'), expected_output.split('\n')):
                print(line)
    return succes
Example #28
0
    def generateContextDiff(self):
        self.clear()
        
        a = self.snapShot.text().splitlines()
        b = self.editor.text().splitlines()

        QtGui.QApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
        lines = 0
        for line in difflib.context_diff(a, b, "Deleted", "Added"):
            if line.startswith('+ '):
                styleType = 1
            elif line.startswith('- '):
                styleType = 2
            elif line.startswith('! '):
                styleType = 3
            elif (line.startswith('*** ') or line.startswith('--- ')) and lines > 1:
                styleType = 4
            else:
                styleType = 0
            self.appendText(line, styleType)
            lines += 1

        if lines == 0:
            self.appendText(self.trUtf8(
                'Nothing has changed.'), self.cNormalFormat)

        QtGui.QApplication.restoreOverrideCursor()
Example #29
0
def _content_diff(artifact_type=None, artifact_in_disk=None, artifact_in_db=None,
                  verbose=False):
    artifact_in_disk_str = json.dumps(
        artifact_in_disk.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    artifact_in_db_str = json.dumps(
        artifact_in_db.__json__(), sort_keys=True,
        indent=4, separators=(',', ': ')
    )
    diffs = difflib.context_diff(artifact_in_db_str.splitlines(),
                                 artifact_in_disk_str.splitlines(),
                                 fromfile='DB contents', tofile='Disk contents')
    printed = False
    for diff in diffs:
        if not printed:
            identifier = getattr(artifact_in_db, 'ref', getattr(artifact_in_db, 'name'))
            print('%s %s in db differs from what is in disk.' % (artifact_type.upper(),
                  identifier))
            printed = True
        print(diff)

    if verbose:
        print('\n\nOriginal contents:')
        print('===================\n')
        print('Artifact in db:\n\n%s\n\n' % artifact_in_db_str)
        print('Artifact in disk:\n\n%s\n\n' % artifact_in_disk_str)
Example #30
0
 def get_diff(self, s1, s2):
     diff = difflib.context_diff(s1, s2)
     str = ''
     for line in diff:
         str += line
         
     return str
Example #31
0
def main():

    usage = "usage: %prog [options] fromfile tofile"
    parser = optparse.OptionParser(usage)
    parser.add_option("-c",
                      action="store_true",
                      default=False,
                      help='Produce a context format diff (default)')
    parser.add_option("-u",
                      action="store_true",
                      default=False,
                      help='Produce a unified format diff')
    parser.add_option(
        "-m",
        action="store_true",
        default=False,
        help='Produce HTML side by side diff (can use -c and -l in conjunction)'
    )
    parser.add_option("-n",
                      action="store_true",
                      default=False,
                      help='Produce a ndiff format diff')
    parser.add_option("-l",
                      "--lines",
                      type="int",
                      default=3,
                      help='Set number of context lines (default 3)')
    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.print_help()
        sys.exit(1)
    if len(args) != 2:
        parser.error("need to specify both a fromfile and tofile")

    n = options.lines
    fromfile, tofile = args

    fromdate = time.ctime(os.stat(fromfile).st_mtime)
    todate = time.ctime(os.stat(tofile).st_mtime)
    fromlines = open(fromfile, 'U').readlines()
    tolines = open(tofile, 'U').readlines()

    if options.u:
        diff = difflib.unified_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)
    elif options.n:
        diff = difflib.ndiff(fromlines, tolines)
    elif options.m:
        diff = difflib.HtmlDiff().make_file(fromlines,
                                            tolines,
                                            fromfile,
                                            tofile,
                                            context=options.c,
                                            numlines=n)
    else:
        diff = difflib.context_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)

    sys.stdout.writelines(diff)
Example #32
0
    def handle(self, *args, **options):

        import os, random, string
        deploy_hash = ''.join(
            random.choice(string.lowercase) for x in range(32))
        host = None
        hostname = 'unknown'
        hosts = []
        if len(args):
            hostname = args[0]

        for HostSettingsClass in HostSettings.__subclasses__():
            name = HostSettingsClass.__name__.replace('HostSettings',
                                                      '').lower()
            hosts.append(name)
            if hostname == name:
                host = HostSettingsClass

        if host is None:
            print 'host should be one of:', hosts
            raise Exception("Unknown host %s" % (hostname, ))

        print colors.red("DEPLOYING TO: %s" % hostname, bold=True)
        print colors.red("DEPLOY HASH = %s" % deploy_hash, bold=True)

        #if
        env.host_string = host.HOST_STRING
        print colors.red("TEST COMMAND:", bold=True)
        run("ls")
        #run("source virtualenv/bin/activate");
        virtpath = host.PYTHON_PREFIX

        if options['initial'] and not files.exists(
                host.SRCROOT) and not files.exists(host.APPROOT):
            print colors.red(
                "initial=true and SRCROOT/APPROOT does not exist. will install now"
            )
            run('mkdir %s' % host.SRCROOT)
            run('mkdir %s' % host.APPROOT)
            run('mkdir %s/logs' % host.APPROOT)
            with cd(host.SRCROOT):
                run("git init")
                run("git remote add origin %s" % host.GIT_REMOTE)
                run("git pull origin master")
                run(host.PYTHON_INSTALL)
                run("%spip install -r requirements.txt" % virtpath)
                sql = "CREATE DATABASE %s; GRANT ALL ON %s.* TO %s@localhost IDENTIFIED BY '%s';" % (
                    host.DATABASE['NAME'], host.DATABASE['NAME'],
                    host.DATABASE['USER'], host.DATABASE['PASSWORD'])
                print colors.red(sql)
                #print sql
                run('echo "%s" | mysql --batch -u %s -p' % (sql, 'root'))
                run("%spython manage.py install" % virtpath)
            #return

        with cd(host.SRCROOT):
            if host.CLASS == 'PROD' and not options['nobackup']:
                print colors.red("PROD BACKUP:", bold=True)
                run("%spython manage.py backup" % (virtpath, ))

            print colors.red("REVERTING SETTINGS:", bold=True)
            run("git checkout -- %s/settings.py" %
                (django_settings.CADO_PROJECT))

            print colors.red("UPDATING CODEBASE:", bold=True)
            run("git pull origin %s" % options['branch'])
            run("git checkout %s" % options['branch'])

            print colors.red("INSTALLING REQUIREMENTS:", bold=True)
            run("%spip install -q -r requirements.txt" % virtpath)

            print colors.red("INSERTING HASH:", bold=True)
            run("sed 's/XXAUTODEPLOYHASHXX/%s/' %s/settings.py > %s/settings.tmp.py"
                % (deploy_hash, django_settings.CADO_PROJECT,
                   django_settings.CADO_PROJECT))
            run("mv %s/settings.tmp.py %s/settings.py" %
                (django_settings.CADO_PROJECT, django_settings.CADO_PROJECT))

            #sed 's/foo/bar/' mydump.sql > fixeddump.sql

            print colors.red("REGENERATIN CONFIG FILES:", bold=True)
            run("%spython manage.py regenerate_config" % virtpath)

            if django_settings.MULTISITE:
                for site in django_settings.SITES:
                    #run("mkdir config/solr/%s" % site.SOLR_CORE_NAME)
                    run("%spython manage.py build_solr_schema %s > config/solr/%s_schema.xml"
                        % (virtpath, site.CADO_PROJECT, site.SOLR_CORE_NAME))
            else:
                run("%spython manage.py build_solr_schema > config/solr/schema.xml"
                    % virtpath)

            for name, getter, setter, combined in host.CONFIGS:
                diff = False
                current = run(getter, quiet=True, warn_only=True).splitlines()
                current = [line + "\n" for line in current]
                new = run("cat config/" + name, quiet=True).splitlines()
                new = [line + "\n" for line in new]

                if combined:
                    combined = []
                    hash = hashlib.md5(host.APPROOT).hexdigest()
                    start_line = "##### CHUNK GENERATED BY CADOCMS %s PLEASE DON'T MODIFY #####\n" % hash
                    end_line = "##### END OF CHUNK GENERATED BY CADOCMS %s #####\n" % hash
                    if start_line not in current:
                        current.append(start_line)
                    if end_line not in current:
                        current.append(end_line)
                    in_chunk = False
                    for line in current:
                        if line == start_line:
                            in_chunk = True
                            combined.append(start_line)
                            combined = combined + new
                            combined.append(end_line)
                        if not in_chunk:
                            combined.append(line)
                        if line == end_line:
                            in_chunk = False
                    tf = tempfile.NamedTemporaryFile()
                    tfName = tf.name
                    tf.seek(0)
                    #print current, new, combined
                    for line in combined:
                        tf.write(line)
                    tf.flush()
                    put(tfName, 'config/%s.combined' % name)
                    new = combined
                    name = name + '.combined'

                for line in context_diff(current,
                                         new,
                                         fromfile='CURRENT',
                                         tofile='NEW'):
                    diff = True
                choice = 'd'
                if diff:
                    while choice == 'd':
                        choice = console.prompt(
                            '%s config file differs. [d]=show diff, [r]=replace, [i]=ignore'
                            % (name, ),
                            default='i',
                            validate='d|r|i')
                        if (choice == 'd'):
                            for line in context_diff(current,
                                                     new,
                                                     fromfile='CURRENT',
                                                     tofile='NEW'):
                                sys.stdout.write(line)
                        if (choice == 'r'):
                            run("cat config/" + name + " " + setter)

            for site in django_settings.SITES:
                print colors.red("INSTALLING SITE %s:" % site.CADO_PROJECT,
                                 bold=True)
                arguments = ''
                if django_settings.MULTISITE:
                    arguments = site.CADO_PROJECT
                #run("git submodule init")
                #run("git submodule update")
                run("%spython manage.py syncdb %s" % (virtpath, arguments))
                run("%spython manage.py migrate %s" %
                    (virtpath, arguments))  #--traceback
                run("%spython manage.py collectstatic %s --noinput" %
                    (virtpath, arguments))
                run("%spython manage.py restyle_tinymce %s" %
                    (virtpath, arguments))

                print colors.yellow("RESTARTING FASTCGI:", bold=True)

                with settings(warn_only=True):
                    run("kill -9 `cat %s/%s.pid`" %
                        (host.APPROOT, site.CADO_PROJECT))

                run("find . -name '*.pyc' -delete")

                maxchildren = 3
                if host.CLASS == 'TEST':
                    maxchildren = 1

                run("%spython manage.py runfcgi %s method=threaded maxchildren=%d socket=%s/%s.sock pidfile=%s/%s.pid"
                    % (virtpath, site.CADO_PROJECT, maxchildren, host.APPROOT,
                       site.CADO_PROJECT, host.APPROOT, site.CADO_PROJECT))
                #run("sleep 3")
                print colors.yellow("NOT CLEARING CACHE:)", bold=True)
                #run("%spython manage.py clear_cache" % (virtpath,))
                #run("chmod 766 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT))
                run("chmod 777 %s/%s.sock" % (host.APPROOT, site.CADO_PROJECT))
                run("%spython manage.py warmup %s" % (virtpath, arguments))
                print colors.green("DONE!", bold=True)
Example #33
0
def bctest(testDir, testObj, buildenv):
    """Runs a single test, comparing output and RC to expected output and RC.

    Raises an error if input can't be read, executable fails, or output/RC
    are not as expected. Error is caught by bctester() and reported.
    """
    # Get the exec names and arguments
    execprog = os.path.join(buildenv["BUILDDIR"], "src", testObj["exec"] + buildenv["EXEEXT"])
    execargs = testObj['args']
    execrun = [execprog] + execargs

    # Read the input data (if there is any)
    stdinCfg = None
    inputData = None
    if "input" in testObj:
        filename = os.path.join(testDir, testObj["input"])
        inputData = open(filename, encoding="utf8").read()
        stdinCfg = subprocess.PIPE

    # Read the expected output data (if there is any)
    outputFn = None
    outputData = None
    outputType = None
    if "output_cmp" in testObj:
        outputFn = testObj['output_cmp']
        outputType = os.path.splitext(outputFn)[1][1:]  # output type from file extension (determines how to compare)
        try:
            outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read()
        except:
            logging.error("Output file " + outputFn + " can not be opened")
            raise
        if not outputData:
            logging.error("Output data missing for " + outputFn)
            raise Exception
        if not outputType:
            logging.error("Output file %s does not have a file extension" % outputFn)
            raise Exception

    # Run the test
    proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
    try:
        outs = proc.communicate(input=inputData)
    except OSError:
        logging.error("OSError, Failed to execute " + execprog)
        raise

    if outputData:
        data_mismatch, formatting_mismatch = False, False
        # Parse command output and expected output
        try:
            a_parsed = parse_output(outs[0], outputType)
        except Exception as e:
            logging.error('Error parsing command output as %s: %s' % (outputType, e))
            raise
        try:
            b_parsed = parse_output(outputData, outputType)
        except Exception as e:
            logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e))
            raise
        # Compare data
        if a_parsed != b_parsed:
            logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
            data_mismatch = True
        # Compare formatting
        if outs[0] != outputData:
            error_message = "Output formatting mismatch for " + outputFn + ":\n"
            error_message += "".join(difflib.context_diff(outputData.splitlines(True),
                                                          outs[0].splitlines(True),
                                                          fromfile=outputFn,
                                                          tofile="returned"))
            logging.error(error_message)
            formatting_mismatch = True

        assert not data_mismatch and not formatting_mismatch

    # Compare the return code to the expected return code
    wantRC = 0
    if "return_code" in testObj:
        wantRC = testObj['return_code']
    if proc.returncode != wantRC:
        logging.error("Return code mismatch for " + outputFn)
        raise Exception

    if "error_txt" in testObj:
        want_error = testObj["error_txt"]
        # Compare error text
        # TODO: ideally, we'd compare the strings exactly and also assert
        # That stderr is empty if no errors are expected. However, vircles-tx
        # emits DISPLAY errors when running as a windows application on
        # linux through wine. Just assert that the expected error text appears
        # somewhere in stderr.
        if want_error not in outs[1]:
            logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip())
            raise Exception
def diff_dumps(initial_dump, migrated_dump):
    """ Perform a diff of two database dumps """
    file_initial = open(initial_dump)
    file_migrated = open(migrated_dump)
    return difflib.context_diff(file_initial.readlines(),
                                file_migrated.readlines())
Example #35
0
def diff_bsps(bsp1, bsp2, full=False) -> str:
    """WARNING: full diffs can be incredibly large!"""
    out = []
    if bsp1.folder == bsp2.folder:
        out.append(f"Comparing {bsp1} -> {bsp2}...")
    else:
        out.append(
            f"Comparing {bsp1.folder}/{bsp1} -> {bsp2.folder}/{bsp2}...")
    # NOTE: comparing lumps by index, same number of lumps expected
    for lump1, lump2 in zip(bsp1.branch.LUMP, bsp2.branch.LUMP):
        lump1 = lump1.name
        lump2 = lump2.name
        # diff headers
        if lump1 not in bsp1.headers or lump2 not in bsp2.headers:
            continue  # lazy fix for rbsp externals
            # TODO: note absent headers (not just for respawn.ExternalLumpManager!)
        bsp1_header = bsp1.headers[lump1]
        bsp2_header = bsp2.headers[lump2]
        lump_name = lump1 if lump1 == lump2 else f"{lump1} -> {lump2}"
        # NOTE: fourCC (decompressed size) vs length is not calculated
        # -- in fact, no check to check opposing compressed state (one compressed, one uncompressed)
        # -- however, LZMA compressed lump contents are always decompressed before comparison
        header_diff = "".join([
            "Y" if bsp1_header.offset == bsp2_header.offset else "N",
            "Y" if bsp1_header.length == bsp2_header.length else "N",
            "Y" if bsp1_header.version == bsp2_header.version else "N",
            "Y" if bsp1_header.fourCC == bsp2_header.fourCC else "N"
        ])
        # diff lump contents
        try:
            lump_1_contents = bsp1.lump_as_bytes(lump1)
            lump_2_contents = bsp2.lump_as_bytes(lump2)
        except Exception as exc:
            out.append(f"{lump_name}  {header_diff} ???? {exc}")
            continue  # skip this lump
        lumps_match = bool(lump_1_contents == lump_2_contents)
        contents_diff = "YES!" if lumps_match else "NOPE"
        out.append(f"{lump_name}  {header_diff} {contents_diff}")
        # was a lump removed / added?
        if (len(lump_1_contents) == 0
                or len(lump_2_contents) == 0) and not lumps_match:
            out.append(" ".join([
                "+" if hasattr(bsp1, lump1) else "-",
                f"{bsp1.filename}.{lump1}"
            ]))
            out.append(" ".join([
                "+" if hasattr(bsp2, lump2) else "-",
                f"{bsp2.filename}.{lump2}"
            ]))
        # detailed comparisons
        elif full:
            if not lumps_match:
                # TODO: measure the scale of the differences
                if lump1 in bsp1.branch.LUMP_CLASSES and lump2 in bsp2.branch.LUMP_CLASSES:
                    diff = difflib.unified_diff(
                        [lc.__repr__() for lc in getattr(bsp1, lump1)],
                        [lc.__repr__() for lc in getattr(bsp2, lump2)],
                        f"{bsp1.filename}.{lump1}", f"{bsp1.filename}.{lump1}")
                    out.extend(diff)
                # SPECIAL_LUMP_CLASSES
                elif all([ln == "ENTITIES" for ln in (lump1, lump2)]):
                    out.append(diff_entities(bsp1.ENTITIES, bsp2.ENTITIES))
                elif all([ln == "PAKFILE" for ln in (lump1, lump2)]):
                    # NOTE: this will fail on nexon.cso2 bsps, as their pakfiles are unmapped
                    out.append(diff_pakfiles(bsp1.PAKFILE, bsp2.PAKFILE))
                # TODO: GAME_LUMP diff model_names
                else:  # BASIC_LUMP_CLASSES / general raw bytes
                    # NOTE: xxd line numbers prevent accurately tracing insertions
                    diff = difflib.context_diff(
                        [*xxd(lump_1_contents, 32)
                         ],  # 32 bytes per line looks nice
                        [*xxd(lump_2_contents, 32)],
                        f"{bsp1.filename}.{lump1}",
                        f"{bsp2.filename}.{lump2}")
                    # TODO: run xxd without creating line numbers
                    # -- then, generate line numbers from diff & update diff with these line numbers
                    out.extend(diff)
        else:
            out.extend([str(bsp1_header), str(bsp2_header)])
    return "\n".join(out)
Example #36
0
 _file = file.replace("\\", "/")
 if _file not in files:
     print("Missing:         %s" % file)
     todo = True
     continue
 tfile = os.path.join(tdir, file)
 if os.path.isdir(file):
     continue
 targ_diff = 0   # no difference
 if not cmp(tfile, file, shallow=False):
     if ".gpr.py" in file:
         with open(file) as sfil:
             with open(tfile) as tfil:
                 diff = list(
                     difflib.context_diff(sfil.readlines(),
                                          tfil.readlines(),
                                          n=0))
                 for line in diff:
                     if 'gramps_target_version' in line:
                         print("gpr differs:     %s %s" %
                               (addon, line), end='')
                         targ_diff = 1  # Potential problem
                         continue
                     if(line.startswith('---') or
                        line.startswith('***') or
                        'version' in line.lower()):
                         continue
                     targ_diff = 2  # just different
     else:
         targ_diff = 2  # just different
 if targ_diff == 0:
Example #37
0
def process_files(repair_files, check_all_files, display_diff,
                  colorize_output):
    if check_all_files:
        process = subprocess.run(['git', 'ls-files', '-z'],
                                 stdout=subprocess.PIPE,
                                 universal_newlines=True)
    else:
        process = subprocess.run([
            'git', 'diff-tree', '--no-commit-id', '--name-only', '-r', 'HEAD',
            '-z'
        ],
                                 stdout=subprocess.PIPE,
                                 universal_newlines=True)
    wrong_license_files = []
    wrong_extensions = [
        '.map', '.txt', '.gitignore', '.clang-format', '.clang-format-ignore',
        '.gitmodules', '.md', '.json', '.xml', '.log', '.csv', '.html', '.css',
        '.pdf', '.rtf'
    ]
    tmp_files = set()
    for file in process.stdout.split('\0'):
        authors = set()
        wrong_ext = False
        for we in wrong_extensions:
            if file.endswith(we):
                wrong_ext = True
        if (not os.path.isfile(file)) or wrong_ext:
            continue
        process = subprocess.run([
            'git', 'log', '--format=%H Author: %aN <%aE>', '--follow', '--',
            file
        ],
                                 stdout=subprocess.PIPE,
                                 universal_newlines=True)
        author_lines = process.stdout.split('\n')
        for author_line in author_lines:
            commit_hash = author_line.split(' ')[0]
            author_line = author_line.lstrip(commit_hash + ' ')
            if author_line.startswith('Author: '):
                author = author_line.lstrip('Author: ')
                process = subprocess.run([
                    'git', 'diff-tree', '--no-commit-id', '--name-only', '-r',
                    '--exit-code', commit_hash, '--', file
                ],
                                         stdout=subprocess.PIPE,
                                         universal_newlines=True)
                # git returns non zero exit code if file was changed in this commit
                if process.returncode != 0 and not '<not.committed.yet>' in author:
                    authors.add(author)
        # find commit date
        process = subprocess.run(['git', 'show', '-s', '--format=%ai', 'HEAD'],
                                 stdout=subprocess.PIPE,
                                 universal_newlines=True)
        commit_year = process.stdout[:4]
        process = subprocess.run(
            ['git', 'rev-list', '--max-parents=0', 'HEAD'],
            stdout=subprocess.PIPE,
            universal_newlines=True)
        first_commit_id = process.stdout.split('\n')[0]
        process = subprocess.run(
            ['git', 'show', '-s', '--format=%ai', first_commit_id],
            stdout=subprocess.PIPE,
            universal_newlines=True)
        first_commit_year = process.stdout[:4]

        if file.endswith('.c') or file.endswith('.h'):
            license_type = 'c_style'
        else:
            license_type = 'hash'
        license = generate_license(authors, commit_year, first_commit_year,
                                   license_type)

        with open(file) as f:
            lines = f.readlines()
            shebang = ''
            if file.endswith('.sh') or file.endswith(
                    '.py') and lines[0].startswith('#!'):  #files with shebang
                shebang = lines[0]
                lines = lines[1:]
            license_len = len(license)
            file_len = len(lines)
            if license_len > file_len:
                print("license length missmatch, want >= {}, got {}".format(
                    license_len, file_len))
            COPYRIGHT_LINE_REG = r'^[#| ]*Copyright \\(c\\) [0-9]{4}.*All rights reserved\.'
            comp = re.compile(COPYRIGHT_LINE_REG)
            copyright_pos = 0
            if license_type == 'c_style':
                copyright_pos += 1

            if comp.match(lines[copyright_pos]):
                print("file without license: {}".format(file))
                wrong_license_files.append(file)
            else:
                for it in range(0, license_len):
                    if lines[it] != license[it]:
                        wrong_license_files.append(file)
                        break
            if file in wrong_license_files:
                end_point = 0
                if license_type == 'c_style':
                    if lines[0].startswith('/*'):
                        for it in range(0, len(lines)):
                            if lines[it].endswith('*/\n'):
                                end_point = it + 1
                                break
                else:
                    for it in range(0, len(lines)):
                        if not (lines[it].isspace()
                                or lines[it].startswith('#')):
                            end_point = it
                            break
                tmp_output = file + '.tmp'
                with open(tmp_output, 'w') as fw:
                    if shebang != '':
                        fw.write(shebang)
                    for line in license:
                        fw.write(line)
                    for it in range(end_point, len(lines)):
                        fw.write(lines[it])
                    tmp_files.add(tmp_output)
                    diff = list(
                        difflib.context_diff(lines[:end_point],
                                             license,
                                             fromfile=file,
                                             tofile=tmp_output))
                    # do not count files with more authors as file with wrong license
                    # it may have more authors because file could be e.g. renamed, merged etc
                    only_more_authors = True
                    CHANGED_LINE_RE = r'^[-+?!] .*'
                    AUTHOR_LINE_RE = r'- # Author: .*<.*>.*$'
                    comp_changed = re.compile(CHANGED_LINE_RE)
                    comp_author = re.compile(AUTHOR_LINE_RE)
                    diff_len = len(
                        diff) - 1  # last line contains diff lines numbers
                    for it in range(0, diff_len):
                        if comp_changed.match(diff[it]) and (
                                not comp_author.match(diff[it])):
                            # does not treat last empty line as error
                            if it == diff_len - 1 and (len(diff[it]) <= 2
                                                       or diff[it] == '- \n'):
                                break
                            only_more_authors = False
                            break
                    if only_more_authors:
                        wrong_license_files.remove(file)
                    else:
                        if display_diff:
                            print_diff(diff, colorize_output)
                        if repair_files:
                            os.rename(tmp_output, file)

    got_error = False
    for wlf in wrong_license_files:
        print("File with wrong license: {}".format(wlf))
        got_error = True
    # when repairing files temporary files are moved to original ones
    if not repair_files:
        for tf in tmp_files:
            os.remove(tf)
    if got_error:
        exit(1)
Example #38
0
def query_interactive(src_fn, dest_fn, src_content, dest_content,
                      simulate, out_=sys.stdout):
    def out(msg):
        out_.write(msg)
        out_.write('\n')
        out_.flush()
    global all_answer
    from difflib import unified_diff, context_diff
    u_diff = list(unified_diff(
        dest_content.splitlines(),
        src_content.splitlines(),
        dest_fn, src_fn))
    c_diff = list(context_diff(
        dest_content.splitlines(),
        src_content.splitlines(),
        dest_fn, src_fn))
    added = len([l for l in u_diff if l.startswith('+')
                   and not l.startswith('+++')])
    removed = len([l for l in u_diff if l.startswith('-')
                   and not l.startswith('---')])
    if added > removed:
        msg = '; %i lines added' % (added-removed)
    elif removed > added:
        msg = '; %i lines removed' % (removed-added)
    else:
        msg = ''
    out('Replace %i bytes with %i bytes (%i/%i lines changed%s)' % (
        len(dest_content), len(src_content),
        removed, len(dest_content.splitlines()), msg))
    prompt = 'Overwrite %s [y/n/d/B/?] ' % dest_fn
    while 1:
        if all_answer is None:
            response = input_(prompt).strip().lower()
        else:
            response = all_answer
        if not response or response[0] == 'b':
            import shutil
            new_dest_fn = dest_fn + '.bak'
            n = 0
            while os.path.exists(new_dest_fn):
                n += 1
                new_dest_fn = dest_fn + '.bak' + str(n)
            out('Backing up %s to %s' % (dest_fn, new_dest_fn))
            if not simulate:
                shutil.copyfile(dest_fn, new_dest_fn)
            return True
        elif response.startswith('all '):
            rest = response[4:].strip()
            if not rest or rest[0] not in ('y', 'n', 'b'):
                out(query_usage)
                continue
            response = all_answer = rest[0]
        if response[0] == 'y':
            return True
        elif response[0] == 'n':
            return False
        elif response == 'dc':
            out('\n'.join(c_diff))
        elif response[0] == 'd':
            out('\n'.join(u_diff))
        else:
            out(query_usage)
def main():

    parser = argparse.ArgumentParser()

    parser.add_argument('-c',
                        action='store_true',
                        default=False,
                        help='Produce a context format diff (default)')

    parser.add_argument('-u',
                        action='store_true',
                        default=False,
                        help='Produce a unified format diff')

    parser.add_argument('-m',
                        action='store_true',
                        default=False,
                        help='Produce HTML side by side diff '
                        '(can use -c and -l in conjunction)')

    parser.add_argument('-n',
                        action='store_true',
                        default=False,
                        help='Produce a ndiff format diff')

    parser.add_argument('-l',
                        '--lines',
                        type=int,
                        default=3,
                        help='Set number of context lines (default 3)')

    parser.add_argument('fromfile')

    parser.add_argument('tofile')

    options = parser.parse_args()

    n = options.lines
    fromfile = options.fromfile

    tofile = options.tofile

    fromdate = file_mtime(fromfile)
    todate = file_mtime(tofile)

    with open(fromfile) as ff:
        fromlines = ff.readlines()

    with open(tofile) as tf:
        tolines = tf.readlines()

    if options.u:
        diff = difflib.unified_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)

    elif options.n:
        diff = difflib.ndiff(fromlines, tolines)

    elif options.m:
        diff = difflib.HtmlDiff().make_file(fromlines,
                                            tolines,
                                            fromfile,
                                            tofile,
                                            context=options.c,
                                            numlines=n)

    else:
        diff = difflib.context_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)

    sys.stdout.writelines(diff)
Example #40
0
    def output_difference(self, runstate=None, colored=True):
        """
        Return a string describing the differences between the expected output
        for a given example (`example`) and the actual output (`got`).
        The `runstate` contains option flags used to compare `want` and `got`.

        Notes:
            This does not check if got matches want, it only outputs the raw
            differences. Got/Want normalization may make the differences appear
            more exagerated than they are.
        """
        got = self.got
        want = self.want

        if runstate is None:
            runstate = directive.RuntimeState()

        # Don't normalize because it usually removes the newlines
        runstate_ = runstate.to_dict()

        # Don't normalize whitespaces in report for better visibility
        runstate_['NORMALIZE_WHITESPACE'] = False
        runstate_['IGNORE_WHITESPACE'] = False
        got, want = normalize(got, want, runstate_)

        # If <BLANKLINE>s are being used, then replace blank lines
        # with <BLANKLINE> in the actual output string.
        # if not runstate['DONT_ACCEPT_BLANKLINE']:
        #     got = re.sub('(?m)^[ ]*(?=\n)', BLANKLINE_MARKER, got)

        got = utils.ensure_unicode(got)

        # Check if we should use diff.
        if self._do_a_fancy_diff(runstate):
            # Split want & got into lines.
            want_lines = want.splitlines(True)
            got_lines = got.splitlines(True)
            # Use difflib to find their differences.
            if runstate['REPORT_UDIFF']:
                diff = difflib.unified_diff(want_lines, got_lines, n=2)
                diff = list(diff)[2:]  # strip the diff header
                kind = 'unified diff with -expected +actual'
            elif runstate['REPORT_CDIFF']:
                diff = difflib.context_diff(want_lines, got_lines, n=2)
                diff = list(diff)[2:]  # strip the diff header
                kind = 'context diff with expected followed by actual'
            elif runstate['REPORT_NDIFF']:
                # TODO: Is there a way to make Differ ignore whitespace if that
                # runtime directive is specified?
                engine = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
                diff = list(engine.compare(want_lines, got_lines))
                kind = 'ndiff with -expected +actual'
            else:
                raise ValueError('Invalid difflib option')

            # Remove trailing whitespace on diff output.
            diff = [line.rstrip() + '\n' for line in diff]
            diff_text = ''.join(diff)
            if colored:
                diff_text = utils.highlight_code(diff_text, lexer_name='diff')

            text = 'Differences (%s):\n' % kind + utils.indent(diff_text)
        else:
            # If we're not using diff, then simply list the expected
            # output followed by the actual output.
            if want and got:
                if colored:
                    got = utils.color_text(got, 'red')
                    want = utils.color_text(want, 'red')
                text = 'Expected:\n%s\nGot:\n%s' % (utils.indent(want),
                                                    utils.indent(got))
            elif want:
                if colored:
                    got = utils.color_text(got, 'red')
                    want = utils.color_text(want, 'red')
                text = 'Expected:\n%s\nGot nothing\n' % utils.indent(want)
            elif got:  # nocover
                raise AssertionError('impossible state')
                text = 'Expected nothing\nGot:\n%s' % utils.indent(got)
            else:  # nocover
                raise AssertionError('impossible state')
                text = 'Expected nothing\nGot nothing\n'
        return text
Example #41
0
#!/usr/bin/env python3
# -*- coding: UTF-8 -*-

import difflib
from difflib_data import *

diff = difflib.context_diff(text1_lines, text2_lines, lineterm='')
print '\n'.join(list(diff))
Example #42
0
def erdiff(expect, result):
    ''' Expects large strings that have newlines in both expect and result '''
    cd = context_diff(expect.splitlines(True), result.splitlines(True))
    for line in cd:
        sys.stdout.write(line)
    assert expect == result
Example #43
0
    fp = FilePath(path)
    children = [x for x in fp.walk() if x.splitext()[1] == ".py"]

    with click.progressbar(children) as c:
        for f in c:

            inp = f.getContent().decode('utf8')
            out = process_from_epydoc_to_sphinx(f, inp)

            f.setContent(u'\n'.join(out).encode('utf8'))


if __name__ == "__main__":
    run()

    from difflib import context_diff
    import sys
    from transdocrify.tests import in_epydoc, out_rst

    with open(in_epydoc.__file__.replace(".pyc", ".py"), 'r') as f:
        inp = str(f.read())

    with open(out_rst.__file__.replace(".pyc", ".py"), 'r') as f:
        outp = f.readlines()

    out3 = process_from_epydoc_to_sphinx(inp)

    out2 = list(map(lambda x: x + "\n", out3))[:-1]

    print(''.join(context_diff(outp, out2)))
Example #44
0
import difflib
from pprint import pprint
import nltk

text1 = "text1.txt"
text2 = "text2.txt"

html = difflib.HtmlDiff()

diff = difflib.Differ()

f = open("result.html", "w+")

# use nltk to tokenize text into word list first before the comparison
content = html.make_file(nltk.word_tokenize(open(text1, "r").read()),
                         nltk.word_tokenize(open(text2, "r").read()),
                         context=True)

f.write(content)

f.close()

# use nltk to tokenize text into word list first before the comparison
pprint(
    list(
        difflib.context_diff(nltk.word_tokenize(open(text1, "r").read()),
                             nltk.word_tokenize(open(text2, "r").read()),
                             fromfile='text1.txt',
                             tofile='text2.txt')))
Example #45
0
def difflib_contextDiff():
    diff = difflib.context_diff(text1_lines, text2_lines, lineterm='')
    print('\n'.join(diff))
Example #46
0
def main():

    args = sys.argv

    if ("--help" in args) or (len(sys.argv) < 3):
        if len(sys.argv) < 3:
            print("ERROR: analyze_mutants requires at least two arguments\n")
        print(
            "USAGE: analyze_mutants <sourcefile> <cmd> [--mutantDir <dir>] [--fromFile <mutantfile>]"
        )
        print(
            "       <cmd> is command to execute to run tests; non-zero return indicates mutant killed"
        )
        print(
            "       --mutantDir: directory with all mutants; defaults to current directory"
        )
        print(
            "       --fromFile: file containing list of mutants to process; others ignored"
        )
        print("       --timeout <val>: change the timeout setting")
        print("       --show: show mutants")
        print("       --verbose: show mutants and output of analysis")
        print("       --seed: random seed for shuffling of mutants")
        print("       --noShuffle: do not randomize order of mutants")
        print(
            "       --resume: use existing killed.txt and notkilled.txt, resume mutation analysis"
        )
        print("       --prefix: add a prefix to killed.txt and notkilled.txt")
        sys.exit(0)

    verbose = "--verbose" in sys.argv
    if verbose:
        args.remove("--verbose")

    showM = "--show" in sys.argv
    if showM:
        args.remove("--show")

    resume = "--resume" in sys.argv
    if resume:
        args.remove("--resume")

    noShuffle = "--noShuffle" in sys.argv
    if noShuffle:
        args.remove("--noShuffle")

    prefix = None
    try:
        prefixpos = args.index("--prefix")
    except ValueError:
        prefixpos = -1

    if prefixpos != -1:
        prefix = args[prefixpos + 1]
        args.remove("--prefix")
        args.remove(prefix)

    fromFile = None
    try:
        filepos = args.index("--fromFile")
    except ValueError:
        filepos = -1

    if filepos != -1:
        fromFile = args[filepos + 1]
        args.remove("--fromFile")
        args.remove(fromFile)

    seed = None
    try:
        seedpos = args.index("--seed")
    except ValueError:
        seedpos = -1

    if seedpos != -1:
        seed = args[seedpos + 1]
        args.remove("--seed")
        args.remove(seed)
        seed = int(seed)

    timeout = 30
    try:
        topos = args.index("--timeout")
    except ValueError:
        topos = -1

    if topos != -1:
        timeout = args[topos + 1]
        args.remove("--timeout")
        args.remove(timeout)
        timeout = float(timeout)

    onlyMutants = None
    if fromFile is not None:
        with open(fromFile, 'r') as file:
            onlyMutants = file.read().split()

    mdir = "."
    try:
        mdirpos = args.index("--mutantDir")
    except ValueError:
        mdirpos = -1

    if mdirpos != -1:
        mdir = args[mdirpos + 1]
        args.remove("--mutantDir")
        args.remove(mdir)
    if mdir[-1] != "/":
        mdir += "/"

    src = args[1]
    tstCmd = [args[2]]
    ignore = []
    if len(args) > 3:
        with open(sys.argv[3]) as file:
            for l in file:
                ignore.append(l.split()[0])

    srcBase = src.split("/")[-1]
    srcEnd = "." + ((src.split(".")[-1]).split("/")[-1])

    count = 0.0
    killCount = 0.0

    killFileName = "killed.txt"
    notkillFileName = "notkilled.txt"
    if prefix is not None:
        killFileName = prefix + "." + killFileName
        notkillFileName = prefix + "." + notkillFileName

    print("ANALYZING", src)
    print("COMMAND: **", tstCmd, "**")

    allTheMutants = glob.glob(mdir +
                              srcBase.replace(srcEnd, ".mutant*" + srcEnd))

    if onlyMutants is not None:
        newMutants = []
        for f1 in onlyMutants:
            for f2 in allTheMutants:
                if f2.split("/")[-1] == f1:
                    newMutants.append(f2)
        allTheMutants = newMutants

    if seed is not None:
        random.seed(seed)

    if not noShuffle:
        random.shuffle(allTheMutants)

    allStart = time.time()

    if resume:
        alreadyKilled = []
        alreadyNotKilled = []
        if not (os.path.exists(killFileName)
                and os.path.exists(notkillFileName)):
            print("ATTEMPTING TO RESUME, BUT NO PREVIOUS RESULTS FOUND")
        else:
            with open(killFileName, 'r') as killed:
                with open(notkillFileName, 'r') as notkilled:
                    for line in killed:
                        if line == "\n":
                            continue
                        alreadyKilled.append(line[:-1])
                        count += 1
                        killCount += 1
                    for line in notkilled:
                        if line == "\n":
                            continue
                        alreadyNotKilled.append(line[:-1])
                        count += 1
            print("RESUMING FROM EXISTING RUN, WITH", int(killCount),
                  "KILLED MUTANTS OUT OF", int(count))

    with open(os.devnull, 'w') as dnull:
        with open(killFileName, 'w') as killed:
            with open(notkillFileName, 'w') as notkilled:
                if resume:
                    for line in alreadyKilled:
                        killed.write(line + "\n")
                        killed.flush()
                    for line in alreadyNotKilled:
                        notkilled.write(line + "\n")
                        notkilled.flush()
                for f in allTheMutants:
                    if resume:
                        if (f.split("/")[-1]
                                in alreadyKilled) or (f.split("/")[-1]
                                                      in alreadyNotKilled):
                            continue
                    if f in ignore:
                        print(f, "SKIPPED")
                    print("=" * 80)
                    print("#" + str(int(count) + 1) + ":", end=" ")
                    print("[" + str(round(time.time() - allStart, 2)) + "s",
                          end=" ")
                    print(
                        str(round(count / len(allTheMutants) * 100.0, 2)) +
                        "% DONE]")
                    if verbose or showM:
                        print("MUTANT:", f)
                        with open(src, 'r') as ff:
                            fromLines = ff.readlines()
                        with open(f, 'r') as tf:
                            toLines = tf.readlines()
                        diff = difflib.context_diff(fromLines, toLines,
                                                    "Original", "Mutant")
                        print(''.join(diff))
                        print()
                        sys.stdout.flush()
                    print("RUNNING", f + "...")
                    sys.stdout.flush()
                    try:
                        shutil.copy(src, src + ".um.backup")
                        shutil.copy(f, src)
                        if srcEnd == ".py":
                            py_compile.compile(src)

                        start = time.time()

                        if not verbose:
                            P = subprocess.Popen(tstCmd,
                                                 shell=True,
                                                 stderr=dnull,
                                                 stdout=dnull,
                                                 preexec_fn=os.setsid)
                        else:
                            P = subprocess.Popen(tstCmd,
                                                 shell=True,
                                                 preexec_fn=os.setsid)

                        try:
                            while P.poll() is None and (time.time() -
                                                        start) < timeout:
                                time.sleep(0.05)
                        finally:
                            if P.poll() is None:
                                print()
                                print(
                                    "HAD TO TERMINATE ANALYSIS (TIMEOUT OR EXCEPTION)"
                                )
                                os.killpg(os.getpgid(P.pid), signal.SIGTERM)
                                # Avoid any weird race conditions from grabbing the return code
                                time.sleep(0.05)
                            r = P.returncode

                        runtime = time.time() - start

                        count += 1
                        if r == 0:
                            print(f, "NOT KILLED")
                            notkilled.write(f.split("/")[-1] + "\n")
                            notkilled.flush()
                        else:
                            killCount += 1
                            print(f, "KILLED IN", runtime, "(RETURN CODE",
                                  str(r) + ")")
                            killed.write(f.split("/")[-1] + "\n")
                            killed.flush()
                        print("  RUNNING SCORE:", killCount / count)
                        sys.stdout.flush()
                    finally:
                        shutil.copy(src + ".um.backup", src)
                        os.remove(src + ".um.backup")
    print("=" * 80)
    print("MUTATION SCORE:", killCount / count)
Example #47
0
def bctest(testDir, testObj, exeext):
    """Runs a single test, comparing output and RC to expected output and RC.

    Raises an error if input can't be read, executable fails, or output/RC
    are not as expected. Error is caught by bctester() and reported.
    """
    # Get the exec names and arguments
    execprog = testObj['exec'] + exeext
    execargs = testObj['args']
    execrun = [execprog] + execargs

    # Read the input data (if there is any)
    stdinCfg = None
    inputData = None
    if "input" in testObj:
        filename = testDir + "/" + testObj['input']
        inputData = open(filename).read()
        stdinCfg = subprocess.PIPE

    # Read the expected output data (if there is any)
    outputFn = None
    outputData = None
    if "output_cmp" in testObj:
        outputFn = testObj['output_cmp']
        outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare)
        try:
            outputData = open(testDir + "/" + outputFn).read()
        except:
            logging.error("Output file " + outputFn + " can not be opened")
            raise
        if not outputData:
            logging.error("Output data missing for " + outputFn)
            raise Exception

    # Run the test
    proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE,universal_newlines=True)
    try:
        outs = proc.communicate(input=inputData)
    except OSError:
        logging.error("OSError, Failed to execute " + execprog)
        raise

    if outputData:
        data_mismatch, formatting_mismatch = False, False
        # Parse command output and expected output
        try:
            a_parsed = parse_output(outs[0], outputType)
        except Exception as e:
            logging.error('Error parsing command output as %s: %s' % (outputType,e))
            raise
        try:
            b_parsed = parse_output(outputData, outputType)
        except Exception as e:
            logging.error('Error parsing expected output %s as %s: %s' % (outputFn,outputType,e))
            raise
        # Compare data
        if a_parsed != b_parsed:
            logging.error("Output data mismatch for " + outputFn + " (format " + outputType + ")")
            data_mismatch = True
        # Compare formatting
        if outs[0] != outputData:
            error_message = "Output formatting mismatch for " + outputFn + ":\n"
            error_message += "".join(difflib.context_diff(outputData.splitlines(True),
                                                          outs[0].splitlines(True),
                                                          fromfile=outputFn,
                                                          tofile="returned"))
            logging.error(error_message)
            formatting_mismatch = True

        assert not data_mismatch and not formatting_mismatch

    # Compare the return code to the expected return code
    wantRC = 0
    if "return_code" in testObj:
        wantRC = testObj['return_code']
    if proc.returncode != wantRC:
        logging.error("Return code mismatch for " + outputFn)
        raise Exception
Example #48
0
def get_test_files(maxn=10,
                   input_template="input%i.txt",
                   expected_template="expected%i.txt"):
    for i in range(maxn):
        input_filename = input_template % i
        output_filename = expected_template % i
        if os.path.exists(input_filename) and os.path.exists(output_filename):
            yield i, input_filename, output_filename


if __name__ == "__main__":
    test_files = list(get_test_files())
    if test_files:
        # If local files exist, run, and then diff actual vs expected.
        for i, input_filename, expected_filename in test_files:
            with open(input_filename) as inf, open(expected_filename) as exf:
                result = list(main(inf))
                expected_text = list(l.strip() for l in exf)

                differences = list(
                    difflib.context_diff(result, expected_text, input_filename,
                                         expected_filename))
                if differences:
                    for diff in differences:
                        print(diff)
                else:
                    print("Test Case %i passed" % i)
    else:
        # Or just run and report the output back to stdout
        sys.stdout.write("\n".join(main(list(sys.stdin))))
Example #49
0
    except urllib2.URLError as e:
        print '%s: URL Error: %s' % (domain, str(e.reason))

    m = hashlib.md5()
    m.update(newJson)
    newHash = m.hexdigest()

    oldDump = '%s.dump' % domain
    if os.path.isfile(oldDump):
        try:
            oldJson = open(oldDump).read()
            oldHash = hashlib.md5(oldJson).hexdigest()
        except IOError as e:
            print 'Cannot read %s.dump: %s (%s)' % (domain, e.strerror,
                                                    e.errno)
            continue

        if newHash != oldHash:
            print "New credentials leaked for %s:" % domain
            diff = difflib.context_diff(oldJson.splitlines(1),
                                        newJson.splitlines(1),
                                        fromfile='Old Hash',
                                        tofile='New Hash')
            print ''.join(diff)
    try:
        w = open(oldDump, 'w')
        w.write(newJson)
        w.close()
    except IOError as e:
        print "Cannot save %s.dump: %s (%s)" % (e.strerror, e.errno)
Example #50
0
 def diff(self, other):
     return list(
         line.rstrip()
         for line in difflib.context_diff(self.lines, other.lines))[2:]
        if l.upper() != l:
            print("error:", l)

        ATRs.append(l.strip())
# 	if l.startswith("\t"):
# 		ATRs.append([atr, l.strip()])
# 	else:
# 		atr = l.strip()

# pp.pprint(ATRs)
sorted_ATRs = list(ATRs)
sorted_ATRs.sort()

# pp.pprint(sorted_ATRs)

for l in difflib.context_diff(ATRs, sorted_ATRs):
    print(l)

# compte le nombre de nouveau ATR
from subprocess import Popen, PIPE

p1 = Popen(["git", "diff"], stdout=PIPE)
p2 = Popen(["grep", "+3[B,F]"], stdin=p1.stdout, stdout=PIPE)
p1.stdout.close()
output = p2.communicate()[0]

output = output.decode("utf-8")

size = len(output.split("\n")) - 1
if size >= 10:
    print()
Example #52
0
def diff_ast(a1, a2):
    for line in difflib.context_diff(repr(a1), repr(a2)):
        sys.stdout.write(line)
Example #53
0
def processFile(pathname):
    print(pathname)


walktree('C:\\Users\Abridge\Desktop\PythonLearning', processFile)

### filecmp
import filecmp
filecmp.cmp('./dataFile.txt', './Day1/datafile.txt', shallow=False)

# returns the file names as a tuple of three lists - match, mismatch, error
result = filecmp.cmpfiles('.', './Day1/', ['datafile.txt'], shallow=False)

import difflib
d = difflib.context_diff(
    open('./dataFile.txt').readlines(),
    open('./Day1/datafile.txt').readlines())
for l in d:
    print(l)

### dircmp
d = filecmp.dircmp('.', './Day1/')
d.report()
d.left
d.common
d.common_dirs
d.left_only

### zipfile and tarfile
import zipfile
z = zipfile.ZipFile('test.zip', 'w')
Example #54
0
def dumb_diff(d1, d2):
    json1 = json.dumps(d1, indent=2, sort_keys=True).splitlines()
    json2 = json.dumps(d2, indent=2, sort_keys=True).splitlines()

    print("\n".join(
        difflib.context_diff(json1, json2, fromfile="old", tofile="new", n=2)))
    def handle_new(self, feed):
        self.handle_possible_rename(feed)
        #check user whether rename.
        cursor = self.database.cursor()
        cursor.execute(
            """SELECT COUNT(*), `deleted`, `content`, `modified`, `edited_list` FROM `feeds` WHERE `id` = %s""",
            (feed['id']))

        info = cursor.fetchone()
        num_previous = info[0]
        if info[1] is not None:
            was_deleted = (int(info[1]) == 1)
        else:
            was_deleted = False

        if num_previous > 0:  #feed exist.
            if feed.has_key('message'):
                if list(difflib.context_diff(info[2], feed.get('message'))):
                    msg = {"message": info[2], "updated_time": str(info[3])}
                    if "null" in str(info[-1]):
                        edited_list = list()
                        edited_list.append(msg)
                    else:
                        edited_list = anyjson.deserialize(info[-1])
                        edited_list.append(msg)
                    cursor.execute(
                        """UPDATE `feeds` SET `user_name`=%s, `politician_id`=%s,`content`=%s, `modified`=%s, `edited_list`=%s WHERE id=%s""",
                        (feed['from']['name'], self.users[feed['from']['id']],
                         feed['message'], feed.get('updated_time').replace(
                             '+0000',
                             ''), anyjson.serialize(edited_list), feed['id']))
                    log.notice(u"Updated {0}'s feed {0}",
                               feed.get('from', {}).get('name'),
                               feed.get('id'))
            else:
                log.info(u"{0}'s feed hasn't message key.",
                         feed['from']['name'])

        else:
            url = ""
            if feed.has_key('actions'):
                url = feed.get('actions')[0].get('link')
                if self.images:
                    log.notice("Queued feed {0} for entity archiving.",
                               feed['id'])
                    self.beanstalk.put(anyjson.serialize(feed))
            else:
                url = "is a activity."

            cursor.execute(
                """INSERT INTO `feeds` (`id`, `user_name`, `politician_id`, `content`, `created`, `modified`, `feed`, `feed_type`, url, edited_list) 
                        VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)""",
                (feed.get('id'), feed.get('from', {}).get('name'),
                 self.users[feed['from']['id']], feed.get('message', ''),
                 feed.get('created_time').replace('+0000', ''),
                 feed.get('updated_time').replace('+0000', ''),
                 anyjson.serialize(feed), feed.get('type'), url, "[]"))

            log.notice(u"Inserted {1}'s new feed {0}", feed.get('id'),
                       feed.get('from', {}).get('name'))

        if was_deleted:
            log.warn("feed deleted {0} before it came!", feed.get('id'))
Example #56
0
#!/usr/bin/env python

import difflib, sys

with open("file1.txt", "r") as file1:
    dataset1 = file1.readlines()

with open("file2.txt", "r") as file2:
    dataset1 = file2.readlines()

for line in difflib.context_diff(file1, file2):
    sys.stdout.write(line)
Example #57
0
def release(session):
    package_name = "packaging"
    version_file = Path(f"{package_name}/__about__.py")
    changelog_file = Path("CHANGELOG.rst")

    try:
        release_version = _get_version_from_arguments(session.posargs)
    except ValueError as e:
        session.error(f"Invalid arguments: {e}")
        return

    # Check state of working directory and git.
    _check_working_directory_state(session)
    _check_git_state(session, release_version)

    # Prepare for release.
    _changelog_update_unreleased_title(release_version, file=changelog_file)
    session.run("git", "add", str(changelog_file), external=True)
    _bump(session, version=release_version, file=version_file, kind="release")

    # Tag the release commit.
    # fmt: off
    session.run(
        "git",
        "tag",
        "-s",
        release_version,
        "-m",
        f"Release {release_version}",
        external=True,
    )
    # fmt: on

    # Prepare for development.
    _changelog_add_unreleased_title(file=changelog_file)
    session.run("git", "add", str(changelog_file), external=True)

    major, minor = map(int, release_version.split("."))
    next_version = f"{major}.{minor + 1}.dev0"
    _bump(session, version=next_version, file=version_file, kind="development")

    # Checkout the git tag.
    session.run("git", "checkout", "-q", release_version, external=True)

    session.install("build", "twine")

    # Build the distribution.
    session.run("python", "-m", "build")

    # Check what files are in dist/ for upload.
    files = sorted(glob.glob("dist/*"))
    expected = [
        f"dist/{package_name}-{release_version}-py3-none-any.whl",
        f"dist/{package_name}-{release_version}.tar.gz",
    ]
    if files != expected:
        diff_generator = difflib.context_diff(expected,
                                              files,
                                              fromfile="expected",
                                              tofile="got",
                                              lineterm="")
        diff = "\n".join(diff_generator)
        session.error(f"Got the wrong files:\n{diff}")

    # Get back out into main.
    session.run("git", "checkout", "-q", "main", external=True)

    # Check and upload distribution files.
    session.run("twine", "check", *files)

    # Push the commits and tag.
    # NOTE: The following fails if pushing to the branch is not allowed. This can
    #       happen on GitHub, if the main branch is protected, there are required
    #       CI checks and "Include administrators" is enabled on the protection.
    session.run("git",
                "push",
                "upstream",
                "main",
                release_version,
                external=True)

    # Upload the distribution.
    session.run("twine", "upload", *files)

    # Open up the GitHub release page.
    webbrowser.open("https://github.com/pypa/packaging/releases")
Example #58
0
def run_regression(proof, dump, wrapper, cvc4_binary, benchmark_path, timeout):
    """Determines the expected output for a benchmark, runs CVC4 on it and then
    checks whether the output corresponds to the expected output. Optionally
    uses a wrapper `wrapper`, tests proof generation (if proof is true), or
    dumps a benchmark and uses that as the input (if dump is true)."""

    if not os.access(cvc4_binary, os.X_OK):
        sys.exit(
            '"{}" does not exist or is not executable'.format(cvc4_binary))
    if not os.path.isfile(benchmark_path):
        sys.exit('"{}" does not exist or is not a file'.format(benchmark_path))

    basic_command_line_args = []

    benchmark_basename = os.path.basename(benchmark_path)
    benchmark_filename, benchmark_ext = os.path.splitext(benchmark_basename)
    benchmark_dir = os.path.dirname(benchmark_path)
    comment_char = '%'
    status_regex = None
    status_to_output = lambda s: s
    if benchmark_ext == '.smt':
        status_regex = r':status\s*(sat|unsat)'
        comment_char = ';'
    elif benchmark_ext == '.smt2':
        status_regex = r'set-info\s*:status\s*(sat|unsat)'
        comment_char = ';'
    elif benchmark_ext == '.cvc':
        pass
    elif benchmark_ext == '.p':
        basic_command_line_args.append('--finite-model-find')
        status_regex = r'% Status\s*:\s*(Theorem|Unsatisfiable|CounterSatisfiable|Satisfiable)'
        status_to_output = lambda s: '% SZS status {} for {}'.format(
            s, benchmark_filename)
    elif benchmark_ext == '.sy':
        comment_char = ';'
        # Do not use proofs/unsat-cores with .sy files
        proof = False
    else:
        sys.exit('"{}" must be *.cvc or *.smt or *.smt2 or *.p or *.sy'.format(
            benchmark_basename))

    # If there is an ".expect" file for the benchmark, read the metadata
    # from there, otherwise from the benchmark file.
    metadata_filename = benchmark_path + '.expect'
    if os.path.isfile(metadata_filename):
        comment_char = '%'
    else:
        metadata_filename = benchmark_path

    metadata_lines = None
    with open(metadata_filename, 'r') as metadata_file:
        metadata_lines = metadata_file.readlines()

    benchmark_content = None
    if metadata_filename == benchmark_path:
        benchmark_content = ''.join(metadata_lines)
    else:
        with open(benchmark_path, 'r') as benchmark_file:
            benchmark_content = benchmark_file.read()

    # Extract the metadata for the benchmark.
    scrubber = None
    error_scrubber = None
    expected_output = ''
    expected_error = ''
    expected_exit_status = None
    command_line = ''
    for line in metadata_lines:
        # Skip lines that do not start with "%"
        if line[0] != comment_char:
            continue
        line = line[2:]

        if line.startswith(SCRUBBER):
            scrubber = line[len(SCRUBBER):]
        elif line.startswith(ERROR_SCRUBBER):
            error_scrubber = line[len(ERROR_SCRUBBER):]
        elif line.startswith(EXPECT):
            expected_output += line[len(EXPECT):]
        elif line.startswith(EXPECT_ERROR):
            expected_error += line[len(EXPECT_ERROR):]
        elif line.startswith(EXIT):
            expected_exit_status = int(line[len(EXIT):])
        elif line.startswith(COMMAND_LINE):
            command_line += line[len(COMMAND_LINE):]
    expected_output = expected_output.strip()
    expected_error = expected_error.strip()

    # Expected output/expected error has not been defined in the metadata for
    # the benchmark. Try to extract the information from the benchmark itself.
    if expected_output == '' and expected_error == '':
        match = None
        if status_regex:
            match = re.search(status_regex, benchmark_content)

        if match:
            expected_output = status_to_output(match.group(1))
        elif expected_exit_status is None:
            # If there is no expected output/error and the exit status has not
            # been set explicitly, the benchmark is invalid.
            sys.exit('Cannot determine status of "{}"'.format(benchmark_path))

    if not proof and ('(get-unsat-core)' in benchmark_content
                      or '(get-unsat-assumptions)' in benchmark_content):
        print(
            '1..0 # Skipped: unsat cores not supported without proof support')
        return

    if expected_exit_status is None:
        expected_exit_status = 0

    if 'CVC4_REGRESSION_ARGS' in os.environ:
        basic_command_line_args += shlex.split(
            os.environ['CVC4_REGRESSION_ARGS'])
    basic_command_line_args += shlex.split(command_line)
    command_line_args_configs = [basic_command_line_args]

    extra_command_line_args = []
    if benchmark_ext == '.sy' and \
        '--no-check-synth-sol' not in basic_command_line_args and \
        '--check-synth-sol' not in basic_command_line_args:
        extra_command_line_args = ['--check-synth-sol']
    if re.search(r'^(sat|invalid|unknown)$', expected_output) and \
       '--no-check-models' not in basic_command_line_args:
        extra_command_line_args = ['--check-models']
    if proof and re.search(r'^(unsat|valid)$', expected_output):
        if '--no-check-proofs' not in basic_command_line_args and \
           '--incremental' not in basic_command_line_args and \
           '--unconstrained-simp' not in basic_command_line_args and \
           not cvc4_binary.endswith('pcvc4'):
            extra_command_line_args = [
                '--check-proofs', '--no-bv-eq', '--no-bv-ineq',
                '--no-bv-algebraic'
            ]
        if '--no-check-unsat-cores' not in basic_command_line_args and \
           '--incremental' not in basic_command_line_args and \
           '--unconstrained-simp' not in basic_command_line_args and \
           not cvc4_binary.endswith('pcvc4'):
            extra_command_line_args += ['--check-unsat-cores']
    if extra_command_line_args:
        command_line_args_configs.append(basic_command_line_args +
                                         extra_command_line_args)

    # Run CVC4 on the benchmark with the different option sets and check
    # whether the exit status, stdout output, stderr output are as expected.
    print('1..{}'.format(len(command_line_args_configs)))
    print('# Starting')
    for command_line_args in command_line_args_configs:
        output, error, exit_status = run_benchmark(dump, wrapper, scrubber,
                                                   error_scrubber, cvc4_binary,
                                                   command_line_args,
                                                   benchmark_dir,
                                                   benchmark_basename, timeout)
        if output != expected_output:
            print(
                'not ok - Differences between expected and actual output on stdout - Flags: {}'
                .format(command_line_args))
            for line in difflib.context_diff(output.splitlines(),
                                             expected_output.splitlines()):
                print(line)
        elif error != expected_error:
            print(
                'not ok - Differences between expected and actual output on stderr - Flags: {}'
                .format(command_line_args))
            for line in difflib.context_diff(error.splitlines(),
                                             expected_error.splitlines()):
                print(line)
        elif expected_exit_status != exit_status:
            print(
                'not ok - Expected exit status "{}" but got "{}" - Flags: {}'.
                format(expected_exit_status, exit_status, command_line_args))
        else:
            print('ok - Flags: {}'.format(command_line_args))
Example #59
0
def diff_main():
    """ Command line interface to difflib.py providing diffs in four formats:

    * ndiff:    lists every line and highlights interline changes.
    * context:  highlights clusters of changes in a before/after format.
    * unified:  highlights clusters of changes in an inline format.
    * html:     generates side by side comparison with change highlights.

    """
    # Configure the option parser
    usage = "usage: %prog [options] fromfile tofile"
    parser = optparse.OptionParser(usage)
    parser.add_option("-c",
                      action="store_true",
                      default=False,
                      help='Produce a context format diff (default)')
    parser.add_option("-u",
                      action="store_true",
                      default=False,
                      help='Produce a unified format diff')
    hlp = 'Produce HTML side by side diff (can use -c and -l in conjunction)'
    parser.add_option("-m", action="store_true", default=False, help=hlp)
    parser.add_option("-n",
                      action="store_true",
                      default=False,
                      help='Produce a ndiff format diff')
    parser.add_option("-l",
                      "--lines",
                      type="int",
                      default=3,
                      help='Set number of context lines (default 3)')
    (options, args) = parser.parse_args()

    if len(args) == 0:
        parser.print_help()
        sys.exit(1)
    if len(args) != 2:
        parser.error("need to specify both a fromfile and tofile")

    n = options.lines
    fromfile, tofile = args  # as specified in the usage string

    # we're passing these as arguments to the diff function
    fromdate = time.ctime(os.stat(fromfile).st_mtime)
    todate = time.ctime(os.stat(tofile).st_mtime)
    fromlines = open(fromfile, 'U').readlines()
    tolines = open(tofile, 'U').readlines()

    if options.u:
        diff = difflib.unified_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)
    elif options.n:
        diff = difflib.ndiff(fromlines, tolines)
    elif options.m:
        diff = difflib.HtmlDiff().make_file(fromlines,
                                            tolines,
                                            fromfile,
                                            tofile,
                                            context=options.c,
                                            numlines=n)
    else:
        diff = difflib.context_diff(fromlines,
                                    tolines,
                                    fromfile,
                                    tofile,
                                    fromdate,
                                    todate,
                                    n=n)

    # we're using writelines because diff is a generator
    sys.stdout.writelines(diff)
Example #60
0
def create_source_map(nodes, code, filename, indices_in_code):
    """Creates a source map between an annotated AST and the code it compiles to.

  Args:
    nodes: Iterable[ast.AST, ...]
    code: Text
    filename: Optional[Text]
    indices_in_code: Union[int, Iterable[int, ...]], the positions at which
        nodes appear in code. The parser always returns a module when parsing
        code. This argument indicates the position in that module's body at
        which the corresponding of node should appear.

  Returns:
    Dict[CodeLocation, OriginInfo], mapping locations in code to locations
    indicated by origin annotations in node.
  """
    reparsed_nodes = parser.parse_str(code)
    reparsed_nodes = [reparsed_nodes.body[i] for i in indices_in_code]

    resolve(reparsed_nodes, code)
    result = {}

    try:
        for before, after in ast_util.parallel_walk(nodes, reparsed_nodes):
            # Note: generated code might not be mapped back to its origin.
            # TODO(mdan): Generated code should always be mapped to something.
            origin_info = anno.getanno(before, anno.Basic.ORIGIN, default=None)
            final_info = anno.getanno(after, anno.Basic.ORIGIN, default=None)
            if origin_info is None or final_info is None:
                continue

            line_loc = LineLocation(filename, final_info.loc.lineno)

            existing_origin = result.get(line_loc)
            if existing_origin is not None:
                # Overlaps may exist because of child nodes, but almost never to
                # different line locations. Exception make decorated functions, where
                # both lines are mapped to the same line in the AST.

                # Line overlaps: keep bottom node.
                if existing_origin.loc.line_loc == origin_info.loc.line_loc:
                    if existing_origin.loc.lineno >= origin_info.loc.lineno:
                        continue

                # In case of overlaps, keep the leftmost node.
                if existing_origin.loc.col_offset <= origin_info.loc.col_offset:
                    continue

            result[line_loc] = origin_info
    except ValueError:
        if logging.has_verbosity(3):
            for n, rn in zip(nodes, reparsed_nodes):
                nodes_str = pretty_printer.fmt(n, color=False, noanno=True)
                reparsed_nodes_str = pretty_printer.fmt(rn,
                                                        color=False,
                                                        noanno=True)
                diff = difflib.context_diff(nodes_str.split('\n'),
                                            reparsed_nodes_str.split('\n'),
                                            fromfile='Original nodes',
                                            tofile='Reparsed nodes',
                                            n=7)
                diff = '\n'.join(diff)
                logging.log(3, 'AST seems to lack integrity. Diff:\n%s', diff)
        raise

    return result