def compare_site_and_disk(config, diff, site, docs, push, get): """ Does both compare and push/get since it's quite similar code-wide""" for f in docs: full_path = "./" + f + ".mediawiki" m_ondisk = hashlib.new(config["hashalg"]) with open(full_path) as fd: on_disk = fd.read() m_ondisk.update(on_disk.encode("utf-8")) m_onsite = hashlib.new(config["hashalg"]) page = site.Pages[f] on_site = page.text().encode("utf-8") + "\n".encode("utf-8") m_onsite.update(on_site) if m_ondisk.digest() != m_onsite.digest(): print("Page {} differ.".format(f)) if diff: # Just display diff in the correct order, we default to push-side-diff if get: mydiff = difflib.unified_diff(blist2str(on_site.splitlines(1)), blist2str(on_disk.splitlines(1))) else: mydiff = difflib.unified_diff(blist2str(on_disk.splitlines(1)), blist2str(on_site.splitlines(1))) sys.stdout.writelines(mydiff) # Now push or get whatever is needed to sync # But never do both push and get at once, would make no sense if get: print("Getting {} from site to disk.".format(f)) with open(full_path, "w") as fd: fd.write(on_site.decode("utf-8")) elif push: check_repos_is_current(config) print("Pushing {} from disk to site.".format(f)) page.save(on_disk, summary=u"Automated sync from {}".format(config["repos"]))
def testrr_helper (filee, jalangi, norm_fn, record_fn, instrument_fn=instrument): try: shutil.rmtree("jalangi_tmp") except: pass os.mkdir("jalangi_tmp") os.mkdir("jalangi_tmp/out") os.putenv("JALANGI_HOME", jalangi.get_home()) os.chdir("jalangi_tmp") (instrumented_f, out) = instrument_fn(os.path.join(os.pardir,filee), jalangi=jalangi) try: # Ignore failures on first iteration os.remove("inputs.js") shutil.copy("jalangi_inputs{}.js".format(i), "inputs.js") except: pass if not os.path.isfile("inputs.js"): util.mkempty("inputs.js") print "---- Running without instrumentation ----" try: os.remove("jalangi_trace") except: pass norm = norm_fn(os.path.join(os.pardir,filee + ".js"), jalangi=jalangi) with open("jalangi_normal", "w") as normfile: normfile.write(norm) print norm print "---- Recording execution of {} ----".format(filee) rec = record_fn(os.path.join(os.pardir,filee), instrumented_f) with open("jalangi_record", "w") as recfile: recfile.write(rec) print rec print "---- Replaying {} ----".format(filee) os.putenv("JALANGI_MODE", "replay") os.putenv("JALANGI_ANALYSIS", "none") rep = replay() with open("jalangi_replay", "w") as repfile: repfile.write(rep) print rep try: wcl = util.count_lines("jalangi_trace") with open("../jalangi_test_results", 'a') as f: f.write("# of lines in jalangi_trace for {}: {}".format(filee,str(wcl))) f.write("\n") except: pass if norm != rep: #TODO: Factor out this. print "{}.js failed".format(filee) import difflib with open("../jalangi_test_results", 'a') as f: f.write("\n") for line in difflib.unified_diff(norm.splitlines(1), rec.splitlines(1), fromfile='normal.{}'.format(filee), tofile='replay.{}'.format(filee)): f.write(line) if rec != rep: print "{}.js failed".format(filee) import difflib with open("../jalangi_test_results", 'a') as f: f.write("\n") for line in difflib.unified_diff(rec.splitlines(1), rep.splitlines(1), fromfile='record.{}'.format(filee), tofile='replay.{}'.format(filee)): f.write(line) util.move_coverage(jalangi)
def unusedClangcheckFile(self, path, ext): if ext not in self.SRCEXTS: return self.SKIP with open(path, 'rt') as f: try: originalraw = f.read() except UnicodeDecodeError as e: print('File "{}" has invalid unicode: {}'.format(path, e)) return False original = originalraw.splitlines(keepends=True) #cmd = ['astyle'] + list(self.ASTYLE_OPTIONS) cmd = ('clang-format', '-style=file') proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) po, pe = proc.communicate(originalraw.encode()) assert proc.returncode == 0, "Return code {}".format(proc.returncode) assert len(pe) == 0, "clang-format error: {}".format(pe) try: formatted = po.decode().splitlines(keepends=True) except UnicodeDecodeError as e: print('File "{}" has invalid unicode during formatting: {}'.format(path, e)) return False diff = list(difflib.unified_diff(original, formatted, 'current', 'formatted')) if not len(diff): return True print('File "{}" has invalid style'.format(path)) for d in difflib.unified_diff(self.hightLightSpaces(original), self.hightLightSpaces(formatted), 'current', 'formatted'): print(d, end='') return False
def test_same_output_with_different_engine_incr(self): """Reference implementation should match mustache and mako implem""" w("""cat <<EOF > .gitchangelog.rc output_engine = mustache('restructuredtext') EOF """) changelog = w('$tprog show 0.0.2..0.0.3') self.assertEqual( changelog, self.INCR_REFERENCE_002_003, msg="Mustache output should match our reference output... " "diff of changelogs:\n%s" % '\n'.join(difflib.unified_diff(self.INCR_REFERENCE_002_003.split("\n"), changelog.split("\n"), lineterm=""))) w("""cat <<EOF > .gitchangelog.rc output_engine = makotemplate('restructuredtext') EOF """) changelog = w('$tprog show 0.0.2..0.0.3') self.assertEqual( changelog, self.INCR_REFERENCE_002_003, msg="Mako output should match our reference output... " "diff of changelogs:\n%s" % '\n'.join(difflib.unified_diff(self.INCR_REFERENCE_002_003.split("\n"), changelog.split("\n"), lineterm="")))
def show_diff(self, chart, installed_chart, installed_values, target_chart, target_values): ''' Produce a unified diff of the installed chart vs our intention TODO(alanmeadows): This needs to be rewritten to produce better unified diff output and avoid the use of print ''' chart_diff = list(difflib.unified_diff(installed_chart .SerializeToString() .split('\n'), target_chart.split('\n'))) if len(chart_diff) > 0: LOG.info("Chart Unified Diff (%s)", chart.release_name) for line in chart_diff: LOG.debug(line) values_diff = list(difflib.unified_diff(installed_values.split('\n'), yaml .safe_dump(target_values) .split('\n'))) if len(values_diff) > 0: LOG.info("Values Unified Diff (%s)", chart.release_name) for line in values_diff: LOG.debug(line) return (len(chart_diff) > 0) or (len(values_diff) > 0)
def runTest(self, fileName, sedCmd, changeCmd = None): new = os.popen("sed '%s' < %s" % (sedCmd, fileName)).readlines() orig = open(fileName).readlines() if changeCmd: target = os.popen("sed '%s' < %s" % (changeCmd, fileName)).readlines() final = os.popen("sed '%s' < %s | sed '%s'" % (sedCmd, fileName, changeCmd)).readlines() check = os.popen("sed '%s' < %s | sed '%s'" % (changeCmd, fileName, sedCmd)).readlines() if "".join(final) != "".join(check): self.fail("sed scripts have conflicting results") else: target = orig final = new diff = difflib.unified_diff(orig, new) # advance past header diff.next() diff.next() d = [ x for x in diff ] (new2, conflicts) = patch(target, d) diff = difflib.unified_diff(final, new2) try: diff.next() print "%s '%s' failed:" % (fileName, sedCmd) diff.next() for line in diff: line = line[:-1] print "\t%s" % line except StopIteration: pass
def main(): opt = parse_args() import psycopg2.tests test = psycopg2.tests if opt.suite: test = getattr(test, opt.suite) sys.stdout.write("test suite %s\n" % test.__name__) for i in range(1, opt.nruns + 1): sys.stdout.write("test suite run %d of %d\n" % (i, opt.nruns)) runner = unittest.TextTestRunner() runner.run(test.test_suite()) dump(i, opt) f1 = open('debug-%02d.txt' % (opt.nruns - 1)).readlines() f2 = open('debug-%02d.txt' % (opt.nruns)).readlines() for line in difflib.unified_diff(f1, f2, "run %d" % (opt.nruns - 1), "run %d" % opt.nruns): sys.stdout.write(line) rv = f1 != f2 and 1 or 0 if opt.objs: f1 = open('objs-%02d.txt' % (opt.nruns - 1)).readlines() f2 = open('objs-%02d.txt' % (opt.nruns)).readlines() for line in difflib.unified_diff(f1, f2, "run %d" % (opt.nruns - 1), "run %d" % opt.nruns): sys.stdout.write(line) return rv
def file_diff(filename1, filename2, lineno=None, context=None): INPUT1=open_possibly_compressed_file(filename1) lines1 = INPUT1.readlines() for i in range(0,len(lines1)): lines1[i] = lines1[i].strip() INPUT1.close() INPUT2=open_possibly_compressed_file(filename2) lines2 = INPUT2.readlines() for i in range(0,len(lines2)): lines2[i] = lines2[i].strip() INPUT2.close() s="" if lineno is None: for line in difflib.unified_diff(lines2,lines1,fromfile=filename2,tofile=filename1): s += line+"\n" else: if context is None: context = 3 start = lineno-context stop = lineno+context if start < 0: start=0 if stop > len(lines1): stop = len(lines1) if stop > len(lines2): stop = len(lines2) for line in difflib.unified_diff(lines2[start:stop],lines1[start:stop],fromfile=filename2,tofile=filename1): s += line+"\n" return s
def compare_site_and_disk(config, diff, site, docs, push, get): ''' Does both compare and push/get since it's quite similar code-wide''' for f in docs: full_path = './'+f+'.mediawiki' m_ondisk = hashlib.new(config['hashalg']) with open(full_path) as fd: on_disk = fd.read() m_ondisk.update(on_disk) m_onsite = hashlib.new(config['hashalg']) page = site.Pages[f] on_site = page.text().encode('utf-8')+'\n' m_onsite.update(on_site) if m_ondisk.digest() != m_onsite.digest(): print("Page {} differ.".format(f)) if (diff): #Just display diff in the correct order, we default to push-side-diff if get: mydiff = difflib.unified_diff(on_site.splitlines(1), on_disk.splitlines(1)) else: mydiff = difflib.unified_diff(on_disk.splitlines(1), on_site.splitlines(1)) sys.stdout.writelines(mydiff) #Now push or get whatever is needed to sync #But never do both push and get at once, would make no sense if get: print("Getting {} from site to disk.".format(f)) with open(full_path, 'w') as fd: fd.write(on_site) elif push: print("Pushing {} from disk to site.".format(f)) page.save(on_disk, summary=u'Automated sync from {}'.format(config['repos']))
def generate_header_diff_list(pre_header_dict, header_dict): # return result list header_diff_list = [] # diff origin data client_request_hdr_pre = [] server_request_hdr_pre = [] server_response_hdr_pre = [] client_response_hdr_pre = [] # based on pre_header_dict, init origin data if pre_header_dict and pre_header_dict['state_machine_id'] == header_dict['state_machine_id']: client_request_hdr_pre = pre_header_dict['header_list'][0]['hdr_info'] server_request_hdr_pre = pre_header_dict['header_list'][1]['hdr_info'] server_response_hdr_pre = pre_header_dict['header_list'][2]['hdr_info'] client_response_hdr_pre = pre_header_dict['header_list'][3]['hdr_info'] # header_dict could not be null, get the new data client_request_hdr = header_dict['header_list'][0]['hdr_info'] server_request_hdr = header_dict['header_list'][1]['hdr_info'] server_response_hdr = header_dict['header_list'][2]['hdr_info'] client_response_hdr = header_dict['header_list'][3]['hdr_info'] # generate diff list header_diff_list.append({'name': 'Client Request', 'hdr_diff': difflib.unified_diff(client_request_hdr_pre, client_request_hdr, lineterm='')}) header_diff_list.append({'name': 'Server Request', 'hdr_diff': difflib.unified_diff(server_request_hdr_pre, server_request_hdr, lineterm='')}) header_diff_list.append({'name': 'Server Response', 'hdr_diff': difflib.unified_diff(server_response_hdr_pre, server_response_hdr, lineterm='')}) header_diff_list.append({'name': 'Client Response', 'hdr_diff': difflib.unified_diff(client_response_hdr_pre, client_response_hdr, lineterm='')}) return header_diff_list
def get_diff() -> str: """Get a HTML diff for the old config files.""" old_conf_lines = [] # type: typing.MutableSequence[str] old_key_lines = [] # type: typing.MutableSequence[str] for filename, dest in [('qutebrowser.conf', old_conf_lines), ('keys.conf', old_key_lines)]: path = os.path.join(standarddir.config(), filename) with open(path, 'r', encoding='utf-8') as f: for line in f: if not line.strip() or line.startswith('#'): continue dest.append(line.rstrip()) conf_delta = difflib.unified_diff(OLD_CONF.lstrip().splitlines(), old_conf_lines) key_delta = difflib.unified_diff(OLD_KEYS_CONF.lstrip().splitlines(), old_key_lines) conf_diff = '\n'.join(conf_delta) key_diff = '\n'.join(key_delta) # pylint: disable=no-member # WORKAROUND for https://bitbucket.org/logilab/pylint/issue/491/ lexer = pygments.lexers.DiffLexer() formatter = pygments.formatters.HtmlFormatter( full=True, linenos='table', title='Diffing pre-1.0 default config with pre-1.0 modified config') # pylint: enable=no-member return pygments.highlight(conf_diff + key_diff, lexer, formatter)
def context_dict(revision, ready_for_l10n=False, revision_approved=False): """Return a dict that fills in the blanks in KB notification templates.""" document = revision.document diff = '' l10n = revision.document.revisions.filter(is_ready_for_localization=True) approved = revision.document.revisions.filter(is_approved=True) if ready_for_l10n and l10n.count() > 1: fromfile = u'[%s] %s #%s' % (revision.document.locale, revision.document.title, l10n.order_by('-created')[1].id) tofile = u'[%s] %s #%s' % (revision.document.locale, revision.document.title, revision.id) diff = clean( u''.join( difflib.unified_diff( l10n.order_by('-created')[1].content.splitlines(1), revision.content.splitlines(1), fromfile=fromfile, tofile=tofile) ), ALLOWED_TAGS, ALLOWED_ATTRIBUTES) elif revision_approved and approved.count() > 1: doc = revision.document approved_rev = approved.order_by('-created')[1] fromfile = u'[%s] %s #%s' % (doc.locale, doc.title, approved_rev.id) tofile = u'[%s] %s #%s' % (doc.locale, doc.title, revision.id) diff = clean( u''.join( difflib.unified_diff( approved_rev.content.splitlines(1), revision.content.splitlines(1), fromfile=fromfile, tofile=tofile) ), ALLOWED_TAGS, ALLOWED_ATTRIBUTES) elif revision.document.current_revision is not None: fromfile = u'[%s] %s #%s' % (revision.document.locale, revision.document.title, revision.document.current_revision.id) tofile = u'[%s] %s #%s' % (revision.document.locale, revision.document.title, revision.id) diff = clean( u''.join( difflib.unified_diff( revision.document.current_revision.content.splitlines(1), revision.content.splitlines(1), fromfile=fromfile, tofile=tofile) ), ALLOWED_TAGS, ALLOWED_ATTRIBUTES) return { 'document_title': document.title, 'creator': revision.creator, 'host': Site.objects.get_current().domain, 'diff': diff, 'fulltext': clean(revision.content, ALLOWED_TAGS, ALLOWED_ATTRIBUTES)}
def __str__(self): ret = TestResult.__str__(self) if self.verbose: if self.content and self.content_old: diff = difflib.unified_diff(open(self.content).read().split("\n"), open(self.content_old).read().split("\n"), "Non-Tor1", "Non-Tor2", lineterm="") for line in diff: ret+=line+"\n" if self.content and self.content_exit: diff = difflib.unified_diff(open(self.content).read().split("\n"), open(self.content_exit).read().split("\n"), "Non-Tor", "Exit", lineterm="") for line in diff: ret+=line+"\n" else: if self.content: ret += " "+self.content+"\n" if self.content_old: ret += " "+self.content_old+"\n" if self.content_exit: ret += " "+self.content_exit+"\n" return ret
def update(self, content): """Update the vim buffer with the new content.""" self.dirty = False if not self.buf.registered: return # build the list of the offsets of the beginning of each line newlist = content.splitlines(1) num_lines = len(newlist) offsets = [offset for offset in misc.offset_gen(newlist)] started = False hunk_a = hunk_b = 0 send_function = self.send_function try: if logger.level <= logging.DEBUG: for line in difflib.unified_diff(self.linelist, newlist): debug(line.strip('\n')) for line in difflib.unified_diff(self.linelist, newlist): if not started: if line.startswith('+++'): started = True continue if hunk_a == hunk_b == 0: matchobj = re_unidiff.match(line.strip()) if matchobj: lnum = int(matchobj.group('lnum')) hunk_a = int(matchobj.group('a')) hunk_b = int(matchobj.group('b')) else: assert False, "missing unified-diff control line" continue if line[0] == ' ': lnum += 1 hunk_a -= 1 hunk_b -= 1 elif line[0] == '+': delta = len(line) - 1 send_function('insert', '%s %s' % (str(offsets[lnum-1]), misc.quote(line[1:]))) self.len += delta lnum += 1 hunk_b -= 1 elif line[0] == '-': delta = len(line) - 1 if lnum <= num_lines: self.remove(offsets[lnum-1], delta) else: # removing (one of) the last line(s) self.remove(len(content), delta) self.len -= delta hunk_a -= 1 else: assert False, "unknown unified-diff line type" finally: self.terminate_editing() self.linelist = newlist
def system(name, **kwargs): ''' Ensure that global network settings are configured properly. name Custom name to represent this configuration change. kwargs The global parameters for the system. ''' ret = { 'name': name, 'changes': {}, 'result': True, 'comment': 'Global network settings are up to date.' } apply_net_settings = False kwargs['test'] = __opts__['test'] # Build global network settings try: old = __salt__['ip.get_network_settings']() new = __salt__['ip.build_network_settings'](kwargs) if __opts__['test']: if old == new: return ret if not old and new: ret['result'] = None ret['comment'] = 'Global network settings are set to be added.' return ret elif old != new: diff = difflib.unified_diff(old, new) ret['result'] = None ret['comment'] = 'Global network settings are set to be updated.' ret['changes']['network_settings'] = ''.join(diff) return ret if not old and new: apply_net_settings = True ret['changes']['network_settings'] = 'Added global network settings.' elif old != new: diff = difflib.unified_diff(old, new) apply_net_settings = True ret['changes']['network_settings'] = ''.join(diff) except AttributeError as error: ret['result'] = False ret['comment'] = error.message return ret # Apply global network settings if apply_net_settings: try: __salt__['ip.apply_network_settings'](kwargs) except AttributeError as error: ret['result'] = False ret['comment'] = error.message return ret return ret
def run(d): method = d['method'] bedtool = d['bedtool'] convert = d['convert'] kwargs = d['kw'].copy() expected = d['test_case']['expected'] bedtool_converter = convert.pop('bedtool') bedtool = ( converters[bedtool_converter](pybedtools.example_bedtool(bedtool)) ) for k, converter_name in convert.items(): kwargs[k] = ( converters[converter_name](pybedtools.example_bedtool(kwargs[k])) ) result = getattr(bedtool, method)(**kwargs) res = str(result) expected = fix(expected) try: assert res == expected except AssertionError: print(result.fn) print('Method call:') args = [] for key, val in list(kwargs.items()): args.append(('%s=%s' % (key, val)).strip()) args = ', '.join(args) print('BedTool.%(method)s(%(args)s)' % locals()) print('Got:') print(res) print('Expected:') print(expected) print('Diff:') for i in ( difflib.unified_diff(res.splitlines(1), expected.splitlines(1)) ): print(i, end=' ') # Make tabs and newlines visible spec_res = res.replace('\t', '\\t').replace('\n', '\\n\n') spec_expected = expected.replace('\t', '\\t').replace('\n', '\\n\n') print('Showing special characters:') print('Got:') print(spec_res) print('Expected:') print(spec_expected) print('Diff:') for i in ( difflib.unified_diff(spec_res.splitlines(1), spec_expected.splitlines(1)) ): print(i, end=' ') raise
def show_diff(old_opt, old_label, new_opt, new_label): if args.sort: sys.stdout.writelines(difflib.unified_diff( sorted(old_opt), sorted(new_opt), fromfile=old_label, tofile=new_label)) else: sys.stdout.writelines(difflib.unified_diff( old_opt, new_opt, fromfile=old_label, tofile=new_label))
def routes(name, **kwargs): ''' Manage network interface static routes. name Interface name to apply the route to. kwargs Named routes ''' ret = { 'name': name, 'changes': {}, 'result': True, 'comment': 'Interface {0} routes are up to date.'.format(name), } apply_routes = False kwargs['test'] = __opts__['test'] # Build interface routes try: old = __salt__['ip.get_routes'](name) new = __salt__['ip.build_routes'](name, **kwargs) if __opts__['test']: if old == new: return ret if not old and new: ret['result'] = None ret['comment'] = 'Interface {0} routes are set to be added.'.format(name) return ret elif old != new: diff = difflib.unified_diff(old, new) ret['result'] = None ret['comment'] = 'Interface {0} routes are set to be updated.'.format(name) ret['changes']['network_routes'] = ''.join(diff) return ret if not old and new: apply_routes = True ret['changes']['network_routes'] = 'Added interface {0} routes.'.format(name) elif old != new: diff = difflib.unified_diff(old, new) apply_routes = True ret['changes']['network_routes'] = ''.join(diff) except AttributeError as error: ret['result'] = False ret['comment'] = error.message return ret # Apply interface routes if apply_routes: try: __salt__['ip.apply_network_settings'](**kwargs) except AttributeError as error: ret['result'] = False ret['comment'] = error.message return ret return ret
def diff(self): texts = [] for old_path, (new_path, old_l, new_l) in self.change_dct.items(): if old_path: udiff = difflib.unified_diff(old_l, new_l) else: udiff = difflib.unified_diff(old_l, new_l, old_path, new_path) texts.append('\n'.join(udiff)) return '\n'.join(texts)
def check_lexer(lx, fn): absfn = os.path.join(TESTDIR, 'examplefiles', fn) fp = open(absfn, 'rb') try: text = fp.read() finally: fp.close() text = text.replace(b'\r\n', b'\n') text = text.strip(b'\n') + b'\n' try: text = text.decode('utf-8') if text.startswith(u'\ufeff'): text = text[len(u'\ufeff'):] except UnicodeError: text = text.decode('latin1') ntext = [] tokens = [] import time t1 = time.time() for type, val in lx.get_tokens(text): ntext.append(val) assert type != Error, \ 'lexer %s generated error token for %s: %r at position %d' % \ (lx, absfn, val, len(u''.join(ntext))) tokens.append((type, val)) t2 = time.time() STATS[os.path.basename(absfn)] = (len(text), 1000 * (t2 - t1), 1000 * (t2 - t1) / len(text)) if u''.join(ntext) != text: print('\n'.join(difflib.unified_diff(u''.join(ntext).splitlines(), text.splitlines()))) raise AssertionError('round trip failed for ' + absfn) # check output against previous run if enabled if STORE_OUTPUT: # no previous output -- store it outfn = os.path.join(TESTDIR, 'examplefiles', 'output', fn) if not os.path.isfile(outfn): fp = open(outfn, 'wb') try: pickle.dump(tokens, fp) finally: fp.close() return # otherwise load it and compare fp = open(outfn, 'rb') try: stored_tokens = pickle.load(fp) finally: fp.close() if stored_tokens != tokens: f1 = pprint.pformat(stored_tokens) f2 = pprint.pformat(tokens) print('\n'.join(difflib.unified_diff(f1.splitlines(), f2.splitlines()))) assert False, absfn
def routes(name, **kwargs): """ Manage network interface static routes. name Interface name to apply the route to. kwargs Named routes """ ret = {"name": name, "changes": {}, "result": True, "comment": "Interface {0} routes are up to date.".format(name)} apply_routes = False if not "test" in kwargs: kwargs["test"] = __opts__.get("test", False) # Build interface routes try: old = __salt__["ip.get_routes"](name) new = __salt__["ip.build_routes"](name, **kwargs) if kwargs["test"]: if old == new: return ret if not old and new: ret["result"] = None ret["comment"] = "Interface {0} routes are set to be added.".format(name) return ret elif old != new: diff = difflib.unified_diff(old, new) ret["result"] = None ret["comment"] = "Interface {0} routes are set to be updated.".format(name) ret["changes"]["network_routes"] = "".join(diff) return ret if not old and new: apply_routes = True ret["comment"] = "Interface {0} routes added.".format(name) ret["changes"]["network_routes"] = "Added interface {0} routes.".format(name) elif old != new: diff = difflib.unified_diff(old, new) apply_routes = True ret["comment"] = "Interface {0} routes updated.".format(name) ret["changes"]["network_routes"] = "".join(diff) except AttributeError as error: ret["result"] = False ret["comment"] = error.message return ret # Apply interface routes if apply_routes: try: __salt__["ip.apply_network_settings"](**kwargs) except AttributeError as error: ret["result"] = False ret["comment"] = error.message return ret return ret
def _get_debug_error_message(module, old_lines, new_lines): current_lines = split_lines(module.get_code(), keepends=True) current_diff = difflib.unified_diff(new_lines, current_lines) old_new_diff = difflib.unified_diff(old_lines, new_lines) import parso return ( "There's an issue with the diff parser. Please " "report (parso v%s) - Old/New:\n%s\nActual Diff (May be empty):\n%s" % (parso.__version__, ''.join(old_new_diff), ''.join(current_diff)) )
def system(name, **kwargs): """ Ensure that global network settings are configured properly. name Custom name to represent this configuration change. kwargs The global parameters for the system. """ ret = {"name": name, "changes": {}, "result": True, "comment": "Global network settings are up to date."} apply_net_settings = False kwargs["test"] = __opts__["test"] # Build global network settings try: old = __salt__["ip.get_network_settings"]() new = __salt__["ip.build_network_settings"](**kwargs) if __opts__["test"]: if old == new: return ret if not old and new: ret["result"] = None ret["comment"] = "Global network settings are set to be added." return ret elif old != new: diff = difflib.unified_diff(old, new) ret["result"] = None ret["comment"] = "Global network settings are set to be updated." ret["changes"]["network_settings"] = "".join(diff) return ret if not old and new: apply_net_settings = True ret["changes"]["network_settings"] = "Added global network settings." elif old != new: diff = difflib.unified_diff(old, new) apply_net_settings = True ret["changes"]["network_settings"] = "".join(diff) except AttributeError as error: ret["result"] = False ret["comment"] = error.message return ret # Apply global network settings if apply_net_settings: try: __salt__["ip.apply_network_settings"](**kwargs) except AttributeError as error: ret["result"] = False ret["comment"] = error.message return ret return ret
def main(): print("Creating patches") mcp = os.path.normpath(sys.argv[1]) fml_dir = os.path.normpath(sys.argv[2]) patchd = os.path.normpath(os.path.join(fml_dir, 'patches')) base = os.path.normpath(os.path.join(mcp, 'src-base')) work = os.path.normpath(os.path.join(mcp, 'src-work')) for path, _, filelist in os.walk(work, followlinks=True): for cur_file in fnmatch.filter(filelist, '*.java'): file_base = os.path.normpath(os.path.join(base, path[len(work)+1:], cur_file)).replace(os.path.sep, '/') file_work = os.path.normpath(os.path.join(work, path[len(work)+1:], cur_file)).replace(os.path.sep, '/') fromlines = open(file_base, 'U').readlines() tolines = open(file_work, 'U').readlines() patch = ''.join(difflib.unified_diff(fromlines, tolines, '../' + file_base[len(mcp)+1:], '../' + file_work[len(mcp)+1:], '', '', n=3)) patch_dir = os.path.join(patchd, path[len(work)+1:]) patch_file = os.path.join(patch_dir, cur_file + '.patch') if len(patch) > 0: print patch_file[len(patchd)+1:] patch = patch.replace('\r\n', '\n') if not os.path.exists(patch_dir): os.makedirs(patch_dir) with open(patch_file, 'wb') as fh: fh.write(patch) else: if os.path.isfile(patch_file): print("Deleting empty patch: %s"%(patch_file)) os.remove(patch_file) cleanDirs(patchd) backup = os.path.join(mcp, 'runtime', 'commands.py.bck') runtime = os.path.join(mcp, 'runtime', 'commands.py') patch_file = os.path.join(fml_dir, 'commands.patch') if not os.path.exists(backup): shutil.copy(runtime, backup) patch = ''.join(difflib.unified_diff(open(backup, 'U').readlines(), open(runtime, 'U').readlines(), 'commands.py', 'commands.py', '', '', n=3)) if len(patch) > 0: print 'Creating commands.py patch' patch = patch.replace('\r\n', '\n') with open(patch_file, 'wb') as fh: fh.write(patch) else: if os.path.isfile(patch_file): print("Deleting empty commands.py patch") os.remove(patch_file)
def getPrettyDiff(self): blobOld = Blob(self.commit.commit.repo, unhexlify(self.oldSha)).data_stream.read() blobNew = Blob(self.commit.commit.repo, unhexlify(self.newSha)).data_stream.read() diff = '' try: diffs = unified_diff(smart_unicode(blobOld).splitlines(), smart_unicode(blobNew).splitlines()) except DjangoUnicodeDecodeError: diffs = unified_diff(str(blobOld).decode('latin1').splitlines(), str(blobNew).decode('latin1').splitlines()) for line in diffs: diff += line + '\n' diff = diff.replace('\n\n', '\n') return diff
def diff_freqs(a, b, output): a, b = edgews_normalize(a, b) a, b = sepws_normalize(a, b) ranks_a, ranks_b = [[l[:l.find(' ') or None] + '\n' for l in x] for x in [a, b]] freqs_a, freqs_b = [sorted(l[(l.find(' ') or 0)+1:] for l in x) if len(l) > 1 else [] for x in [a, b]] if not (ranks_a == ranks_b and freqs_a == freqs_b): output.write("Diff in sorted rank numbers:\n") output.writelines(difflib.unified_diff(ranks_a, ranks_b)) output.write("Diff in sorted frequencies:\n") output.writelines(difflib.unified_diff(freqs_a, freqs_b)) return False return True
def present(name, config=None, **kwargs): ''' Ensure the job is present in the Jenkins configured jobs name The unique name for the Jenkins job config The Salt URL for the file to use for configuring the job. ''' ret = {'name': name, 'result': True, 'changes': {}, 'comment': ['Job {0} is up to date.'.format(name)]} _job_exists = __salt__['jenkins.job_exists'](name) if _job_exists: _current_job_config = __salt__['jenkins.get_job_config'](name) buf = six.moves.StringIO(_current_job_config) _current_job_config = buf.readlines() cached_source_path = __salt__['cp.cache_file'](config, __env__) with salt.utils.fopen(cached_source_path) as _fp: new_config_xml = _fp.readlines() if _current_job_config != new_config_xml: diff = difflib.unified_diff(_current_job_config, new_config_xml, lineterm='') __salt__['jenkins.update_job'](name, config, __env__) ret['changes'] = ''.join(diff) ret['comment'].append('Job {0} updated.'.format(name)) else: cached_source_path = __salt__['cp.cache_file'](config, __env__) with salt.utils.fopen(cached_source_path) as _fp: new_config_xml = _fp.read() __salt__['jenkins.create_job'](name, config, __env__) buf = six.moves.StringIO(new_config_xml) _current_job_config = buf.readlines() diff = difflib.unified_diff('', buf, lineterm='') ret['changes'] = ''.join(diff) ret['comment'].append('Job {0} added.'.format(name)) ret['comment'] = '\n'.join(ret['comment']) return ret
def compareGeom(golden, outfile, headerDecoration): if not os.path.exists(golden): raise RuntimeError("Failed to find the orignal geometry '%s'" % golden) if not os.path.exists(outfile): raise RuntimeError("Failed to find the new geometry '%s'" % outfile) oldData = open(golden, 'r').readlines() newData = open(outfile, 'r').readlines() # do the "quick" check to see if anything has been changed diff = difflib.unified_diff(oldData, newData, golden, outfile, n=0) leftDiff = 0 rightDiff = 0 oldModified=None newModified=None for line in diff: if line.startswith('-') or line.startswith('+'): if not (line.startswith('---') or line.startswith('+++')): if 'last-modified' in line: modified = getModified(line) newline = line.replace(modified, "") if line.startswith('-'): oldModified = (modified, newline[1:]) else: newModified = (modified, newline[1:]) if oldModified is None or oldModified[1] != newModified[1]: leftDiff += 1 rightDiff += 1 else: if line.startswith('-'): leftDiff += 1 if line.startswith('+'): rightDiff += 1 # log the modified dates according to the file if oldModified is not None: logging.info(golden+" "+oldModified[0]) else: logging.info(golden+" oldModified is None") logging.info(outfile+" "+newModified[0]) # only print diff if there is a meaningful one if (leftDiff+rightDiff) > 0: if headerDecoration: print "========================================" print "%d lines removed and %d lines added" % (leftDiff, rightDiff) # rerun and write out diff = difflib.unified_diff(oldData, newData, golden, outfile) sys.stdout.writelines(diff) else: print "%s and %s match" % (os.path.split(golden)[1], \ os.path.split(outfile)[1])
def compareSnapshots(snapfile1, snapfile2): try: pkl_file = open(snapfile1, 'rb') dirs1 = pickle.load(pkl_file) files1 = pickle.load(pkl_file) pkl_file.close() pk2_file = open(snapfile2, 'rb') dirs2 = pickle.load(pk2_file) files2 = pickle.load(pk2_file) pk2_file.close() except: print "Problems encountered accessing snapshot files!" raw_input("\n\nPress [Enter] to continue...") return result_dirs = list(difflib.unified_diff(dirs1, dirs2)) result_files = list(difflib.unified_diff(files1, files2)) added_dirs = [] removed_dirs = [] added_files = [] removed_files = [] for result in result_files: if result.endswith("\n") == -1: if result.startswith('+'): resultadd = result.strip('+') added_files.append(resultadd) elif result.startswith('-'): resultsubtract = result.strip('-') removed_files.append(resultsubtract) for result in result_dirs: if result.endswith("\n") == -1: if result.startswith('+'): resultadd = result.strip('+') added_dirs.append(resultadd) elif result.startswith('-'): resultsubtract = result.strip('-') removed_dirs.append(resultsubtract) print "\n\nAdded Directories:\n" printList(added_dirs) print "\n\nAdded Files:\n" printList(added_files) print "\n\nRemoved Directories:\n" printList(removed_dirs) print "\n\nRemoved Files:\n" printList(removed_files) raw_input("\n\nPress [Enter] to continue...")
def compareGeom(golden, outfile): if not os.path.exists(golden): logging.warning(" Failed to find the original geometry " + golden + " - not comparing") return if not os.path.exists(outfile): logging.warning(" Failed to find the new geometry " + outfile) return oldData = open(golden).readlines() newData = open(outfile).readlines() # do the "quick" check to see if anything has been changed diff = difflib.unified_diff(oldData, newData, golden, outfile, n=0) leftDiff = 0 rightDiff = 0 oldModified = None oldDate = None newDate = None for line in diff: if line.startswith('-<instrument'): oldDate, oldModified = getModified(line) elif line.startswith('+<instrument'): newDate, newModified = getModified(line) if oldModified != newModified: leftDiff += 1 rightDiff += 1 else: if line.startswith('-') and not line.startswith('---'): leftDiff += 1 if line.startswith('+') and not line.startswith('+++'): rightDiff += 1 if oldDate is not None: logging.info(" Compare " + golden + " " + oldDate) else: logging.info(golden+" oldDate is None") if newDate is not None: logging.info(" With " + outfile + " " + newDate) else: logging.info(outfile+" newDate is None") # only print diff if there is a meaningful one if (leftDiff+rightDiff) > 0: logging.info(" ========================================") logging.info(" " + str(leftDiff) + " line(s) removed and " + str(rightDiff) + " line(s) added") # rerun and write out diff = difflib.unified_diff(oldData, newData, golden, outfile) sys.stdout.writelines(diff) else: logging.info(" " + os.path.split(golden)[1] + " and " + os.path.split(outfile)[1] + " match")
def main(argv=None): if argv is None: argv = sys.argv parser = optparse.OptionParser() parser.add_option( '-d', '--diff', action='store_true', dest='diff', default=False, help= 'create examples for the .in file and output a unified diff against the related .xml file' ) parser.add_option( '-r', '--run', action='store_true', dest='run', default=False, help='create examples for the .in file, but do not change any files') parser.add_option( '-s', '--simple_diff', action='store_true', dest='simple', default=False, help='use a simpler output for the diff mode (no unified diff!)') parser.add_option( '-u', '--update', action='store_true', dest='update', default=False, help='create examples for the .in file and update the related .xml file' ) opts, args = parser.parse_args(argv[1:]) if opts.diff: import StringIO import difflib if not args: args = glob.glob('doc/user/*.in') for arg in sorted(args): diff = None s = StringIO.StringIO() process(arg, s) filename = arg[:-2] + 'xml' try: fxml = open(filename, 'r') xmlcontent = fxml.read() fxml.close() if opts.simple: diff = list( difflib.context_diff(xmlcontent.splitlines(), s.getvalue().splitlines(), fromfile=arg, tofile=filename)) else: diff = list( difflib.unified_diff(xmlcontent.splitlines(), s.getvalue().splitlines(), fromfile=arg, tofile=filename, lineterm='')) except EnvironmentError, e: sys.stderr.write('%s: %s\n' % (filename, e)) s.close() if diff: print "%s:" % arg print '\n'.join(diff)
def main(): env_file, rest = sys.argv[1], sys.argv[2:] # Parse some argument flags. header_dir = None resource_dir = None input_file = None for i, arg in enumerate(rest): if arg == '-h' and len(rest) > i + 1: assert header_dir == None header_dir = rest[i + 1] elif arg == '-r' and len(rest) > i + 1: assert resource_dir == None resource_dir = rest[i + 1] elif arg.endswith('.mc') or arg.endswith('.man'): assert input_file == None input_file = arg # Copy checked-in outputs to final location. THIS_DIR = os.path.abspath(os.path.dirname(__file__)) assert header_dir == resource_dir source = os.path.join(THIS_DIR, "..", "..", "third_party", "win_build_output", re.sub(r'^(?:[^/]+/)?gen/', 'mc/', header_dir)) distutils.dir_util.copy_tree(source, header_dir, preserve_times=False) # On non-Windows, that's all we can do. if sys.platform != 'win32': return # On Windows, run mc.exe on the input and check that its outputs are # identical to the checked-in outputs. # Read the environment block from the file. This is stored in the format used # by CreateProcess. Drop last 2 NULs, one for list terminator, one for # trailing vs. separator. env_pairs = open(env_file).read()[:-2].split('\0') env_dict = dict([item.split('=', 1) for item in env_pairs]) extension = os.path.splitext(input_file)[1] if extension in ['.man', '.mc']: # For .man files, mc's output changed significantly from Version 10.0.15063 # to Version 10.0.16299. We should always have the output of the current # default SDK checked in and compare to that. Early out if a different SDK # is active. This also happens with .mc files. # TODO(thakis): Check in new baselines and compare to 16299 instead once # we use the 2017 Fall Creator's Update by default. mc_help = subprocess.check_output(['mc.exe', '/?'], env=env_dict, stderr=subprocess.STDOUT, shell=True) version = re.search(r'Message Compiler\s+Version (\S+)', mc_help).group(1) if version != '10.0.15063': return # mc writes to stderr, so this explicitly redirects to stdout and eats it. try: tmp_dir = tempfile.mkdtemp() delete_tmp_dir = True if header_dir: rest[rest.index('-h') + 1] = tmp_dir header_dir = tmp_dir if resource_dir: rest[rest.index('-r') + 1] = tmp_dir resource_dir = tmp_dir # This needs shell=True to search the path in env_dict for the mc # executable. subprocess.check_output(['mc.exe'] + rest, env=env_dict, stderr=subprocess.STDOUT, shell=True) # We require all source code (in particular, the header generated here) to # be UTF-8. jinja can output the intermediate .mc file in UTF-8 or UTF-16LE. # However, mc.exe only supports Unicode via the -u flag, and it assumes when # that is specified that the input is UTF-16LE (and errors out on UTF-8 # files, assuming they're ANSI). Even with -u specified and UTF16-LE input, # it generates an ANSI header, and includes broken versions of the message # text in the comment before the value. To work around this, for any invalid # // comment lines, we simply drop the line in the header after building it. # Also, mc.exe apparently doesn't always write #define lines in # deterministic order, so manually sort each block of #defines. if header_dir: header_file = os.path.join( header_dir, os.path.splitext(os.path.basename(input_file))[0] + '.h') header_contents = [] with open(header_file, 'rb') as f: define_block = [] # The current contiguous block of #defines. for line in f.readlines(): if line.startswith('//') and '?' in line: continue if line.startswith('#define '): define_block.append(line) continue # On the first non-#define line, emit the sorted preceding #define # block. header_contents += sorted(define_block, key=lambda s: s.split()[-1]) define_block = [] header_contents.append(line) # If the .h file ends with a #define block, flush the final block. header_contents += sorted(define_block, key=lambda s: s.split()[-1]) with open(header_file, 'wb') as f: f.write(''.join(header_contents)) # mc.exe invocation and post-processing are complete, now compare the output # in tmp_dir to the checked-in outputs. diff = filecmp.dircmp(tmp_dir, source) if diff.diff_files or set(diff.left_list) != set(diff.right_list): print 'mc.exe output different from files in %s, see %s' % ( source, tmp_dir) diff.report() for f in diff.diff_files: if f.endswith('.bin'): continue fromfile = os.path.join(source, f) tofile = os.path.join(tmp_dir, f) print ''.join( difflib.unified_diff( open(fromfile, 'U').readlines(), open(tofile, 'U').readlines(), fromfile, tofile)) delete_tmp_dir = False sys.exit(1) except subprocess.CalledProcessError as e: print e.output sys.exit(e.returncode) finally: if os.path.exists(tmp_dir) and delete_tmp_dir: shutil.rmtree(tmp_dir)
def testme(testDir, testObj, exeext): """Runs a single test, comparing output and RC to expected output and RC Raises an error if input can't be read, executable fails, or output/RC are not as expected. Error is caught by tester() and reported """ # Get the exec names and arguments execprog = testObj['exec'] + exeext execargs = testObj['args'] execrun = [execprog] + execargs # Read the input data (if there is any) stdinCfg = None inputData = None if "input" in testObj: filename = testDir + "/" + testObj['input'] inputData = open(filename).read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) outputFn = None outputData = None if "output_cmp" in testObj: outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][ 1:] # output type from file extension (determines how to compare) try: outputData = open(testDir + "/" + outputFn).read() except: logging.error("Output file " + outputFn + " can not be opened") raise if not outputData: logging.error("Output data missing for " + outputFn) raise Exception # Run the test proc = subprocess.Popen(execrun, stdin=stdinCfg, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) try: outs = proc.communicate(input=inputData) except OSError: logging.error("OSError, Failed to execute " + execprog) raise if outputData: # Parse command output and expected output try: a_parsed = parse_output(outs[0], outputType) except Exception as e: logging.error('Error parsing command output as %s: %s' % (outputType, e)) raise try: b_parsed = parse_output(outputData, outputType) except Exception as e: logging.error('Error parsing expected output %s as %s: %s' % (outputFn, outputType, e)) raise # Compare data if a_parsed != b_parsed: data_mismatch_message = "Output data mismatch for " + outputFn + " (format " + outputType + ")" logging.error(data_mismatch_message) logging.info(outs[0]) raise Exception # Compare formatting if outs[0] != outputData: formatting_mismatch_message = "Output formatting mismatch for " + outputFn + ":\n" formatting_mismatch_message += "".join( difflib.unified_diff(outputData.splitlines(True), outs[0].splitlines(True), fromfile=outputFn, tofile="returned")) logging.error(formatting_mismatch_message) raise Exception # Compare the return code to the expected return code wantRC = 0 if "return_code" in testObj: wantRC = testObj['return_code'] if proc.returncode != wantRC: logging.error("Return code mismatch for " + outputFn) raise Exception if "error_txt" in testObj: want_error = testObj["error_txt"] # Compare error text # TODO: ideally, we'd compare the strings exactly and also assert # That stderr is empty if no errors are expected. However, bitcoin-tx # emits DISPLAY errors when running as a windows application on # linux through wine. Just assert that the expected error text appears # somewhere in stderr if want_error not in outs[1]: logging.error("Error mismatch:\n" + "Expected: " + want_error + "\nReceived: " + outs[1].rstrip()) raise Exception
def review_config(self): diff_lines = '' current_rules = '' current_modified_since_prev_setup = False interactive = True if os.path.isfile(osetupcons.FileLocations.SYSCONFIG_IPTABLES): with open(osetupcons.FileLocations.SYSCONFIG_IPTABLES, 'r') as current: current_rules = current.read().splitlines() if os.path.isfile(osetupcons.FileLocations.OVIRT_IPTABLES_EXAMPLE): with open(osetupcons.FileLocations.OVIRT_IPTABLES_EXAMPLE, 'r') as prev_setup_example: prev_setup_rules = prev_setup_example.read().splitlines() diff_prev_cur = difflib.unified_diff( current_rules, prev_setup_rules, lineterm='', ) diff_lines = '\n'.join(diff_prev_cur) if len(diff_lines) > 0: current_modified_since_prev_setup = True diff = difflib.unified_diff( current_rules, self._get_rules().splitlines(), lineterm='', fromfile=_('current'), tofile=_('proposed'), ) diff_lines = '\n'.join(diff) if len(diff_lines) > 0: if current_modified_since_prev_setup: self.logger.warning( _("It seams that previously generated iptables " "configuration was manually edited,\n" "please carefully review the proposed " "configuration")) if self.environment[ osetupcons.ConfigEnv.FIREWALL_CHANGES_REVIEW] is None: self.environment[ osetupcons.ConfigEnv. FIREWALL_CHANGES_REVIEW] = dialog.queryBoolean( dialog=self.plugin.dialog, name='OVESETUP_REVIEW_IPTABLES_CHANGES', note=_('Generated iptables rules are different ' 'from current ones.\n' 'Do you want to review them? ' '(@VALUES@) [@DEFAULT@]: '), prompt=True, true=_('Yes'), false=_('No'), default=False, ) else: interactive = False if self.environment[osetupcons.ConfigEnv. FIREWALL_CHANGES_REVIEW] and interactive: confirmed = dialog.queryBoolean( dialog=self.plugin.dialog, name='OVESETUP_CONFIRM_IPTABLES_CHANGES', note=_('Please review the changes:\n\n' '{diff}\n\n' 'Do you want to proceed with firewall ' 'configuration? ' '(@VALUES@) [@DEFAULT@]: ').format( diff=diff_lines), prompt=True, true=_('Yes'), false=_('No'), default=True, ) if not confirmed: raise RuntimeError( _('iptables proposed configuration ' 'was rejected by user'))
def _notify_trunk_changes(self, changeset_attribs, branch_attribs): """Email owner and/or access-list users on changes to trunk.""" # Notify only if users' email addresses can be determined conf = ResourceLocator.default().get_conf() user_tool_name = conf.get_value(["rosa-svn", "user-tool"]) if not user_tool_name: return notify_who_str = conf.get_value( ["rosa-svn", "notify-who-on-trunk-commit"], "") if not notify_who_str.strip(): return notify_who = shlex.split(notify_who_str) # Build the message text info_file_path = "%s/trunk/%s" % ("/".join( branch_attribs["sid"]), self.INFO_FILE) text = "" for changed_line in branch_attribs["changed_lines"]: text += changed_line # For suite info file change, add diff as well if (changed_line[4:].strip() == info_file_path and branch_attribs["status_info_file"] == self.ST_MODIFIED): old_strio = StringIO() metomi.rose.config.dump(branch_attribs["old_info"], old_strio) new_strio = StringIO() metomi.rose.config.dump(branch_attribs["info"], new_strio) for diff_line in unified_diff( old_strio.getvalue().splitlines(True), new_strio.getvalue().splitlines(True), "@%d" % (int(changeset_attribs["revision"]) - 1), "@%d" % (int(changeset_attribs["revision"]))): text += " " * 4 + diff_line # Determine who to notify users = set() for key in ["old_info", "info"]: if branch_attribs[key] is not None: info_conf = branch_attribs[key] if "owner" in notify_who: users.add(info_conf.get_value(["owner"])) if "access-list" in notify_who: users.update( info_conf.get_value(["access-list"], "").split()) users.discard("*") # Determine email addresses user_tool = self.usertools_manager.get_handler(user_tool_name) if "author" in notify_who: users.add(changeset_attribs["author"]) else: users.discard(changeset_attribs["author"]) emails = sorted(user_tool.get_emails(users)) if not emails: return # Send notification msg = MIMEText(text) msg.set_charset("utf-8") msg["From"] = conf.get_value(["rosa-svn", "notification-from"], "notications@" + socket.getfqdn()) msg["To"] = ", ".join(emails) msg["Subject"] = "%s-%s/trunk@%d" % ( changeset_attribs["prefix"], branch_attribs["sid"], int(changeset_attribs["revision"])) smtp_host = conf.get_value(["rosa-svn", "smtp-host"], default="localhost") smtp = SMTP(smtp_host) smtp.sendmail(msg["From"], emails, msg.as_string()) smtp.quit()
def diff_not_equal(path, want, have): want_lines, have_lines = want.splitlines(), have.splitlines() diff = "\n".join(difflib.unified_diff(want_lines, have_lines, fromfile=path, tofile="running", lineterm="")) return Difference("Diff:\n%s", path, diff)
def __changeset(request, repo, rev): """ display the file history ``name`` the name of the repository ``rev`` the revision in the changesets history """ ctx = repo.get_context().repository[rev] to_rev = rev files = [] for f in ctx.files(): mimetype = mimetypes.guess_type(f) try: fc = ctx[f] if mimetype[0] is not None: if mimetype[0] not in BINARY_MIMETYPES: data1 = fc.data().split('\n') from_rev = int(rev) - int(fc.filerev()) ctx2 = repo.get_context().repository[from_rev] data2 = ctx2.filectx(f).data().split('\n') diff = difflib.unified_diff(data1, data2, fromfile=f + '@' + str(from_rev), tofile=f + '@' + str(to_rev)) else: diff = None else: diff = None files.append({ 'name': f, 'size': len(fc.data()), 'mimetype': mimetype[0], 'diff': diff }) except: files.append({'name': f, 'size': -1}) if 'HTTP_REFERER' in request.META: referer = request.META['HTTP_REFERER'] else: referer = reverse('hg-repo-action', kwargs={ 'name': repo.name, 'action': 'changesets' }) return render_to_response('django_hg/changeset.html', { 'ctx': { "user": ctx.user(), "description": ctx.description(), "time": datetime.fromtimestamp(ctx.date()[0]).time(), "date": date.fromtimestamp(ctx.date()[0]), "hash": ctx, "rev": rev, "files_count": len(ctx.files()) }, 'files': files, 'repo': repo, 'referer': referer, 'rev': rev, 'groups': _get_left_side_cont(), }, context_instance=RequestContext(request))
def diff(source_data, target_data, source_channel, target_channel): return list(unified_diff(source_data, target_data, source_channel, target_channel))
def update_copyright(fn_path, year, pretend=False): """ Check file for a Copyright statement, and update its year. The patterns used for replacing copyrights are taken from echangelog. Only the first lines of each file that start with a hash ('#') are considered, until a line is found that doesn't start with a hash. Files are read and written in binary mode, so that this function will work correctly with files encoded in any character set, as long as the copyright statements consist of plain ASCII. @param fn_path: file path @type str @param year: current year @type str @param pretend: pretend mode @type bool @rtype: bool @return: True if copyright update was needed, False otherwise """ try: fn_hdl = io.open(_unicode_encode(fn_path, encoding=_encodings['fs'], errors='strict'), mode='rb') except EnvironmentError: return orig_header = [] new_header = [] for line in fn_hdl: line_strip = line.strip() orig_header.append(line) if not line_strip or line_strip[:1] != b'#': new_header.append(line) break line = update_copyright_year(year, line) new_header.append(line) difflines = 0 for diffline in difflib.unified_diff( [_unicode_decode(diffline) for diffline in orig_header], [_unicode_decode(diffline) for diffline in new_header], fromfile=fn_path, tofile=fn_path, n=0): util.writemsg_stdout(diffline, noiselevel=-1) difflines += 1 util.writemsg_stdout("\n", noiselevel=-1) # unified diff has three lines to start with if difflines > 3 and not pretend: # write new file with changed header f, fnnew_path = mkstemp() f = io.open(f, mode='wb') for line in new_header: f.write(line) for line in fn_hdl: f.write(line) f.close() try: fn_stat = os.stat(fn_path) except OSError: fn_stat = None shutil.move(fnnew_path, fn_path) if fn_stat is None: util.apply_permissions(fn_path, mode=0o644) else: util.apply_stat_permissions(fn_path, fn_stat) fn_hdl.close() return difflines > 3
def system(name, **kwargs): """ Ensure that global network settings are configured properly. name Custom name to represent this configuration change. kwargs The global parameters for the system. """ ret = { "name": name, "changes": {}, "result": True, "comment": "Global network settings are up to date.", } apply_net_settings = False kwargs["test"] = __opts__["test"] # Build global network settings try: old = __salt__["ip.get_network_settings"]() new = __salt__["ip.build_network_settings"](**kwargs) if __opts__["test"]: if old == new: return ret if not old and new: ret["result"] = None ret["comment"] = "Global network settings are set to be added." return ret elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["result"] = None ret["comment"] = ( "Global network settings are set to be " "updated:\n{}".format("\n".join(diff)) ) return ret if not old and new: apply_net_settings = True ret["changes"]["network_settings"] = "Added global network settings." elif old != new: diff = difflib.unified_diff(old, new, lineterm="") apply_net_settings = True ret["changes"]["network_settings"] = "\n".join(diff) except AttributeError as error: ret["result"] = False ret["comment"] = str(error) return ret except KeyError as error: ret["result"] = False ret["comment"] = str(error) return ret # Apply global network settings if apply_net_settings: try: __salt__["ip.apply_network_settings"](**kwargs) except AttributeError as error: ret["result"] = False ret["comment"] = str(error) return ret return ret
def routes(name, **kwargs): """ Manage network interface static routes. name Interface name to apply the route to. kwargs Named routes """ ret = { "name": name, "changes": {}, "result": True, "comment": "Interface {} routes are up to date.".format(name), } apply_routes = False if "test" not in kwargs: kwargs["test"] = __opts__.get("test", False) # Build interface routes try: old = __salt__["ip.get_routes"](name) new = __salt__["ip.build_routes"](name, **kwargs) if kwargs["test"]: if old == new: return ret if not old and new: ret["result"] = None ret["comment"] = "Interface {} routes are set to be added.".format(name) return ret elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["result"] = None ret["comment"] = ( "Interface {} routes are set to be " "updated:\n{}".format(name, "\n".join(diff)) ) return ret if not old and new: apply_routes = True ret["comment"] = "Interface {} routes added.".format(name) ret["changes"]["network_routes"] = "Added interface {} routes.".format(name) elif old != new: diff = difflib.unified_diff(old, new, lineterm="") apply_routes = True ret["comment"] = "Interface {} routes updated.".format(name) ret["changes"]["network_routes"] = "\n".join(diff) except AttributeError as error: ret["result"] = False ret["comment"] = str(error) return ret # Apply interface routes if apply_routes: try: __salt__["ip.apply_network_settings"](**kwargs) except AttributeError as error: ret["result"] = False ret["comment"] = str(error) return ret return ret
def test_build( compiled, format, available_targets, generate_reference, update_reference, archive_differences, ): name = compiled scrub = SCRUBBERS[format] output_pattern = OUTPUT_PATTERN[format] assert_equals = ASSERT_EQUALS.get(format, None) encoding = "utf8" if format == "html" and name.startswith("html-encoding-"): encoding = re.match("^html-encoding-(.*)$", name).group(1) os.chdir(os.path.join(basedir, name)) assert run(["make", format]) if generate_reference: # pragma: no cover generate_reference_data(output_pattern) whole_diff_output = [] for coverage_file, reference_file in find_reference_files(output_pattern): with io.open(coverage_file, encoding=encoding) as f: coverage = scrub(f.read()) with io.open(reference_file, encoding=encoding) as f: reference = scrub(f.read()) try: if assert_equals is not None: assert_equals(reference, coverage) else: diff_out = list( difflib.unified_diff( reference.splitlines(keepends=True), coverage.splitlines(keepends=True), fromfile=reference_file, tofile=coverage_file, )) diff_is_empty = len(diff_out) == 0 assert diff_is_empty, "".join(diff_out) except Exception as e: # pragma: no cover whole_diff_output += " " + str(e) + "\n" if update_reference: reference_file = update_reference_data(coverage_file, reference_file) if archive_differences: archive_difference_data(name, coverage_file, reference_file) if generate_reference or update_reference: # pragma: no cover remove_duplicate_data(encoding, scrub, coverage, coverage_file, reference_file) diff_is_empty = len(whole_diff_output) == 0 assert diff_is_empty, "Diff output:\n" + "".join(whole_diff_output) # some tests require additional cleanup after each test if "clean-each" in available_targets: # pragma: no cover assert run(["make", "clean-each"]) os.chdir(basedir)
def main(argv): parser = argparse.ArgumentParser(description='Generate source code for this repository') parser.add_argument('registry', metavar='REGISTRY_PATH', help='path to the Vulkan-Headers registry directory') group = parser.add_mutually_exclusive_group() group.add_argument('-i', '--incremental', action='store_true', help='only update repo files that change') group.add_argument('-v', '--verify', action='store_true', help='verify repo files match generator output') args = parser.parse_args(argv) gen_cmds = [*[[common_codegen.repo_relative('scripts/lvl_genvk.py'), '-registry', os.path.abspath(os.path.join(args.registry, 'vk.xml')), '-quiet', filename] for filename in ["chassis.cpp", "chassis.h", "chassis_dispatch_helper.h", "layer_chassis_dispatch.cpp", "layer_chassis_dispatch.h", "object_tracker.cpp", "object_tracker.h", "parameter_validation.cpp", "parameter_validation.h", "synchronization_validation_types.cpp", "synchronization_validation_types.h", "thread_safety.cpp", "thread_safety.h", "vk_dispatch_table_helper.h", "vk_enum_string_helper.h", "vk_extension_helper.h", "vk_layer_dispatch_table.h", "vk_object_types.h", "vk_safe_struct.cpp", "vk_safe_struct.h", "lvt_function_pointers.cpp", "lvt_function_pointers.h", "vk_typemap_helper.h", "best_practices.h", "best_practices.cpp", "spirv_validation_helper.cpp", "corechecks_optick_instrumentation.cpp", "corechecks_optick_instrumentation.h", "command_counter_helper.cpp", "command_counter_helper.h"]], [common_codegen.repo_relative('scripts/vk_validation_stats.py'), os.path.abspath(os.path.join(args.registry, 'validusage.json')), '-export_header'], [common_codegen.repo_relative('scripts/external_revision_generator.py'), '--json_file', common_codegen.repo_relative('scripts/known_good.json'), '--json_keys', 'repos,0,commit', '-s', 'SPIRV_TOOLS_COMMIT_ID', '-o', 'spirv_tools_commit_id.h']] repo_dir = common_codegen.repo_relative('layers/generated') # get directory where generators will run if args.verify or args.incremental: # generate in temp directory so we can compare or copy later temp_obj = tempfile.TemporaryDirectory(prefix='VulkanVL_generated_source_') temp_dir = temp_obj.name gen_dir = temp_dir else: # generate directly in the repo gen_dir = repo_dir # run each code generator for cmd in gen_cmds: print(' '.join(cmd)) try: subprocess.check_call([sys.executable] + cmd, cwd=gen_dir) except Exception as e: print('ERROR:', str(e)) return 1 # optional post-generation steps if args.verify: # compare contents of temp dir and repo temp_files = set(os.listdir(temp_dir)) repo_files = set(os.listdir(repo_dir)) files_match = True for filename in sorted((temp_files | repo_files) - set(verify_exclude)): temp_filename = os.path.join(temp_dir, filename) repo_filename = os.path.join(repo_dir, filename) if filename not in repo_files: print('ERROR: Missing repo file', filename) files_match = False elif filename not in temp_files: print('ERROR: Missing generator for', filename) files_match = False elif not filecmp.cmp(temp_filename, repo_filename, shallow=False): print('ERROR: Repo files do not match generator output for', filename) files_match = False # print line diff on file mismatch with open(temp_filename) as temp_file, open(repo_filename) as repo_file: print(''.join(difflib.unified_diff(temp_file.readlines(), repo_file.readlines(), fromfile='temp/' + filename, tofile= 'repo/' + filename))) # return code for test scripts if files_match: print('SUCCESS: Repo files match generator output') return 0 return 1 elif args.incremental: # copy missing or differing files from temp directory to repo for filename in os.listdir(temp_dir): temp_filename = os.path.join(temp_dir, filename) repo_filename = os.path.join(repo_dir, filename) if not os.path.exists(repo_filename) or \ not filecmp.cmp(temp_filename, repo_filename, shallow=False): print('update', repo_filename) shutil.copyfile(temp_filename, repo_filename) return 0
def format_file( notebook_path: PathLike, mode: black.FileMode, check: bool, diff: bool, compact_diff: bool, assert_consistent_execution: bool, ): with open(notebook_path) as fd: nb = nbf.read(fd, as_version=4) cells_errored = 0 cells_changed = 0 cells_unchanged = 0 current_execution_count = 1 for i, cell in enumerate(nb['cells']): if cell['cell_type'] != 'code': continue # check execution consistency if ( assert_consistent_execution and cell['execution_count'] != current_execution_count ): if check: cells_errored += 1 continue else: raise RuntimeError( f'[{notebook_path}] Cell {i} has inconsistent execution count' ) current_execution_count += 1 # format code orig_source = cell['source'] # check whether we should really process cell if any(orig_source.startswith(f'%%{magic}') for magic in SKIPPABLE_MAGIC_CODES): continue # black expects empty line at end of non-empty file # for notebook cells, this does not make sense if len(orig_source) > 0: orig_source += '\n' # Jupyter cell magic can mess up black # TODO: this is a bad hack orig_source = re.sub('^%', '#%#jupylint#', orig_source, flags=re.M) orig_source = re.sub('^!', '#!#jupylint#', orig_source, flags=re.M) try: fmted_source = black.format_str(orig_source, mode=mode) except black.InvalidInput as e: if check: print(f'[{notebook_path}] Error while formatting cell {i}: {e}') cells_errored += 1 continue else: raise RuntimeError( f'[{notebook_path}] Error while formatting cell {i}: {e}' ) if orig_source != fmted_source: fmted_source = re.sub('^#%#jupylint#', '%', fmted_source, flags=re.M) fmted_source = re.sub('^#!#jupylint#', '!', fmted_source, flags=re.M) if compact_diff: diff_result = difflib.unified_diff( orig_source.splitlines(keepends=True), fmted_source.splitlines(keepends=True), fromfile=f'{notebook_path} - Cell {i} (original)', tofile=f'{notebook_path} - Cell {i} (formatted)', ) elif diff: diff_result = difflib.ndiff( orig_source.splitlines(keepends=True), fmted_source.splitlines(keepends=True), ) if compact_diff or diff: diff_str = ''.join(diff_result) print(diff_str) if fmted_source.endswith('\n'): # remove dummy newline we added earlier fmted_source = fmted_source[:-1] fmted_cell = nbf.v4.new_code_cell(fmted_source) nb['cells'][i] = fmted_cell cells_changed += 1 else: cells_unchanged += 1 if cells_errored == 0 and not check and not compact_diff and not diff: with open(notebook_path, 'w') as fd: nbf.write(nb, fd) if check: if not diff and not compact_diff: print(notebook_path) if cells_errored > 0: print(f'{cells_errored} cell(s) raised parsing errors 🤕') if cells_changed > 0: print(f'{cells_changed} cell(s) would be changed 😬') if cells_unchanged > 0: print(f'{cells_unchanged} cell(s) would be left unchanged 🎉') print() return cells_errored, cells_changed, cells_unchanged
def _AssertGolden(expected_lines, actual_lines): expected = list(expected_lines) actual = list(l + '\n' for l in actual_lines) assert actual == expected, ( 'Did not match .golden.\n' + ''.join(difflib.unified_diff(expected, actual, 'expected', 'actual')))
def __svnDiff(self, path, content, modified): """Performs diff. The given content vs. the current version in repository """ # Get the SVN content first client = self.getSVNClient(self.getSettings()) try: localStatus = self.getLocalStatus(path) localRevisionNumber = getLocalRevisionNumber(client, path) reposRevisionNumber = getReposRevisionNumber(client, path) if reposRevisionNumber is None: logging.info("The path " + path + " does not exist in repository") return localAtLeft = False if localStatus == IND_UPTODATE: if localRevisionNumber < reposRevisionNumber: localAtLeft = True repositoryVersion = client.cat(path) # Calculate difference if localAtLeft: diff = difflib.unified_diff(content.splitlines(), repositoryVersion.splitlines(), n=5) else: diff = difflib.unified_diff(repositoryVersion.splitlines(), content.splitlines(), n=5) nodiffMessage = path + " has no difference to the " \ "repository at revision HEAD" if modified: nodiffMessage = "Editing buffer with " + nodiffMessage if diff is None: logging.info(nodiffMessage) return # There are changes, so replace the text and tell about the changes diffAsText = '\n'.join(list(diff)) if diffAsText.strip() == '': logging.info(nodiffMessage) return if modified: localSpec = "editing buffer with " + \ os.path.basename(path) + " based on rev." + \ str(localRevisionNumber) else: localSpec = "local " + os.path.basename(path) + \ " (rev." + str(localRevisionNumber) + ")" reposSpec = "repository at revision HEAD (rev." + \ str(reposRevisionNumber) + ")" if localAtLeft: diffAsText = diffAsText.replace("--- ", "--- " + localSpec, 1) diffAsText = diffAsText.replace("+++ ", "+++ " + reposSpec, 1) else: diffAsText = diffAsText.replace("+++ ", "+++ " + localSpec, 1) diffAsText = diffAsText.replace("--- ", "--- " + reposSpec, 1) except Exception as excpt: logging.error(str(excpt)) return self.ide.mainWindow.showDiff(diffAsText, "SVN diff for " + path)
def show_diff(self, location: Tuple[str, int] = None) -> List[str]: new_lines = self.read_file(self.filename) old_filename = self.options.get('diff') old_lines = self.read_file(old_filename) diff = unified_diff(old_lines, new_lines, old_filename, self.filename) return list(diff)
def print_diff(store1, store2): store1_lines = bytes(store1).decode(store1.encoding).split('\n') store2_lines = bytes(store2).decode(store2.encoding).split('\n') for line in difflib.unified_diff(store1_lines, store2_lines): print(line)
def managed(name, enabled=True, **kwargs): """ Ensure that the named interface is configured properly. name The name of the interface to manage type : eth Type of interface and configuration .. versionchanged:: 3002 enabled Designates the state of this interface. """ # For this function we are purposefully overwriting a bif # to enhance the user experience. This does not look like # it will cause a problem. Just giving a heads up in case # it does create a problem. ret = { "name": name, "changes": {}, "result": True, "comment": "Interface {} is up to date.".format(name), } if "test" not in kwargs: kwargs["test"] = __opts__.get("test", False) # set ranged status apply_ranged_setting = False # Pull interface type out of kwargs iface_type = str(kwargs.pop("type", "eth")) if "addr" in kwargs: hwaddr = kwargs.pop("addr") msg = "'addr' is not a valid argument name, " if "hwaddr" not in kwargs: msg += "its value has been assigned to 'hwaddr' instead." kwargs["hwaddr"] = hwaddr else: msg += "it has been ignored in favor of 'hwaddr'." msg += " Update your SLS file to get rid of this warning." ret.setdefault("warnings", []).append(msg) # Build interface try: old = __salt__["ip.get_interface"](name) new = __salt__["ip.build_interface"](name, iface_type, enabled, **kwargs) if kwargs["test"]: if old == new: pass if not old and new: ret["result"] = None ret["comment"] = "Interface {} is set to be added.".format(name) elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["result"] = None ret["comment"] = "Interface {} is set to be updated:\n{}".format( name, "\n".join(diff) ) else: if not old and new: ret["comment"] = "Interface {} added.".format(name) ret["changes"]["interface"] = "Added network interface." apply_ranged_setting = True elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["comment"] = "Interface {} updated.".format(name) ret["changes"]["interface"] = "\n".join(diff) apply_ranged_setting = True except AttributeError as error: ret["result"] = False ret["comment"] = str(error) return ret # Debian based system can have a type of source # in the interfaces file, we don't ifup or ifdown it if iface_type == "source": return ret # Setup up bond modprobe script if required if iface_type == "bond" and "ip.get_bond" in __salt__: try: old = __salt__["ip.get_bond"](name) new = __salt__["ip.build_bond"](name, **kwargs) if kwargs["test"]: if not old and new: ret["result"] = None ret["comment"] = "Bond interface {} is set to be added.".format( name ) elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["result"] = None ret["comment"] = ( "Bond interface {} is set to be " "updated:\n{}".format(name, "\n".join(diff)) ) else: if not old and new: ret["comment"] = "Bond interface {} added.".format(name) ret["changes"]["bond"] = "Added bond {}.".format(name) apply_ranged_setting = True elif old != new: diff = difflib.unified_diff(old, new, lineterm="") ret["comment"] = "Bond interface {} updated.".format(name) ret["changes"]["bond"] = "\n".join(diff) apply_ranged_setting = True except AttributeError as error: # TODO Add a way of reversing the interface changes. ret["result"] = False ret["comment"] = str(error) return ret if kwargs["test"]: return ret # For Redhat/Centos ranged network if "range" in name: if apply_ranged_setting: try: ret["result"] = __salt__["service.restart"]("network") ret["comment"] = "network restarted for change of ranged interfaces" return ret except Exception as error: # pylint: disable=broad-except ret["result"] = False ret["comment"] = str(error) return ret ret["result"] = True ret["comment"] = "no change, passing it" return ret # Bring up/shutdown interface try: # Get Interface current status interfaces = salt.utils.network.interfaces() interface_status = False if name in interfaces: interface_status = interfaces[name].get("up") else: for iface in interfaces: if "secondary" in interfaces[iface]: for second in interfaces[iface]["secondary"]: if second.get("label", "") == name: interface_status = True if iface == "lo": if "inet" in interfaces[iface]: inet_data = interfaces[iface]["inet"] if len(inet_data) > 1: for data in inet_data: if data.get("label", "") == name: interface_status = True if "inet6" in interfaces[iface]: inet6_data = interfaces[iface]["inet6"] if len(inet6_data) > 1: for data in inet6_data: if data.get("label", "") == name: interface_status = True if enabled: if "noifupdown" not in kwargs: if interface_status: if ret["changes"]: # Interface should restart to validate if it's up __salt__["ip.down"](name, iface_type) __salt__["ip.up"](name, iface_type) ret["changes"][ "status" ] = "Interface {} restart to validate".format(name) else: __salt__["ip.up"](name, iface_type) ret["changes"]["status"] = "Interface {} is up".format(name) else: if "noifupdown" not in kwargs: if interface_status: __salt__["ip.down"](name, iface_type) ret["changes"]["status"] = "Interface {} down".format(name) except Exception as error: # pylint: disable=broad-except ret["result"] = False ret["comment"] = str(error) return ret # Try to enslave bonding interfaces after master was created if iface_type == "bond" and "noifupdown" not in kwargs: if "slaves" in kwargs and kwargs["slaves"]: # Check that there are new slaves for this master present_slaves = __salt__["cmd.run"]( ["cat", "/sys/class/net/{}/bonding/slaves".format(name)] ).split() desired_slaves = kwargs["slaves"].split() missing_slaves = set(desired_slaves) - set(present_slaves) # Enslave only slaves missing in master if missing_slaves: ifenslave_path = __salt__["cmd.run"](["which", "ifenslave"]).strip() if ifenslave_path: log.info( "Adding slaves '%s' to the master %s", " ".join(missing_slaves), name, ) cmd = [ifenslave_path, name] + list(missing_slaves) __salt__["cmd.run"](cmd, python_shell=False) else: log.error("Command 'ifenslave' not found") ret["changes"]["enslave"] = "Added slaves '{}' to master '{}'".format( " ".join(missing_slaves), name ) else: log.info( "All slaves '%s' are already added to the master %s" ", no actions required", " ".join(missing_slaves), name, ) if enabled and interface_status: # Interface was restarted, return return ret # Make sure that the network grains reflect any changes made here __salt__["saltutil.refresh_grains"]() return ret
def main(argv=sys.argv[1:]): config_file = os.path.join(os.path.dirname(__file__), 'configuration', 'ament_code_style.cfg') extensions = ['c', 'cc', 'cpp', 'cxx', 'h', 'hh', 'hpp', 'hxx'] parser = argparse.ArgumentParser( description='Check code style using uncrustify.', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-c', metavar='CFG', default=config_file, dest='config_file', help='The config file') parser.add_argument( '--linelength', metavar='N', type=int, help='The maximum line length (default: specified in the config file)') parser.add_argument( 'paths', nargs='*', default=[os.curdir], help='The files or directories to check. For directories files ending ' 'in %s will be considered.' % ', '.join(["'.%s'" % e for e in extensions])) parser.add_argument( '--exclude', nargs='*', default=[], help='Exclude specific file names and directory names from the check') parser.add_argument('--reformat', action='store_true', help='Reformat the files in place') # not using a file handle directly # in order to prevent leaving an empty file when something fails early parser.add_argument('--xunit-file', help='Generate a xunit compliant XML file') args = parser.parse_args(argv) if not os.path.exists(args.config_file): print("Could not config file '%s'" % args.config_file, file=sys.stderr) return 1 temp_config = None temp_path = None try: if args.linelength is not None: # check if different from config file config = ConfigParser() with open(args.config_file, 'r') as h: config_str = h.read() config.read_string('[DEFAULT]\n' + config_str) code_width = config['DEFAULT']['code_width'] code_width = int(re.split('[ \t#]', code_width, maxsplit=1)[0]) if args.linelength != code_width: # generate temporary config file with custom line length temp_config_fd, args.config_file = tempfile.mkstemp( prefix='uncrustify_') temp_config = os.fdopen(temp_config_fd, 'w') temp_config.write(config_str + '\ncode_width=%d' % args.linelength) temp_config.close() if args.xunit_file: start_time = time.time() files = get_files(args.paths, extensions, args.exclude) if not files: print('No files found', file=sys.stderr) return 1 uncrustify_bin = find_executable('uncrustify') if not uncrustify_bin: print("Could not find 'uncrustify' executable", file=sys.stderr) return 1 suffix = '.uncrustify' report = [] temp_path = tempfile.mkdtemp(prefix='uncrustify_') # invoke uncrustify on all files input_files = [os.path.abspath(f) for f in files] # on Windows uncrustify fails to concatenate # the absolute prefix path with the absolute input files # https://github.com/bengardner/uncrustify/issues/364 cwd = None if os.name == 'nt': cwd = os.path.commonprefix(input_files) if not os.path.isdir(cwd): cwd = os.path.dirname(cwd) assert os.path.isdir(cwd), \ 'Could not determine common prefix of input files' input_files = [os.path.relpath(f, start=cwd) for f in input_files] try: cmd = [ uncrustify_bin, '-c', args.config_file, '--prefix', temp_path, '--suffix', suffix ] cmd.extend(input_files) subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: if e.output: print(e.output.decode(), file=sys.stderr) print( "The invocation of 'uncrustify' failed with error code %d: %s" % (e.returncode, e), file=sys.stderr) return 1 if cwd: # input files are relative # prepend temp path, append suffix output_files = [ os.path.join(temp_path, f + suffix) for f in input_files ] else: # input files are absolute # remove leading slash, prepend temp path, append suffix output_files = [ os.path.join(temp_path, os.sep.join(f.split(os.sep)[1:]) + suffix) for f in input_files ] uncrustified_files = output_files i = 1 while True: # identify files which have changed since the latest uncrustify run changed_files = [] for input_filename, output_filename in zip(input_files, uncrustified_files): if cwd and not os.path.isabs(input_filename): input_filename = os.path.join(cwd, input_filename) if not filecmp.cmp(input_filename, output_filename): if output_filename == input_filename + suffix: # for repeated invocations # replace the previous uncrustified file os.rename(output_filename, input_filename) changed_files.append(input_filename) else: # after first invocation remove suffix # otherwise uncrustify behaves different output_filename_without_suffix = \ output_filename[:-len(suffix)] os.rename(output_filename, output_filename_without_suffix) changed_files.append(output_filename_without_suffix) if not changed_files: break # reinvoke uncrustify for previously changed files input_files = changed_files try: cmd = [ uncrustify_bin, '-c', args.config_file, '--suffix', suffix ] cmd.extend(input_files) subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: if e.output: print(e.output, file=sys.stderr) print( "The invocation of 'uncrustify' failed with error code %d: %s" % (e.returncode, e), file=sys.stderr) return 1 uncrustified_files = [f + suffix for f in input_files] i += 1 if i >= 5: print("'uncrustify' did not settle on a final result even " "after %d invocations" % i, file=sys.stderr) return 1 # compute diff for index, filename in enumerate(files): modified_filename = output_files[index] with open(filename, 'r', encoding='utf-8') as original_file: with open(modified_filename, 'r', encoding='utf-8') as modified_file: diff_lines = list( difflib.unified_diff(original_file.readlines(), modified_file.readlines(), fromfile=filename, tofile=filename + suffix, n=0)) report.append((filename, diff_lines)) if args.reformat: # overwrite original with reformatted file with (open(filename, 'wb')) as original_file: with open(modified_filename, 'rb') as modified_file: original_file.write(modified_file.read()) finally: if temp_config: os.remove(args.config_file) if temp_path and os.path.exists(temp_path): shutil.rmtree(temp_path) # output diffs for (filename, diff_lines) in report: if diff_lines: if not args.reformat: print("Code style divergence in file '%s':" % filename, file=sys.stderr) print('', file=sys.stderr) for line in diff_lines: print(line.rstrip('\r\n'), file=sys.stderr) print('', file=sys.stderr) else: print("Code style divergence in file '%s': reformatted file" % filename) else: print("No code style divergence in file '%s'" % filename) if not args.reformat: print('') # output summary error_count = sum([1 if r[1] else 0 for r in report]) if not error_count: print('No problems found') rc = 0 else: print('%d files with code style divergence' % error_count, file=sys.stderr) rc = 1 # generate xunit file if args.xunit_file: folder_name = os.path.basename(os.path.dirname(args.xunit_file)) file_name = os.path.basename(args.xunit_file) suffix = '.xml' if file_name.endswith(suffix): file_name = file_name[0:-len(suffix)] suffix = '.xunit' if file_name.endswith(suffix): file_name = file_name[0:-len(suffix)] testname = '%s.%s' % (folder_name, file_name) xml = get_xunit_content(report, testname, time.time() - start_time) path = os.path.dirname(os.path.abspath(args.xunit_file)) if not os.path.exists(path): os.makedirs(path) with open(args.xunit_file, 'w') as f: f.write(xml) return rc
# Go ! if len(args) != 2: parser.print_help() exit(1) # Open the 2 documents, diff only for ODT doc1 = odf_get_document(args[0]) doc2 = odf_get_document(args[1]) if doc1.get_type() != 'text' or doc2.get_type() != 'text': parser.print_help() exit(1) # Convert in text before the diff text1 = doc1.get_formatted_text(True).splitlines(True) text2 = doc2.get_formatted_text(True).splitlines(True) # Make the diff ! if options.ndiff: result = ndiff(text1, text2, None, None) result = [ line for line in result if not line.startswith(u' ') ] else: fromdate = ctime(stat(args[0]).st_mtime) todate = ctime(stat(args[1]).st_mtime) result = unified_diff(text1, text2, args[0], args[1], fromdate, todate) result = u''.join(result) encoding = stdout.encoding if stdout.encoding is not None else 'utf-8' result = result.encode(encoding) # And print it ! print result
result_set_1 = Path(sys.argv[1]) result_set_2 = Path(sys.argv[2]) config_1 = result_set_1 / "config" config_2 = result_set_2 / "config" # Look for the differences between configurations and output for c_file in config_files: text_1 = (config_1 / c_file).read_text() lines_1 = text_1.split("\n") text_2 = (config_2 / c_file).read_text() lines_2 = text_2.split("\n") print("Comparing " + c_file) for line in difflib.unified_diff(lines_1, lines_2, "file_1", "file_2", lineterm="\n"): print(line) # plot the error from each sequence error_path_1 = result_set_1 / 'plot_error' error_path_2 = result_set_2 / 'plot_error' for seq in sequences: for suffix in suffixes: error_1 = (error_path_1 / (seq + suffix)).read_text().splitlines() error_2 = (error_path_2 / (seq + suffix)).read_text().splitlines() x1 = [] y1 = [] x2 = [] y2 = []
def change(name, context=None, changes=None, lens=None, load_path=None, **kwargs): """ .. versionadded:: 2014.7.0 This state replaces :py:func:`~salt.states.augeas.setvalue`. Issue changes to Augeas, optionally for a specific context, with a specific lens. name State name context A file path, prefixed by ``/files``. Should resolve to an actual file (not an arbitrary augeas path). This is used to avoid duplicating the file name for each item in the changes list (for example, ``set bind 0.0.0.0`` in the example below operates on the file specified by ``context``). If ``context`` is not specified, a file path prefixed by ``/files`` should be included with the ``set`` command. The file path is examined to determine if the specified changes are already present. .. code-block:: yaml redis-conf: augeas.change: - context: /files/etc/redis/redis.conf - changes: - set bind 0.0.0.0 - set maxmemory 1G changes List of changes that are issued to Augeas. Available commands are ``set``, ``setm``, ``mv``/``move``, ``ins``/``insert``, and ``rm``/``remove``. lens The lens to use, needs to be suffixed with `.lns`, e.g.: `Nginx.lns`. See the `list of stock lenses <http://augeas.net/stock_lenses.html>`_ shipped with Augeas. .. versionadded:: 2016.3.0 load_path A list of directories that modules should be searched in. This is in addition to the standard load path and the directories in AUGEAS_LENS_LIB. Usage examples: Set the ``bind`` parameter in ``/etc/redis/redis.conf``: .. code-block:: yaml redis-conf: augeas.change: - changes: - set /files/etc/redis/redis.conf/bind 0.0.0.0 .. note:: Use the ``context`` parameter to specify the file you want to manipulate. This way you don't have to include this in the changes every time: .. code-block:: yaml redis-conf: augeas.change: - context: /files/etc/redis/redis.conf - changes: - set bind 0.0.0.0 - set databases 4 - set maxmemory 1G Augeas is aware of a lot of common configuration files and their syntax. It knows the difference between for example ini and yaml files, but also files with very specific syntax, like the hosts file. This is done with *lenses*, which provide mappings between the Augeas tree and the file. There are many `preconfigured lenses`_ that come with Augeas by default, and they specify the common locations for configuration files. So most of the time Augeas will know how to manipulate a file. In the event that you need to manipulate a file that Augeas doesn't know about, you can specify the lens to use like this: .. code-block:: yaml redis-conf: augeas.change: - lens: redis.lns - context: /files/etc/redis/redis.conf - changes: - set bind 0.0.0.0 .. note:: Even though Augeas knows that ``/etc/redis/redis.conf`` is a Redis configuration file and knows how to parse it, it is recommended to specify the lens anyway. This is because by default, Augeas loads all known lenses and their associated file paths. All these files are parsed when Augeas is loaded, which can take some time. When specifying a lens, Augeas is loaded with only that lens, which speeds things up quite a bit. .. _preconfigured lenses: http://augeas.net/stock_lenses.html A more complex example, this adds an entry to the services file for Zabbix, and removes an obsolete service: .. code-block:: yaml zabbix-service: augeas.change: - lens: services.lns - context: /files/etc/services - changes: - ins service-name after service-name[last()] - set service-name[last()] "zabbix-agent" - set "service-name[. = 'zabbix-agent']/port" 10050 - set "service-name[. = 'zabbix-agent']/protocol" tcp - set "service-name[. = 'zabbix-agent']/#comment" "Zabbix Agent service" - rm "service-name[. = 'im-obsolete']" - unless: grep "zabbix-agent" /etc/services .. warning:: Don't forget the ``unless`` here, otherwise it will fail on next runs because the service is already defined. Additionally you have to quote lines containing ``service-name[. = 'zabbix-agent']`` otherwise :mod:`augeas_cfg <salt.modules.augeas_cfg>` execute will fail because it will receive more parameters than expected. .. note:: Order is important when defining a service with Augeas, in this case it's ``port``, ``protocol`` and ``#comment``. For more info about the lens check `services lens documentation`_. .. _services lens documentation: http://augeas.net/docs/references/lenses/files/services-aug.html#Services.record """ ret = {"name": name, "result": False, "comment": "", "changes": {}} if not changes or not isinstance(changes, list): ret["comment"] = "'changes' must be specified as a list" return ret if load_path is not None: if not isinstance(load_path, list): ret["comment"] = "'load_path' must be specified as a list" return ret else: load_path = ":".join(load_path) filename = None if context is None: try: filename = _check_filepath(changes) except ValueError as err: ret["comment"] = "Error: {}".format(err) return ret else: filename = re.sub("^/files|/$", "", context) if __opts__["test"]: ret["result"] = True ret["comment"] = "Executing commands" if context: ret["comment"] += ' in file "{}":\n'.format(context) ret["comment"] += "\n".join(changes) return ret old_file = [] if filename is not None and os.path.isfile(filename): with salt.utils.files.fopen(filename, "r") as file_: old_file = [salt.utils.stringutils.to_unicode(x) for x in file_.readlines()] result = __salt__["augeas.execute"]( context=context, lens=lens, commands=changes, load_path=load_path ) ret["result"] = result["retval"] if ret["result"] is False: ret["comment"] = "Error: {}".format(result["error"]) return ret if filename is not None and os.path.isfile(filename): with salt.utils.files.fopen(filename, "r") as file_: new_file = [salt.utils.stringutils.to_unicode(x) for x in file_.readlines()] diff = "".join(difflib.unified_diff(old_file, new_file, n=0)) if diff: ret["comment"] = "Changes have been saved" ret["changes"] = {"diff": diff} else: ret["comment"] = "No changes made" else: ret["comment"] = "Changes have been saved" ret["changes"] = {"updates": changes} return ret
def compare_outputs(outputs, raise_error=True, ignore_keywords=[], ignore_hdus=[], ignore_fields=[], rtol=0.0, atol=0.0, input_path=[], docopy=True, results_root=None, verbose=True): """ Compare output with "truth" using appropriate diff routine; namely: * ``fitsdiff`` for FITS file comparisons. * ``unified_diff`` for ASCII products. Only after all elements of ``outputs`` have been processed will the method report any success or failure, with failure of any one comparison *not* preventing the rest of the comparisons to be performed. Parameters ---------- outputs : list of tuple or dict This list defines what outputs from running the test will be compared. Three distinct types of values as list elements are supported: * 2-tuple : ``(test output filename, truth filename)`` * 3-tuple : ``(test output filename, truth filename, HDU names)`` * dict : ``{'files': (output, truth), 'pars': {key: val}}`` If filename contains extension such as ``[hdrtab]``, it will be interpreted as specifying comparison of just that HDU. raise_error : bool Raise ``AssertionError`` if difference is found. ignore_keywords : list of str List of FITS header keywords to be ignored by ``FITSDiff`` and ``HDUDiff``. ignore_hdus : list of str List of FITS HDU names to ignore by ``FITSDiff``. This is only available for ``astropy>=3.1``. ignore_fields : list of str List FITS table column names to be ignored by ``FITSDiff`` and ``HDUDiff``. rtol, atol : float Relative and absolute tolerance to be used by ``FITSDiff`` and ``HDUDiff``. input_path : list or tuple A series of sub-directory names under :func:`get_bigdata_root` that leads to the path of the 'truth' files to be compared against. If not provided, it assumes that 'truth' is in the working directory. For example, with :func:`get_bigdata_root` pointing to ``/grp/test_data``, a file at:: /grp/test_data/pipeline/dev/ins/test_1/test_a.py would require ``input_path`` of:: ["pipeline", "dev", "ins", "test_1"] docopy : bool If `True`, 'truth' will be copied to output directory before comparison is done. results_root : str or `None` If not `None`, for every failed comparison, the test output is automatically renamed to the given 'truth' in the output directory and :func:`generate_upload_schema` will be called to generate a JSON scheme for Artifactory upload. If you do not need this functionality, use ``results_root=None``. verbose : bool Print extra info to screen. Returns ------- creature_report : str Report from FITS or ASCII comparator. This is part of error message if ``raise_error=True``. Examples -------- There are multiple use cases for this method, specifically related to how ``outputs`` are defined upon calling this method. The specification of the ``outputs`` can be any combination of the following patterns: 1. 2-tuple inputs:: outputs = [('file1.fits', 'file1_truth.fits')] This definition indicates that ``file1.fits`` should be compared as a whole with ``file1_truth.fits``. 2. 2-tuple inputs with extensions:: outputs = [('file1.fits[hdrtab]', 'file1_truth.fits[hdrtab]')] This definition indicates that only the HDRTAB extension from ``file1.fits`` will be compared to the HDRTAB extension from ``file1_truth.fits``. 3. 3-tuple inputs:: outputs = [('file1.fits', 'file1_truth.fits', ['primary', 'sci'])] This definition indicates that only the PRIMARY and SCI extensions should be compared between the two files. This creates a temporary ``HDUList`` object comprising only the given extensions for comparison. 4. Dictionary of inputs and parameters:: outputs = [{'files': ('file1.fits', 'file1_truth.fits'), 'pars': {'ignore_keywords': ['ROOTNAME']}}] This definition indicates that ROOTNAME will be ignored during the comparison between the files specified in ``'files'``. Any input parameter for ``FITSDiff`` or ``HDUDiff`` can be specified as part of the ``'pars'`` dictionary. In addition, the input files listed in ``'files'`` can also include an extension specification, such as ``[hdrtab]``, to limit the comparison to just that extension. This example from an actual test definition demonstrates how multiple input defintions can be used at the same time:: outputs = [ ('jw99999_nircam_f140m-maskbar_psfstack.fits', 'jw99999_nircam_f140m-maskbar_psfstack_ref.fits' ), ('jw9999947001_02102_00002_nrcb3_a3001_crfints.fits', 'jw9999947001_02102_00002_nrcb3_a3001_crfints_ref.fits' ), {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits'), 'pars': {'ignore_hdus': ['HDRTAB']}, {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits', ['primary','sci','dq']), 'pars': {'rtol': 0.000001} }, {'files': ('jw99999_nircam_f140m-maskbar_i2d.fits[hdrtab]', 'jw99999_nircam_f140m-maskbar_i2d_ref.fits[hdrtab]'), 'pars': {'ignore_keywords': ['NAXIS1', 'TFORM*'], 'ignore_fields': ['COL1', 'COL2']} }] .. note:: Each ``outputs`` entry in the list gets interpreted and processed separately. """ if ASTROPY_LT_3_1: if len(ignore_hdus) > 0: # pragma: no cover raise ValueError('ignore_hdus cannot be used for astropy<3.1') default_kwargs = { 'rtol': rtol, 'atol': atol, 'ignore_keywords': ignore_keywords, 'ignore_fields': ignore_fields } else: default_kwargs = { 'rtol': rtol, 'atol': atol, 'ignore_keywords': ignore_keywords, 'ignore_fields': ignore_fields, 'ignore_hdus': ignore_hdus } all_okay = True creature_report = '' updated_outputs = [] # To track outputs for Artifactory JSON schema for entry in outputs: diff_kwargs = copy.deepcopy(default_kwargs) extn_list = None num_entries = len(entry) if isinstance(entry, dict): entry_files = entry['files'] actual = entry_files[0] desired = entry_files[1] if len(entry_files) > 2: extn_list = entry_files[2] diff_kwargs.update(entry.get('pars', {})) elif num_entries == 2: actual, desired = entry elif num_entries == 3: actual, desired, extn_list = entry else: all_okay = False creature_report += '\nERROR: Cannot handle entry {}\n'.format( entry) continue # TODO: Use regex? if actual.endswith(']'): if extn_list is not None: all_okay = False creature_report += ( '\nERROR: Ambiguous extension requirements ' 'for {} ({})\n'.format(actual, extn_list)) continue actual_name, actual_extn = actual.split('[') actual_extn = actual_extn.replace(']', '') else: actual_name = actual actual_extn = None if desired.endswith(']'): if extn_list is not None: all_okay = False creature_report += ( '\nERROR: Ambiguous extension requirements ' 'for {} ({})\n'.format(desired, extn_list)) continue desired_name, desired_extn = desired.split('[') desired_extn = desired_extn.replace(']', '') else: desired_name = desired desired_extn = None # Get "truth" image try: desired = get_bigdata(*input_path, desired_name, docopy=docopy) except BigdataError: all_okay = False creature_report += '\nERROR: Cannot find {} in {}\n'.format( desired_name, input_path) continue if desired_extn is not None: desired_name = desired desired = "{}[{}]".format(desired, desired_extn) if verbose: print("\nComparing:\n {} \nto\n {}".format(actual, desired)) if actual.endswith('.fits') and desired.endswith('.fits'): # Build HDULists for comparison based on user-specified extensions if extn_list is not None: with fits.open(actual) as f_act: with fits.open(desired) as f_des: actual_hdu = fits.HDUList( [f_act[extn] for extn in extn_list]) desired_hdu = fits.HDUList( [f_des[extn] for extn in extn_list]) fdiff = FITSDiff(actual_hdu, desired_hdu, **diff_kwargs) creature_report += '\na: {}\nb: {}\n'.format( actual, desired) # diff report only gives hash # Working with FITS files... else: fdiff = FITSDiff(actual, desired, **diff_kwargs) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) elif actual_extn is not None or desired_extn is not None: if 'ignore_hdus' in diff_kwargs: # pragma: no cover diff_kwargs.pop('ignore_hdus') # Not applicable # Specific element of FITS file specified with fits.open(actual_name) as f_act: with fits.open(desired_name) as f_des: actual_hdu = f_act[actual_extn] desired_hdu = f_des[desired_extn] fdiff = HDUDiff(actual_hdu, desired_hdu, **diff_kwargs) creature_report += '\na: {}\nb: {}\n'.format(actual, desired) creature_report += fdiff.report() if not fdiff.identical: all_okay = False # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual_name, desired_name)) else: # ASCII-based diff with open(actual) as afile: actual_lines = afile.readlines() with open(desired) as dfile: desired_lines = dfile.readlines() udiff = unified_diff(actual_lines, desired_lines, fromfile=actual, tofile=desired) udiffIO = StringIO() udiffIO.writelines(udiff) udiff_report = udiffIO.getvalue() udiffIO.close() if len(udiff_report) == 0: creature_report += ('\na: {}\nb: {}\nNo differences ' 'found.\n'.format(actual, desired)) else: all_okay = False creature_report += udiff_report # Only keep track of failed results which need to # be used to replace the truth files (if OK). updated_outputs.append((actual, desired)) if not all_okay and results_root is not None: # pragma: no cover schema_pattern, tree, testname = generate_upload_params( results_root, updated_outputs, verbose=verbose) generate_upload_schema(schema_pattern, tree, testname) if not all_okay and raise_error: raise AssertionError(os.linesep + creature_report) return creature_report
def main(): tools_clang_directory = os.path.dirname( os.path.dirname(os.path.realpath(__file__))) tools_clang_scripts_directory = os.path.join(tools_clang_directory, 'scripts') test_directory_for_tool = os.path.join(tools_clang_directory, 'translation_unit', 'test_files') compile_database = os.path.join(test_directory_for_tool, 'compile_commands.json') compile_database_template = compile_database + '.template' source_files = glob.glob(os.path.join(test_directory_for_tool, '*.cc')) # Generate a temporary compilation database to run the tool over. with open(compile_database, 'w') as f: f.write( _GenerateCompileCommands(compile_database_template, test_directory_for_tool)) args = [ 'python', os.path.join(tools_clang_scripts_directory, 'run_tool.py'), 'translation_unit', test_directory_for_tool ] args.extend(source_files) run_tool = subprocess.Popen(args, stdout=subprocess.PIPE) stdout, _ = run_tool.communicate() if run_tool.returncode != 0: print 'run_tool failed:\n%s' % stdout sys.exit(1) passed = 0 failed = 0 for actual in source_files: actual += '.filepaths' expected = actual + '.expected' print '[ RUN ] %s' % os.path.relpath(actual) expected_output = actual_output = None with open(expected, 'r') as f: expected_output = f.readlines() with open(actual, 'r') as f: actual_output = f.readlines() has_same_filepaths = True for expected_line, actual_line in zip(expected_output, actual_output): if '//' in actual_line: actual_line = '//' + actual_line.split('//')[1] if ntpath.basename(expected_line) != ntpath.basename(actual_line): sys.stdout.write('expected: %s' % ntpath.basename(expected_line)) sys.stdout.write('actual: %s' % ntpath.basename(actual_line)) has_same_filepaths = False break if not has_same_filepaths: failed += 1 for line in difflib.unified_diff( expected_output, actual_output, fromfile=os.path.relpath(expected), tofile=os.path.relpath(actual)): sys.stdout.write(line) print '[ FAILED ] %s' % os.path.relpath(actual) # Don't clean up the file on failure, so the results can be referenced # more easily. continue print '[ OK ] %s' % os.path.relpath(actual) passed += 1 os.remove(actual) if failed == 0: os.remove(compile_database) print '[==========] %s ran.' % _NumberOfTestsToString(len(source_files)) if passed > 0: print '[ PASSED ] %s.' % _NumberOfTestsToString(passed) if failed > 0: print '[ FAILED ] %s.' % _NumberOfTestsToString(failed)
def assertXMLEqual(self, response, expected, msg=''): """Compare XML line by line and sorted attributes""" response_lines = response.splitlines() expected_lines = expected.splitlines() line_no = 1 diffs = [] for diff in difflib.unified_diff( [l.decode('utf8') for l in expected_lines], [l.decode('utf8') for l in response_lines]): diffs.append(diff) self.assertEqual( len(expected_lines), len(response_lines), "Expected and response have different number of lines!\n{}\n{}". format(msg, '\n'.join(diffs))) for expected_line in expected_lines: expected_line = expected_line.strip() response_line = response_lines[line_no - 1].strip() response_line = response_line.replace(b'e+6', b'e+06') # Compare tag if re.match(RE_ELEMENT, expected_line): expected_elements = re.findall(RE_ELEMENT, expected_line) response_elements = re.findall(RE_ELEMENT, response_line) self.assertEqual(expected_elements[0], response_elements[0], msg=msg + "\nTag mismatch on line %s: %s != %s" % (line_no, expected_line, response_line)) # Compare content if len(expected_elements ) == 2 and expected_elements[0] == expected_elements[1]: expected_element_content = re.findall( RE_ELEMENT_CONTENT, expected_line) response_element_content = re.findall( RE_ELEMENT_CONTENT, response_line) self.assertEqual( len(expected_element_content), len(response_element_content), msg=msg + "\nContent mismatch on line %s: %s != %s" % (line_no, expected_line, response_line)) if len(expected_element_content): self.assertEqual( expected_element_content[0], response_element_content[0], msg=msg + "\nContent mismatch on line %s: %s != %s" % (line_no, expected_line, response_line)) else: self.assertEqual(expected_line, response_line, msg=msg + "\nTag line mismatch %s: %s != %s\n%s" % (line_no, expected_line, response_line, msg)) # print("---->%s\t%s == %s" % (line_no, expected_line, response_line)) # Compare attributes if re.findall(RE_ATTRIBUTES, expected_line): # has attrs expected_attrs, expected_values = zip( *sorted(re.findall(RE_ATTRIBUTES, expected_line))) self.assertTrue( re.findall(RE_ATTRIBUTES, response_line), msg=msg + "\nXML attributes differ at line {0}: {1} != {2}".format( line_no, expected_line, response_line)) response_attrs, response_values = zip( *sorted(re.findall(RE_ATTRIBUTES, response_line))) self.assertEqual( expected_attrs, response_attrs, msg=msg + "\nXML attributes differ at line {0}: {1} != {2}".format( line_no, expected_attrs, response_attrs)) self.assertEqual( expected_values, response_values, msg=msg + "\nXML attribute values differ at line {0}: {1} != {2}". format(line_no, expected_values, response_values)) line_no += 1
if m: actual += m.expand(r"\1\2\n") # Gets the expected errors and their line number errors = [] for i, line in enumerate(src, 1): m = re.search(r"(?:^\s*!ERROR: )(.*)", line) if m: errors.append(m.group(1)) continue if errors: for x in errors: expect += f"{i}: {x}\n" errors = [] # Compares the expected errors with the compiler errors for line in unified_diff(actual.split("\n"), expect.split("\n"), n=0): line = re.sub(r"(^\-)(\d+:)", r"\nactual at \g<2>", line) line = re.sub(r"(^\+)(\d+:)", r"\nexpect at \g<2>", line) diffs += line if diffs != "": print(diffs) print() print("FAIL") sys.exit(1) else: print() print("PASS")
''' slow += '}' open('fuzz.slow.js', 'w').write(slow) open('fuzz.cpp', 'w').write(fast) print '_' slow_out = subprocess.Popen(['mozjs', '-m', '-n', 'fuzz.slow.js'], stdout=subprocess.PIPE).communicate()[0] print '.' subprocess.call(['g++', 'fuzz.cpp', 'Relooper.o', '-o', 'fuzz', '-g']) print '*' subprocess.call(['./fuzz'], stdout=open('fuzz.fast.js', 'w')) print '-' fast_out = subprocess.Popen(['mozjs', '-m', '-n', 'fuzz.fast.js'], stdout=subprocess.PIPE).communicate()[0] print if slow_out != fast_out: print ''.join([ a.rstrip() + '\n' for a in difflib.unified_diff(slow_out.split('\n'), fast_out.split('\n'), fromfile='slow', tofile='fast') ]) assert False #break
ap = ArchivesParserStorage() # Parse the old message, so we can ap.parse(StringIO(rawtxt)) ap.analyze() if ap.bodytxt != bodytxt.decode('utf8'): print "Failed to fix %s!" % msgid # Generate diff to show what we changed print "CHANGED:" print "\n".join( difflib.unified_diff(s.getvalue(), fixed, fromfile='old', tofile='new', n=2, lineterm='')) print "----" # Generate a diff to show what's left print "REMAINING:" print "\n".join( difflib.unified_diff(bodytxt.decode('utf8').splitlines(), ap.bodytxt.splitlines(), fromfile='old', tofile='new', n=2, lineterm='')) print "--------------" while True: a = raw_input('Save this change anyway?').lower()
def export_notebook(notebook, output_path=None, ext='.ty', identifier='_export_', diff=True, invert=False, stale_time=None): """ Given a v3-format .ipynb notebook file (from IPython 0.13 or later), allows the contents of labelled code cells to be exported to a plaintext file. The typical use case is for generating a runnable plain Python source file from selected cells of an IPython notebook. Code is selected for export by placing the given identifier on the first line of the chosen code cells. By default, only the labelled cells are exported. This behavior may be inverted to exclude labelled cells by setting the invert flag. notebook The filename of the notebook (.ipynb). output_path Optional output file path. Otherwise, uses the notebook basename. ext The file extension of the output. identifier The identifier used to label cells. diff Whether to print a diff when overwriting content. invert When set, only the non-labelled cells are exported. stale_time Number of seconds that may elapse since the last notebook save before a staleness warning is issued. Useful when exporting from an active IPython notebook. """ print("Deprecation Warning: Please use the %define_exporter magic instead") lines = [] if output_path is None: output_path = os.path.splitext(os.path.basename(notebook))[0] + ext # Assumes the v3 version of the ipynb format. import IPython.nbformat.current nbnode = IPython.nbformat.current.read(open(notebook, 'r'), 'ipynb') for cell in nbnode['worksheets'][0]['cells']: if cell['cell_type'] == 'code': celllines = cell['input'].rsplit("\n") labelled = (celllines[0].strip() == identifier) if labelled and not invert: lines.append('\n'.join(celllines[1:])) if invert and not labelled: lines.append('\n'.join(celllines[1:])) if stale_time: modified_time = time.time() - os.path.getmtime(notebook) if modified_time > stale_time: print "Notebook last saved %.1f seconds ago." % modified_time new_contents = "\n".join(lines) overwrite = os.path.isfile(output_path) if overwrite: old_contents = open(output_path, 'r').read() with open(output_path, 'w') as outfile: outfile.write(new_contents) if diff and overwrite: deltas = difflib.unified_diff(old_contents.splitlines(), new_contents.splitlines(), lineterm='') print '\n'.join(list(deltas))