def run(fnin, fnout=None, strict=False, verbose=False): lines = open(fnin, 'r').read().split('\n') tags = dict() bitss = dict() for line in lines: line = line.strip() if line == '': continue # TODO: figure out what to do with masks if line.startswith("bit "): continue tag, bits, mode, _ = util.parse_db_line(line) if strict: if mode != "always": assert not mode, "strict: got ill defined line: %s" % (line, ) if tag in tags: print("Original line: %s" % tags[tag], file=sys.stderr) print("New line: %s" % line, file=sys.stderr) assert 0, "strict: got duplicate tag %s" % (tag, ) assert bits not in bitss, "strict: got duplicate bits %s: %s %s" % ( bits, tag, bitss[bits]) tags[tag] = line if bits != None: bitss[bits] = tag if fnout: with open(fnout, "w") as fout: for line in sorted(lines): line = line.strip() if line == '': continue fout.write(line + '\n')
def group_tags(lines, tag_groups, bit_groups): """ Implements tag grouping. If a tag belongs to a group then the common bits of that group are added to is as zeros. >>> tg = [{"A", "B"}] >>> bg = [{(1, 2), (3, 4)}] >>> res = group_tags({"A 1_2", "B 3_4"}, tg, bg) >>> (res[0], sorted(list(res[1]))) (2, ['A 01_02 !03_04', 'B !01_02 03_04']) >>> tg = [{"A", "B"}] >>> bg = [{(1, 2), (3, 4)}] >>> res = group_tags({"A 1_2", "B 3_4", "C 1_2"}, tg, bg) >>> (res[0], sorted(list(res[1]))) (2, ['A 01_02 !03_04', 'B !01_02 03_04', 'C 01_02']) """ changes = 0 new_lines = set() # Process lines for line in lines: line = line.strip() if not len(line): continue # Parse the line tag, bits, mode, _ = util.parse_db_line(line) if not bits: bits = set() else: bits = set([util.parse_tagbit(b) for b in bits]) # Check if the tag belongs to a group for tag_group, bit_group in zip(tag_groups, bit_groups): if tag in tag_group: # Add zero bits to the tag if not already there bit_coords = set([b[1] for b in bits]) for zero_bit in bit_group: if zero_bit not in bit_coords: bits.add((False, zero_bit)) # Format the line new_line = format_bits(tag, bits) # Add the line new_lines.add(new_line) changes += 1 break # It does not, pass it through unchanged else: new_lines.add(format_bits(tag, bits)) return changes, new_lines
def maketodo(pipfile, dbfile, intre, not_endswith=None, verbose=False): ''' db files start with INT., but pipfile lines start with INT_L Normalize by removing before the first dot 050-intpips doesn't care about contents, but most fuzzers use the tile type prefix ''' todos, tile_type = load_pipfile(pipfile, verbose=verbose) verbose and print('%s: %u entries' % (pipfile, len(todos))) verbose and print("pipfile todo sample: %s" % list(todos)[0]) if 0 and verbose: print("TODOs") for todo in sorted(list(todos)): print(' %s' % todo) verbose and print('Pre db %s: %u entries' % (dbfile, len(todos))) # Allow against empty db if os.path.exists(dbfile): verbose and print("Loading %s" % dbfile) with open(dbfile, "r") as f: # INT.BYP_ALT0.BYP_BOUNCE_N3_3 !22_07 !23_07 !25_07 21_07 24_07 for line in f: tag, _bits, mode = util.parse_db_line(line.strip()) # Only count resolved entries if mode: continue # INT.BLAH => INT_L.BLAH tag = tile_type + '.' + noprefix(tag) # bipips works on a subset if tag in todos: todos.remove(tag) else: verbose and print("WARNING: couldnt remove %s (line %s)" % (tag, line)) else: verbose and print("WARNING: dbfile doesnt exist: %s" % dbfile) verbose and print('Post db %s: %u entries' % (dbfile, len(todos))) drops = 0 lines = 0 for line in todos: if re.match(intre, line) and (not_endswith is None or not line.endswith(not_endswith)): print(line) else: drops += 1 lines += 1 verbose and print('Print %u entries w/ %u drops' % (lines, drops))
def find_common_bits_for_tag_groups(lines, tag_groups): """ For each tag group finds a common set of bits that have value of one. """ bit_groups = [] for tag_group in tag_groups: bit_group = set() for line in lines: tag, bits, mode, _ = util.parse_db_line(line) if not bits: continue bits = set([util.parse_tagbit(b) for b in bits]) if tag in tag_group and len(bits): ones = set([b[1] for b in bits if b[0]]) bit_group |= ones bit_groups.append(bit_group) return bit_groups
def add_zero_bits(fn_in, zero_db, clb_int=False, strict=True, verbose=False): ''' Add multibit entries This requires adding some zero bits (ex: !31_09) If an entry has any of the ''' zero_groups = ZeroGroups(zero_db) lines = [] new_lines = set() changes = 0 llast = None drops = 0 with open(fn_in, "r") as f: for line in f: # Hack: skip duplicate lines # This happens while merging a new multibit entry line = line.strip() if line == llast: continue lines.append(line) tag, bits, mode, _ = util.parse_db_line(line) if bits is not None and mode is None: zero_groups.add_tag_bits(tag, bits) if verbose: zero_groups.print_groups() for line in lines: tag, bits, mode, _ = util.parse_db_line(line) # an enum that needs masking # check below asserts that a mask was actually applied if mode and mode != "<0 candidates>" and not strict: verbose and print("WARNING: dropping unresolved line: %s" % line) drops += 1 continue assert mode not in ( "<const0>", "<const1>"), "Entries must be resolved. line: %s" % (line, ) if mode == "always": new_line = line else: if mode: assert mode == "<0 candidates>", line bits = set() else: bits = set(bits) """ This appears to be a large range of one hot interconnect bits They are immediately before the first CLB real bits """ if clb_int: zero_range(tag, bits, 22, 25) zero_groups.add_bits_from_zero_groups(tag, bits, strict=strict, verbose=verbose) if strict: assert len(bits) > 0, 'Line {} found no bits.'.format(line) elif len(bits) == 0: verbose and print( "WARNING: dropping unresolved line: %s" % line) drops += 1 continue new_line = " ".join([tag] + sorted(bits)) if re.match(r'.*<.*>.*', new_line): print("Original line: %s" % line) assert 0, "Failed to remove line mode: %s" % (new_line) if new_line != line: changes += 1 new_lines.add(new_line) llast = line if drops: print("WARNING: %s dropped %s unresolved lines" % (fn_in, drops)) return changes, new_lines
def add_zero_bits(fn_in, lines, zero_db, clb_int=False, strict=True, verbose=False): ''' Add multibit entries This requires adding some zero bits (ex: !31_09) If an entry has any of the ''' zero_groups = ZeroGroups(zero_db) new_lines = set() changes = 0 drops = 0 for line in lines: tag, bits, mode, _ = util.parse_db_line(line) if bits is not None and mode is None: zero_groups.add_tag_bits(tag, bits) if verbose: zero_groups.print_groups() for line in lines: tag, bits, mode, _ = util.parse_db_line(line) # an enum that needs masking # check below asserts that a mask was actually applied if mode and mode != "<0 candidates>" and not strict: verbose and print("WARNING: dropping unresolved line: %s" % line) drops += 1 continue assert mode not in ( "<const0>", "<const1>"), "Entries must be resolved. line: %s" % (line, ) if mode == "always": new_line = line else: if mode: assert mode == "<0 candidates>", line bits = set() else: bits = set(bits) """ This appears to be a large range of one hot interconnect bits They are immediately before the first CLB real bits """ if clb_int: zero_range(tag, bits, 22, 25) set_bits = [bit for bit in bits if bit[0] != '!'] if len(set_bits) not in [2, 4]: # All INT bits appear to be only have 2 or 4 bits. verbose and print( "WARNING: dropping line with %d bits, not [2, 4]: %s, %s" % (len(set_bits), bits, line)) drops += 1 continue zero_groups.add_bits_from_zero_groups(tag, bits, strict=strict, verbose=verbose) if strict: assert len(bits) > 0, 'Line {} found no bits.'.format(line) elif len(bits) == 0: verbose and print( "WARNING: dropping unresolved line: %s" % line) drops += 1 continue new_line = " ".join([tag] + sorted(bits)) if re.match(r'.*<.*>.*', new_line): print("Original line: %s" % line) assert 0, "Failed to remove line mode: %s" % (new_line) if new_line != line: changes += 1 new_lines.add(new_line) if drops: print("WARNING: %s dropped %s unresolved lines" % (fn_in, drops)) return changes, new_lines
def group_tags(lines, tag_groups, bit_groups): """ Implements tag grouping. If a tag belongs to a group then the common bits of that group are added to is as zeros. >>> tg = [{"A", "B"}] >>> bg = [{(1, 2), (3, 4)}] >>> res = group_tags({"A 1_2", "B 3_4"}, tg, bg) >>> (res[0], sorted(list(res[1]))) (2, ['A 1_2 !3_4', 'B !1_2 3_4']) >>> tg = [{"A", "B"}] >>> bg = [{(1, 2), (3, 4)}] >>> res = group_tags({"A 1_2", "B 3_4", "C 1_2"}, tg, bg) >>> (res[0], sorted(list(res[1]))) (2, ['A 1_2 !3_4', 'B !1_2 3_4', 'C 1_2']) """ changes = 0 new_lines = set() # Process lines for line in lines: line = line.strip() if not len(line): continue # Parse the line tag, bits, mode, _ = util.parse_db_line(line) if not bits: bits = set() else: bits = set([util.parse_tagbit(b) for b in bits]) # Check if the tag belongs to a group for tag_group, bit_group in zip(tag_groups, bit_groups): if tag in tag_group: # Add zero bits to the tag if not already there bit_coords = set([b[1] for b in bits]) for zero_bit in bit_group: if zero_bit not in bit_coords: bits.add((False, zero_bit)) # Format the line bit_strs = [] for bit in sorted(list(bits), key=lambda b: b[1]): s = "!" if not bit[0] else "" s += "{}_{}".format(bit[1][0], bit[1][1]) bit_strs.append(s) new_line = " ".join([tag] + bit_strs) # Add the line new_lines.add(new_line) changes += 1 break # It does not, pass it through unchanged else: new_lines.add(line) return changes, new_lines