def import_last_names(): last_names_array = [] file = util.find_all("last_names.txt") with open(file, "r") as last_names: for line in last_names: data = line.split("\t") last_names_array.append(data[0].capitalize()) return last_names_array
def pep8_all(): def pep8_file(file_): checker = pep8.Checker(filename=file_, max_line_length=99) incorrect = checker.check_all() if incorrect != 0: notifier.failure(str(file_)) notifier.success(str(file_)) map(pep8_file, util.find_all(os.environ["PORTER"], ".py"))
def coverage_test_package(package): def path_to_name(name): return os.path.split(name)[1].split('.')[0] for module in functional.removed( map(path_to_name, util.find_all( os.path.join('src', package), '.py')), '__init__'): command = coverage_module(package, module) notifier.success(command)
def load_location(): states = [] file = util.find_all("location.json") with open(file, 'r') as location_file: global JSON_DATA JSON_DATA = json.load(location_file) for state in JSON_DATA.keys(): states.append(capitalize_states(state)) return states
def locate_firmware_volumes(self, data, where=None): """Search through `data` looking for firmware volume headers `where` optionally specifies where it came from (e.g. compressed section) """ FVH_OFFSET_WITHIN_HEADER = 0x28 subtract_offset = lambda x: x - FVH_OFFSET_WITHIN_HEADER items = find_all(data, "_FVH", subtract_offset) #print "Found the following:", items return [FirmwareVolume(data, position, where) for position in items]
def import_prisons(): file = util.find_all("prison_list.txt") prisons = [] with open(file, "r") as prison_list: for row in prison_list: row = row.split("\t") name = row[0] state = row[1].strip("\n") temp = [name, state] prisons.append(temp) return prisons
def import_first_names(): male_first_names = [] female_first_names = [] file = util.find_all("people.txt") with open(file, "r") as person_file: for line in person_file: data = line.split("\t") for index, token in enumerate(data): if token.isalpha(): if index == 1: female_first_names.append(token) else: male_first_names.append(token) first_names = {"Male": male_first_names, "Female": female_first_names} return first_names
def generate_victim_offender(): victims = [] offenders = [] final = [] file_path = util.find_all("people.json") count = 0 with open(file_path, "r") as people_json: people = json.load(people_json) for person in people: if person['Victim']: victims.append(person['SSN']) else: offenders.append(person['SSN']) count += 1 util.process(count) # now lets make the connection between the two arrays. print(f"length of victims: {len(victims)}") print(f"length of offenders: {len(offenders)}") # length of victims is always larger than offenders (percentages) # therefore, we have a temp_offenders array that stores all deleted offenders # in case we run out of offenders to pair with victimes, hence the if stmt temp_offenders = [] for victim in victims: length = len(offenders) - 1 if length < 0: l = len(temp_offenders) - 1 num = random.randint(0, l) offender = temp_offenders[num] temp_offenders.remove(temp_offenders[num]) temp = [victim, offender] final.append(temp) else: num = random.randint(0, length) offender = offenders[num] temp_offenders.append(offenders[num]) offenders.remove(offenders[num]) temp = [victim, offender] final.append(temp) util.output_SQL("VICTIM_OFFENDER", final, "/mysql/insertVictimOffender.sql") util.output_json(final, "victim_offender.json")
def generate_weapons(): file_name = util.find_all("weapons.csv") with open(file_name) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') line_count = 0 weapons = [] for row in csv_reader: if line_count == 0: # some glitch (weird UTF-8 symbol) weapon = row[0][1:] else: weapon = row[0] fatalities = row[1] temp = {weapon: fatalities} weapons.append(temp) line_count += 1 util.process(line_count) return weapons
def get_outlier_points(stroke, estimated_position, limit): """ Finds the group of points, that is the closest to the stroke's estimated location, and returns the list of those points, which are not in this group. :param stroke: The inspected stroke. :param estimated_position: The stroke's estimated position. :param limit: The length limit of an edge between two vertices in the graph. The graph consists of the points of the stroke and it is represented as a graph for the algorithm that groups the points. :return: Ordered list of the outlier points' indexes. """ def index_to_point(indexes, point_objects): """ Gets the corresponding point objects in the stroke for the given set of indexes. :param indexes: Indexes to be interpreted as points. :param point_objects: A single stroke. :return: List of point objects. """ return [ point for point_index, point in enumerate(point_objects) if point_index in indexes ] # The connected vertices are stored as ones in the matrix. adjacency_matrix = np.ones((len(stroke), len(stroke))) for row in range(len(adjacency_matrix)): for col in range(len(adjacency_matrix[row])): if row == col: adjacency_matrix[row][col] = -1 elif util.point_2_point(stroke[row], stroke[col]) > limit: adjacency_matrix[row][col] = 0 # The matrix is converted into a dict, that stores the vertex sequence numbers as keys, and # the corresponding connected vertices as values. adjacency_list = OrderedDict() for index, row in enumerate(adjacency_matrix): adjacency_list[index] = util.find_all(row, 1) # The connected vertices are organised into groups. groups = [] while len(adjacency_list) > 0: group = util.dfs(adjacency_list) if len(group) != 0: groups.append(group) for index in group: if index in adjacency_list: del adjacency_list[index] # The groups are represented by their average position. average_positions = [] for group in groups: average_positions.append( util.get_average_point(index_to_point(group, stroke))) # The distances between the average position and the predicted location is calculated. distances = [] for position in average_positions: distances.append( util.point_2_point( position, util.Point(estimated_position[1], estimated_position[2]))) # The group that is closest to the predicted location is chosen. closest_group = distances.index(min(distances)) return [ index for index, point in enumerate(stroke) if index not in groups[closest_group] ]
def get_edge(s, local_types, method_name): local_types = list(local_types['types'][method_name].s.values())[0] r = RedBaron(s) defnodes = [ tnode for tnode in util.find_all(r, 'DefNode') if tnode.name == method_name ] defnode = defnodes[0] base_fornodes = util.find_fornodes(r) lo_Ls = [] hi_Ls = [] var_Ls = [] chosen_radius_Ls = [] conditionals = [] for base_fornode in base_fornodes: # Find one or more continuous for loops inside which all other code blocks are nested all_fornode_L = base_fornode.find_all('ForNode') fornode_L = [] for (i, fornode) in enumerate(all_fornode_L): fornode_L.append(fornode) subnodes = [ node for node in fornode.value if not isinstance(node, redbaron.CommentNode) ] if len(subnodes) == 1 and i + 1 < len( all_fornode_L) and subnodes[0] == all_fornode_L[i + 1]: pass else: break # For each such for loop, identify variable name, lower, upper bounds assert len(fornode_L) if not all( isinstance(x.iterator, redbaron.NameNode) for x in fornode_L): raise TransformError( 'each for loop must have a single var for LoopRemoveConditionals' ) var_L = [x.iterator.name.value for x in fornode_L] lo_L = [] hi_L = [] zero = RedBaron('0').find_all('IntNode')[0] is_parallel_L = [] for fornode in fornode_L: fornode_value_str = fornode.target.value.name.value if len(fornode.target.value) >= 3 and [ fornode.target.value[j].value for j in range(3) ] == ['cython', 'parallel', 'prange']: is_parallel = True elif fornode_value_str in ['range', 'xrange']: is_parallel = False else: raise TransformError( 'each for loop must have range or xrange for LoopRemoveConditionals ({})', fornode_value_str) is_parallel_L.append(is_parallel) # Get positional arguments call_L = [ c.value for c in fornode.target.value.call if c.target is None ] if len(call_L) == 1: lo_L.append(zero) hi_L.append(call_L[0]) elif len(call_L) == 2: lo_L.append(call_L[0]) hi_L.append(call_L[1]) elif len(call_L) == 3: success = False if util.is_int_constant( call_L[2].dumps() ): #isinstance(call_L[2], (redbaron.FloatNode, redbaron.IntNode)) or (isinstance(call_L[2], redbaron.UnitaryOperatorNode) and isinstance(call_L[2].target, (redbaron.FloatNode, redbaron.IntNode))): val = eval(call_L[2].dumps()) if val == 1: lo_L.append(call_L[0]) hi_L.append(call_L[1]) success = True if not success: raise TransformError( 'for loop range currently must have constant step of exactly 1 to apply LoopRemoveConditionals' ) else: raise TransformError( 'for loop range must have 1 to 3 arguments to apply LoopRemoveConditionals' ) # Get z3 expression strings lo_z3_L = [] hi_z3_L = [] def rewrite_expr_z3(r, is_redbaron=True): # Rewrites RedBaron expression to a str expression that could be used in z3 # Return (z3_expr_str, z3_varnames) z3_expr_str = (r.dumps() if is_redbaron else r).strip() z3_expr_str = z3_expr_str.replace('.', '_').replace('[', '_').replace( ']', '_') rp = RedBaron(z3_expr_str) for node in rp.find_all('UnitaryOperatorNode'): if node.value == 'not': node.replace('z3.Not(' + node.target.dumps() + ')') for node in rp.find_all('BooleanOperatorNode'): if node.value == 'and': node.replace('z3.And(' + node.first.dumps() + ',' + node.second.dumps() + ')') elif node.value == 'or': node.replace('z3.Or(' + node.first.dumps() + ',' + node.second.dumps() + ')') z3_expr_str = rp.dumps() z3_vars = set() for node in rp.find_all('NameNode'): z3_vars.add(node.value) return (z3_expr_str, z3_vars) all_z3_vars = set(var_L) for i in range(len(lo_L)): (lo_z3, z3_vars) = rewrite_expr_z3(lo_L[i]) all_z3_vars |= z3_vars (hi_z3, z3_vars) = rewrite_expr_z3(hi_L[i]) all_z3_vars |= z3_vars lo_z3_L.append(lo_z3) hi_z3_L.append(hi_z3) # Find all array getitem/setitem expressions (e.g. a[y,x]) getitem_L = [] for node in base_fornode.find_all('AtomtrailersNode'): if len(node) == 2 and isinstance( node.value[0], redbaron.NameNode) and isinstance( node.value[1], redbaron.GetitemNode): getitem_L.append(node) # For each dimension j of each expression of an array A, find radius r_j such that we can prove that the expression is always in bounds if we are more than r_j away from edge of array. Associate r_j with corresponding for loop dimension i (i.e. var_L[i]). # List of all candidate radii radius_L = [str(rval) for rval in range(min_radius, max_radius + 1)] radius_set = set(radius_L) getitem_arrayname_args_L = [] for getitem_node in getitem_L: arrayname = getitem_node.value[0].value args = getitem_node.getitem.value if not isinstance(args, redbaron.TupleNode): args = [args] args = list(args) getitem_arrayname_args_L.append((getitem_node, arrayname, args)) for arg in args: for current_node in util.all_nodes(arg): try: current_z3 = rewrite_expr_z3(current_node)[0].strip() except: continue if len(current_z3): if current_z3 not in radius_set: radius_set.add(current_z3) radius_L.append(current_z3) # Intersection of success radii from each dimension success_radius_intersect_L = [set(radius_L) for j in range(len(var_L))] def get_bound(i, is_upper, radius='0'): bound = lo_z3_L[i] if not is_upper else hi_z3_L[i] ineq = '>=' if not is_upper else '<' pad = ('+' if not is_upper else '-') + '(' + radius + ')' z3_expr = '(' + var_L[ i] + ')' + ineq + '((' + bound + ')' + pad + ')' return z3_expr for (getitem_node, arrayname, args) in getitem_arrayname_args_L: try: # Add assumptions that current argument to the array 'arrayname' is out of bounds along dimension j, # on either the lower bound side (if is_lo_bound) or else the upper bound side. for (j, arg) in enumerate(args): # Skip indices that are greater than the number of loops if j >= len(var_L): continue # Skip getitem index expressions that involve only a single variable, e.g. 'r' and 'c' in A[r,c] if isinstance(arg, redbaron.NameNode): continue success_radius_L = [] # Find radii that work out of the list of candidate radii, where we can prove in-bounds property on # both sides. for radius in radius_L: radius_success = True radius_is_constant = util.is_int_constant(radius) arg_s = arg.dumps() for is_lo_bound in [False, True]: if ':' in arg_s: continue solver = z3.Solver() current_z3_vars = {} for z3_var in all_z3_vars: current_z3_vars[z3_var] = z3.Int(z3_var) # Add assumptions that each for loop variable is in bounds, with a distance 'radius' to the edge. i = j for k in range(2): z3_expr = get_bound(i, k, radius) try: z3_expr_eval = eval( z3_expr, globals(), current_z3_vars) except: radius_success = False break #raise TransformError('LoopRemoveConditionals: could not eval expression {}'.format(z3_expr)) solver.add(z3_expr_eval) if not radius_success: break # Add dubious assumption that all arrays with the same shapes during unit testing are always equal in shape. # Really we should probably prove this is true by inspecting the numpy code used and its dependencies # instead of just assuming it. arraytype = local_types[arrayname] for (arrayname_p, arraytype_p) in local_types.items(): if (arraytype_p.cython_type == arraytype.cython_type and arraytype_p.shape_list == arraytype.shape_list): # It is an array of the same size, so assume equal shape for k in range(len(arraytype_p.shape)): z3_var_p = rewrite_expr_z3( arrayname_p + '.shape[{}]'.format(k), False)[0] z3_var_current = rewrite_expr_z3( arrayname + '.shape[{}]'.format(k), False)[0] for z3_v in [z3_var_p, z3_var_current]: if z3_v not in current_z3_vars: current_z3_vars[z3_v] = z3.Int( z3_v) solver.add( eval(z3_var_p, globals(), current_z3_vars) == eval( z3_var_current, globals(), current_z3_vars)) # Assume that current variable used in getitem is out of bounds (either too low or too high), # and attempt to derive a contradiction. if is_lo_bound: out_of_bounds = arg_s + ' < 0' else: getitem_hi = rewrite_expr_z3( arrayname + '.shape[{}]'.format(j), False)[0] out_of_bounds = arg_s + ' >= (' + getitem_hi + ')' try: out_of_bounds_eval = eval( out_of_bounds, globals(), current_z3_vars) except: raise LoopRemoveConditionalsProofFailed( 'LoopRemoveConditionals: could not eval out of bounds expression {}' .format(out_of_bounds)) solver.add(out_of_bounds_eval) check = solver.check() if check != z3.unsat: # We did not derive a contraction. This radius is invalid. radius_success = False break if radius_success: success_radius_L.append(radius) success_radius_intersect_L[j] &= set(success_radius_L) except LoopRemoveConditionalsProofFailed: pass # Place all constraints in a list constraints = [] for j in range(len(var_L)): for k in range(2): constraints.append(get_bound(j, k)) # Reduce success_radius_intersect_L to a single radius along each dimension (chosen_radius_L) chosen_radius_L = [] for j in range(len(var_L)): success_L = list(success_radius_intersect_L[j]) is_constant_L = [util.is_int_constant(x) for x in success_L] constant_val_L = [(int(success_L[i].strip('()')) if is_constant_L[i] else numpy.nan) for i in range(len(success_L))] try: min_constant_index = numpy.nanargmin(constant_val_L) except ValueError: raise TransformError( 'LoopRemoveConditionals not valid because no radius could be proved' ) success_L = [success_L[min_constant_index]] + [ success_L[i] for i in range(len(success_L)) if not is_constant_L[i] ] try: chosen_radius_L.append( util.prove_smallest(success_L, all_z3_vars, constraints)) except util.ProveSmallestError: raise TransformError( 'LoopRemoveConditionals could not prove smallest element') """loop_body = fornode.value.dumps() conditional_status = {} for nodetype in ['IfNode', 'ElifNode', 'TernaryOperatorNode']: for node in fornode.find_all(nodetype): conditional = node.test if nodetype != 'TernaryOperatorNode' else node.value #build a grid for j in range(3): for k in range(3): for prove_true in [False, True]: radius_r = chosen_radius_L[0] radius_c = chosen_radius_L[1] var_r = var_L[0] var_c = var_L[1] if j == 0: lo_r = lo_L[0].dumps() hi_r = '(' + '(' + lo_r + ') + ' + radius_r + ')' lo_r_extra = lo_r hi_r_extra = '(' + '(' + hi_L[0].dumps() + ') - ' + radius_r + ')' elif j == 1: lo_r = '(' + '(' + lo_L[0].dumps() + ') + ' + radius_r + ')' hi_r = '(' + '(' + hi_L[0].dumps() + ') - ' + radius_r + ')' lo_r_extra = lo_r hi_r_extra = hi_r elif j == 2: lo_r = '(' + '(' + hi_L[0].dumps() + ') - ' + radius_r + ')' hi_r = hi_L[0].dumps() lo_r_extra = '(' + '(' + lo_L[0].dumps() + ') + ' + radius_r + ')' hi_r_extra = hi_r if k == 0: lo_c = lo_L[1].dumps() hi_c = '(' + '(' + lo_c + ') + ' + radius_c + ')' lo_c_extra = lo_c hi_c_extra = '(' + '(' + hi_L[1].dumps() + ') - ' + radius_c + ')' elif k == 1: lo_c = '(' + '(' + lo_L[1].dumps() + ') + ' + radius_c + ')' hi_c = '(' + '(' + hi_L[1].dumps() + ') - ' + radius_c + ')' lo_c_extra = lo_c hi_c_extra = hi_c elif k == 2: lo_c = '(' + '(' + hi_L[1].dumps() + ') - ' + radius_c + ')' hi_c = hi_L[1].dumps() lo_c_extra = '(' + '(' + lo_L[1].dumps() + ') + ' + radius_c + ')' hi_c_extra = hi_c current_z3_vars = {} for z3_var in all_z3_vars: current_z3_vars[z3_var] = z3.Int(z3_var) lo_constraint_r = rewrite_expr_z3('{var_r} >= {lo_r}'.format(**locals()), False)[0] hi_constraint_r = rewrite_expr_z3('{var_r} < {hi_r}'.format(**locals()), False)[0] lo_constraint_r_extra = rewrite_expr_z3('{var_r} >= {lo_r_extra}'.format(**locals()), False)[0] hi_constraint_r_extra = rewrite_expr_z3('{var_r} < {hi_r_extra}'.format(**locals()), False)[0] lo_constraint_c = rewrite_expr_z3('{var_c} >= {lo_c}'.format(**locals()), False)[0] hi_constraint_c = rewrite_expr_z3('{var_c} < {hi_c}'.format(**locals()), False)[0] lo_constraint_c_extra = rewrite_expr_z3('{var_c} >= {lo_c_extra}'.format(**locals()), False)[0] hi_constraint_c_extra = rewrite_expr_z3('{var_c} < {hi_c_extra}'.format(**locals()), False)[0] solver = z3.Solver() solver.add(eval(lo_constraint_r, globals(), current_z3_vars)) solver.add(eval(hi_constraint_r, globals(), current_z3_vars)) solver.add(eval(lo_constraint_c, globals(), current_z3_vars)) solver.add(eval(hi_constraint_c, globals(), current_z3_vars)) solver.add(eval(lo_constraint_r_extra, globals(), current_z3_vars)) solver.add(eval(hi_constraint_r_extra, globals(), current_z3_vars)) solver.add(eval(lo_constraint_c_extra, globals(), current_z3_vars)) solver.add(eval(hi_constraint_c_extra, globals(), current_z3_vars)) conditional_s = conditional.dumps() if prove_true: conditional_s = 'not (' + conditional_s + ')' conditional_s = rewrite_expr_z3(conditional_s, False)[0] conditional_eval = eval(conditional_s, globals(), current_z3_vars) solver.add(conditional_eval) if solver.check() == z3.unsat: conditional_status.setdefault((j, k), []).append((node, prove_true)) break""" lo_Ls.append(lo_L) hi_Ls.append(hi_L) var_Ls.append(var_L) chosen_radius_Ls.append(chosen_radius_L) #conditionals.append(conditional_status) return (lo_Ls, hi_Ls, var_Ls, chosen_radius_Ls, all_z3_vars)
def locate_packs(self, setuputility_binary): "Searches for Forms and the English StringTable using a set of heuristics" # 1st byte: upper bits from length: almost certainly zero # 2-3: Short typecode, 3 == form # 4-5: 0x0e is the formset opcode, and 0x24 is is length # This magic string appears three bytes into the header form_magic = "\x00\x03\x00\x0e\x24" form_magic_offset = -3 # HiipackHeaderSize HHS = 6 english_attribute_magic = "\x00" * 4 def create_stringtable(magic_location): def test_stringtable(poss_header_location): # We started at attributes, start at lnoff poss_header_location -= 12 dat = setuputility_binary[poss_header_location:] lnoff, plnoff, count, attributes = unpack_from("<IIII", dat) # This check is extraordinarily unlikely to succeed in the case # we haven't actually found the string table header. if (magic_location - poss_header_location + HHS == lnoff and magic_location - poss_header_location + HHS + 8 == plnoff): string_table_loc = poss_header_location - HHS stable = StringTable(setuputility_binary, string_table_loc) raise FindStopSearching(stable) raise FindBadPosition result = find_all_backwards(setuputility_binary, english_attribute_magic, test_stringtable, magic_location) if result: raise FindStopSearching(result) raise FindBadPosition string_magic = u"eng\x00English".encode("utf-16le") english_stringtable = find_all(setuputility_binary, string_magic, create_stringtable) if not english_stringtable: raise RuntimeError #~ english_stringtable.showinfo() def create_form(location): location += form_magic_offset try: return Form(setuputility_binary, location, english_stringtable) except AttributeError: raise FindBadPosition forms = find_all(setuputility_binary, form_magic, create_form) found = False # We've finally reached the bottom layer! # Now we just search for the location of the VT switch.. for form in forms: for opcode in form.fetch_opcodes(FormOp.EFI_IFR_ONE_OF_OP): qid, width, pid, hid = unpack_from("<HBHH", opcode.payload) prnt_string = english_stringtable[pid] help_string = english_stringtable[hid] args = (qid, width, prnt_string, help_string) if "Vanderpool" in help_string: found = True print "Location = 0x%03x<%d>, name='%s' help='%s'" % args if not found: print "Sorry, I couldn't locate the VT flag? :("
def load_incar(): global INCAR_JSON file = util.find_all("incarceration.json") with open(file, 'r') as incar_file: INCAR_JSON = json.load(incar_file)
def load_weapons(): weapon = [] global WEAPON_JSON file = util.find_all("weapons.json") with open(file, 'r') as weapon_json: WEAPON_JSON = json.load(weapon_json)
def load_victim_offender(): global V_O_JSON file = util.find_all("victim_offender.json") with open(file, 'r') as vo_file: V_O_JSON = json.load(vo_file)