def update_test(self, new_row, testfile): '''Create an update test which includes reversing the update''' ### TODO It would be nice to have a bunch of fields modified, but lets ### start with just one field. mod_field = sprandom.choice(tu.HI_ENTROPY_FIELDS) new_value = sprandom.choice(tu.HI_ENTROPY_FIELD_VALUES[mod_field]) orig_value = new_row[mod_field] mid = self.mid_id_generator.get_id() (qfiles, qids) = \ self.create_update_queries(mid, mod_field, new_value, orig_value, \ new_row, out_dir) self.create_update_test_script(mid, qfiles, new_row, mod_field, \ new_value, out_dir, testfile) self.fill_join_table(mid, qids) # # Put it back now # mid = self.mid_id_generator.get_id() (qfiles, qids) = \ self.create_update_queries(mid, mod_field, orig_value, new_value, \ new_row, out_dir) self.create_update_test_script(mid, qfiles, new_row, mod_field, \ orig_value, out_dir, testfile) self.fill_join_table(mid, qids)
def test_get_complete_constraint_list(self): db_num_records = 800 db_record_size = 10 cat = t1s.CATEGORIES.to_string(t1s.CATEGORIES.P1) subcat = t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].to_string( t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].eqcnf) subsubcat = sr.choice( t1s.SUBSUBCATEGORIES[(t1s.CATEGORIES.P1, t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].eqcnf)]) selection_cols = sr.choice(t1s.SELECTION_COLS) field = sr.choice(sv.VARS.numbers_list()) inp = t1ai.Input() inp[t1s.DBF_NUMRECORDS] = db_num_records inp[t1s.DBF_RECORDSIZE] = db_record_size inp[t1s.DBF_CAT] = cat inp[t1s.DBF_SUBCAT] = subcat inp[t1s.DBF_SUBSUBCAT] = subsubcat inp[t1s.DBP_SELECTIONCOLS] = selection_cols inp[t1s.DBA_FIELD] = field expected_constraint_list = [ (t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS, db_num_records), (t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE, db_record_size), (t1s.DBF_TABLENAME, t1s.DBF_CAT, cat), (t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat), (t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT, subsubcat), (t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols), (t1s.DBA_TABLENAME, t1s.DBA_FIELD, field) ] actual_constraint_list = inp.get_constraint_list() self.assertEquals(expected_constraint_list, actual_constraint_list)
def generate_circuit_by_depth(L, D, W, gate_maker): """ This function this generates an IBM circuit. It is called with the following inputs: L, the number of bits per batch, D, the depth of the circuit as defined by IBM, W, the number of input 'wires' (batch inputs taken) in the circuit, and gate_maker, the function used to produce each gate. """ # Create the circuit: circuit = ic.IBMCircuit(L) # Create W input wires: wires = [ iw.IBMInputWire("".join(("W", str(wire_ind))), circuit) for wire_ind in xrange(W) ] circuit.set_input_wires(wires) ultimate_level = wires # the last level created penultimate_level = [] # the second-to-last level created # Keep track of the smallest gate depth at the last level: min_depth = 0 # Maintain a list of gates at depth D: depth_D_gates = [] # Maintain a list of gates between depth D and D+1, not including D: depth_around_D_gates = [] # Initialize the global gate counter, which acts as the unique numerical id # of each gate: unique_gate_num = W while (min_depth <= D): new_level = [] for gate_index in range(W): # create W new gates new_gate = gate_maker(L, ultimate_level, penultimate_level, "".join(["G", str(unique_gate_num)]), circuit) # If the new gate has depth D, add it to depth_D_gates: if (new_gate.get_depth() == D): depth_D_gates.append(new_gate) # If the new gate has D < depth <= D+1, add it to # depth_around_D_gates: if ((new_gate.get_depth() > D) and (new_gate.get_depth() < D + 1)): depth_around_D_gates.append(new_gate) # Add the new gate to the new level: new_level.append(new_gate) # Increment the unique gate number: unique_gate_num += 1 # Increment the smallest gate depth at the last level as needed: min_depth = min(gate.get_depth() for gate in new_level) # Update the ultimate_level and penultimate_level pointers: penultimate_level = ultimate_level ultimate_level = new_level # If there is at least one gate of depth exactly D, select the output # gate from among such gates at random. Otherwise, select the output # gate from among gates between depth D and D+1. if (len(depth_D_gates) > 0): output_gate = sr.choice(depth_D_gates) else: output_gate = sr.choice(depth_around_D_gates) circuit.set_output_gate(output_gate) return circuit
def tests_by_type(self, test_type, num_rows_to_do=1): '''Generate tests of type insert_delete or update''' # get a unique number for this test test_id = test_type + "_" + self.test_id_generator.get_id() # create a directory for this test out_dir = os.path.join(self.base_output_dir, test_id) if not os.path.exists(out_dir): os.makedirs(out_dir) # open the testscript file testfilename = os.path.join(self.base_output_dir, test_script_name) testfile = open(testfilename, 'w') for row_count in range(num_rows_to_do): new_row = sprandom.choice(self.xtra_rows) self.insert_test(new_row, testfile) if (test_type == 'update'): self.update_test(new_row, testfile) self.delete_test(new_row, testfile) testfile.write('RootModeMasterScript SHUTDOWN\n') testfile.close()
def word_weight(): ''' Returns a weight for a word somewhere between 1 and 100 ''' weights = [1, 1, 1, 1, 1, 1, 1, 10, 10, 100] return spar_random.choice(weights)
def generate_conditional_pdf(self, min, max, ind_vars): """ Returns a random item according to pdf values of min and max conditioned on the values for the independent vars in ind_var """ try: underlying_dist = self._get_underlying_dist(ind_vars) except KeyError: # Oops. We have not seen these ind_vars before. # Get a random underlying_dict. ind_var_tuples = self._underlying_dists.keys() random_tuple = spar_random.choice(ind_var_tuples) underlying_dist = self._underlying_dists[random_tuple] return underlying_dist.generate_pdf(min, max)
def create_insert_query(self, mid, new_row, out_dir): '''Create an insert query and write to the query file''' # Choose field randomly from HI_ENTROPY_FIELDS rfield = sprandom.choice(tu.HI_ENTROPY_FIELDS) where_clause = self.select_str + 'FROM main WHERE ' + rfield + ' = ' +\ "'" + new_row[rfield] + "'" qid = self.qid_id_generator.get_id() qfilename = os.path.join(out_dir, "insert_query_" + str(qid) + ".q") qfile = open(qfilename, 'w') self.write_query(qid, where_clause, qfile) qfile.close() self.fill_mod_query_table(mid, qid, where_clause) return (qfilename, qid)
def generate(self, ind_vars): """ Returns a random item that relects the distribution of add calls, conditioned on the values for the independent vars in ind_vars. """ try: underlying_dist = self._get_underlying_dist(ind_vars) except KeyError: # Oops. We have not seen these ind_vars before. # Get a random underlying_dict. ind_var_tuples = self._underlying_dists.keys() random_tuple = spar_random.choice(ind_var_tuples) underlying_dist = self._underlying_dists[random_tuple] return underlying_dist.generate()
def generate_circuit_by_level(L, num_levels, W, gate_maker): """ This function this generates a random IBM circuit with num_levels instead of depth specified. It is called with the following inputs: L, the number of bits per batch, num_levels, the the number of levels in the circuit, W, the number of input 'wires' (batch inputs taken) in the circuit, and gate_maker, the function used to produce each gate. """ # Create the circuit: circuit = ic.IBMCircuit(L) # Create W input wires: wires = [ iw.IBMInputWire("".join(("W", str(wire_ind))), circuit) for wire_ind in xrange(W) ] circuit.set_input_wires(wires) ultimate_level = wires # the last level created penultimate_level = [] # the second-to-last level created # Initialize the global gate counter, which acts as the unique numerical id # of each gate: unique_gate_num = W for level in xrange(num_levels): new_level = [] for gate_index in xrange(W): # create W new gates new_gate = gate_maker(L, ultimate_level, penultimate_level, "".join(["G", str(unique_gate_num)]), circuit) # Add the new gate to the new level: new_level.append(new_gate) # Increment the unique gate number: unique_gate_num += 1 # Update the ultimate_level and penultimate_level pointers: penultimate_level = ultimate_level ultimate_level = new_level # Select the output gate from the last level: output_gate = sr.choice(ultimate_level) circuit.set_output_gate(output_gate) return circuit
def generate_two_sided(self, record_set_size, range_size, db_size): '''Returns a tuple containing a the lower and upper values of a randomly generated range''' #Get the desired density by dividing by the range of the bucket desired_density = record_set_size / range_size #Find all bit lengths that have that density density = self.__bucket_density(db_size) close_enoughs = [] min_bit_length = range_size.bit_length() + 1 for bit_len in xrange(min_bit_length, 65): if self.__close_enough(density[bit_len], desired_density): close_enoughs.append(bit_len) if len(close_enoughs) == 0: raise FooInputs elif len(close_enoughs) == 1: bit_length = close_enoughs[0] else: bit_length = spar_random.choice(close_enoughs) #Generate a random range of bit length range_size range = 1 for _ in xrange(range_size.bit_length() - 1): range *= 2 range += spar_random.randbit() assert range.bit_length() == range_size.bit_length() #generate random starting point within the bucket for the range, #the lower value is generated randomly and then multiplied by 2 #because randint cannot compute values larger than 2**63 lower_half = spar_random.randint(2**(bit_length - 1) / 2, (2**(bit_length) - range_size) / 2) lower = lower_half * 2 upper = lower + range return (lower, upper)
def generate(self): """Populates the circuit, input and output files with a circuit, an input, and the corresponding output with the appropriate parameters.""" # create the header and write it to the circuit file: header_string = self._create_circuit_header() # create the input wires and write the inputs to the input file: input_wires = self._create_input_wires() # set set of all circuit objects already created: levels = [input_wires] # initialize the global gate counter, which acts as the unique numerical # id of each gate: unique_gate_num_gen = itertools.count() # for each level: for level_index in xrange(self._D): # Create the list of gates at this level: this_level = [None] * self._W for gate_ind in xrange(self._W): displayname = "".join(["G",str(unique_gate_num_gen.next())]) # make the new gate: new_gate = self._gate_maker(levels, displayname) new_gate_output = sr.randbit() new_gate.balance(new_gate_output) # add this gate to our list of gates at this level: this_level[gate_ind] = new_gate # set things up for the next level: ultimate_level = this_level levels.append(this_level) output_gate = sr.choice(levels[-1]) output_gate.set_name("output_gate") # choose a random output, and write it to the output file: output = sr.randbit() self._output_file.write(str(output)) # balance the output gate with respect to the chosen output: output_gate.balance(output) circ = sc.StealthCircuit(input_wires, output_gate) # write the circuit to the circuit file: self._circuit_file.write(circ.display())
def make_random_gate(L, ultimate_level, penultimate_level, gate_name, circuit): """Creates a random gate with uniformly distributed type.""" gate_type = sr.choice( [g_type for g_type in GATE_TYPES.numbers_generator()]) generate = TEST_TYPE_TO_GATE_MAKER[gate_type] return generate(L, ultimate_level, penultimate_level, gate_name, circuit)
def create_update_queries(self, mid, mod_field, new_value, orig_value, new_row, \ out_dir): '''Create update queries. One that matches before the update, one after, and one both times''' # Put 3 queries in one query file # choose a HI_ENTROPY_FIELD other than mod_field unmatched_entropy_field = list(tu.HI_ENTROPY_FIELDS) try: unmatched_entropy_field.remove(mod_field) except ValueError: # Value wasn't in the list of hi_entropy_fields pass other_field = sprandom.choice(unmatched_entropy_field) other_field_value = new_row[other_field] qid_list = list() # (pre) matches the row before the change qid = self.qid_id_generator.get_id() qid_list.append(qid) # name the query file based on the first qid of the 3 - should be ok qfilename = os.path.join(out_dir, "update_query_" + str(qid) + ".q") qfile = open(qfilename, 'w') # put ZIP last since it will probably have more matches if mod_field == sv.enum_to_sql_name(sv.VARS.ZIP_CODE): first_pre_and_term = other_field + ' = ' + \ "'" + other_field_value + "'" first_post_and_term = first_pre_and_term second_pre_and_term = mod_field + ' = ' + "'" + orig_value + "'" second_post_and_term = mod_field + ' = ' + "'" + new_value + "'" else: first_pre_and_term = mod_field + ' = ' + "'" + orig_value + "'" first_post_and_term = mod_field + ' = ' + "'" + new_value + "'" second_pre_and_term = other_field + ' = ' + \ "'" + other_field_value + "'" second_post_and_term = second_pre_and_term where_clause = self.select_str + ' FROM main WHERE ' + \ first_pre_and_term + ' AND ' + second_pre_and_term self.write_query(qid, where_clause, qfile) self.fill_mod_query_table(mid, qid, where_clause) # (post) matches the row after the change qid = self.qid_id_generator.get_id() qid_list.append(qid) where_clause = self.select_str + ' FROM main WHERE ' + \ first_post_and_term + ' AND ' + second_post_and_term self.write_query(qid, where_clause, qfile) self.fill_mod_query_table(mid, qid, where_clause) # (both) matches the row before and after the change qid = self.qid_id_generator.get_id() qid_list.append(qid) where_clause = self.select_str + ' FROM main WHERE ' + other_field +\ ' = ' + "'" + other_field_value + "'" self.write_query(qid, where_clause, qfile) self.fill_mod_query_table(mid, qid, where_clause) qfile.close() return ([qfilename], qid_list)