def update_test(self, new_row, testfile):
'''Create an update test which includes reversing the update'''
### TODO It would be nice to have a bunch of fields modified, but lets
### start with just one field.
mod_field = sprandom.choice(tu.HI_ENTROPY_FIELDS)
new_value = sprandom.choice(tu.HI_ENTROPY_FIELD_VALUES[mod_field])
orig_value = new_row[mod_field]
mid = self.mid_id_generator.get_id()
(qfiles, qids) = \
self.create_update_queries(mid, mod_field, new_value, orig_value, \
new_row, out_dir)
self.create_update_test_script(mid, qfiles, new_row, mod_field, \
new_value, out_dir, testfile)
self.fill_join_table(mid, qids)
#
# Put it back now
#
mid = self.mid_id_generator.get_id()
(qfiles, qids) = \
self.create_update_queries(mid, mod_field, orig_value, new_value, \
new_row, out_dir)
self.create_update_test_script(mid, qfiles, new_row, mod_field, \
orig_value, out_dir, testfile)
self.fill_join_table(mid, qids)
def test_get_complete_constraint_list(self):
db_num_records = 800
db_record_size = 10
cat = t1s.CATEGORIES.to_string(t1s.CATEGORIES.P1)
subcat = t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].to_string(
t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].eqcnf)
subsubcat = sr.choice(
t1s.SUBSUBCATEGORIES[(t1s.CATEGORIES.P1,
t1s.SUBCATEGORIES[t1s.CATEGORIES.P1].eqcnf)])
selection_cols = sr.choice(t1s.SELECTION_COLS)
field = sr.choice(sv.VARS.numbers_list())
inp = t1ai.Input()
inp[t1s.DBF_NUMRECORDS] = db_num_records
inp[t1s.DBF_RECORDSIZE] = db_record_size
inp[t1s.DBF_CAT] = cat
inp[t1s.DBF_SUBCAT] = subcat
inp[t1s.DBF_SUBSUBCAT] = subsubcat
inp[t1s.DBP_SELECTIONCOLS] = selection_cols
inp[t1s.DBA_FIELD] = field
expected_constraint_list = [
(t1s.DBF_TABLENAME, t1s.DBF_NUMRECORDS, db_num_records),
(t1s.DBF_TABLENAME, t1s.DBF_RECORDSIZE, db_record_size),
(t1s.DBF_TABLENAME, t1s.DBF_CAT, cat),
(t1s.DBF_TABLENAME, t1s.DBF_SUBCAT, subcat),
(t1s.DBF_TABLENAME, t1s.DBF_SUBSUBCAT, subsubcat),
(t1s.DBP_TABLENAME, t1s.DBP_SELECTIONCOLS, selection_cols),
(t1s.DBA_TABLENAME, t1s.DBA_FIELD, field)
]
actual_constraint_list = inp.get_constraint_list()
self.assertEquals(expected_constraint_list, actual_constraint_list)
def generate_circuit_by_depth(L, D, W, gate_maker):
"""
This function this generates an IBM circuit.
It is called with the following inputs:
L, the number of bits per batch,
D, the depth of the circuit as defined by IBM,
W, the number of input 'wires' (batch inputs taken) in the circuit, and
gate_maker, the function used to produce each gate.
"""
# Create the circuit:
circuit = ic.IBMCircuit(L)
# Create W input wires:
wires = [
iw.IBMInputWire("".join(("W", str(wire_ind))), circuit)
for wire_ind in xrange(W)
]
circuit.set_input_wires(wires)
ultimate_level = wires # the last level created
penultimate_level = [] # the second-to-last level created
# Keep track of the smallest gate depth at the last level:
min_depth = 0
# Maintain a list of gates at depth D:
depth_D_gates = []
# Maintain a list of gates between depth D and D+1, not including D:
depth_around_D_gates = []
# Initialize the global gate counter, which acts as the unique numerical id
# of each gate:
unique_gate_num = W
while (min_depth <= D):
new_level = []
for gate_index in range(W): # create W new gates
new_gate = gate_maker(L, ultimate_level, penultimate_level,
"".join(["G",
str(unique_gate_num)]), circuit)
# If the new gate has depth D, add it to depth_D_gates:
if (new_gate.get_depth() == D):
depth_D_gates.append(new_gate)
# If the new gate has D < depth <= D+1, add it to
# depth_around_D_gates:
if ((new_gate.get_depth() > D) and (new_gate.get_depth() < D + 1)):
depth_around_D_gates.append(new_gate)
# Add the new gate to the new level:
new_level.append(new_gate)
# Increment the unique gate number:
unique_gate_num += 1
# Increment the smallest gate depth at the last level as needed:
min_depth = min(gate.get_depth() for gate in new_level)
# Update the ultimate_level and penultimate_level pointers:
penultimate_level = ultimate_level
ultimate_level = new_level
# If there is at least one gate of depth exactly D, select the output
# gate from among such gates at random. Otherwise, select the output
# gate from among gates between depth D and D+1.
if (len(depth_D_gates) > 0):
output_gate = sr.choice(depth_D_gates)
else:
output_gate = sr.choice(depth_around_D_gates)
circuit.set_output_gate(output_gate)
return circuit
def tests_by_type(self, test_type, num_rows_to_do=1):
'''Generate tests of type insert_delete or update'''
# get a unique number for this test
test_id = test_type + "_" + self.test_id_generator.get_id()
# create a directory for this test
out_dir = os.path.join(self.base_output_dir, test_id)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
# open the testscript file
testfilename = os.path.join(self.base_output_dir, test_script_name)
testfile = open(testfilename, 'w')
for row_count in range(num_rows_to_do):
new_row = sprandom.choice(self.xtra_rows)
self.insert_test(new_row, testfile)
if (test_type == 'update'):
self.update_test(new_row, testfile)
self.delete_test(new_row, testfile)
testfile.write('RootModeMasterScript SHUTDOWN\n')
testfile.close()
def word_weight():
'''
Returns a weight for a word somewhere between 1 and
100
'''
weights = [1, 1, 1, 1, 1, 1, 1, 10, 10, 100]
return spar_random.choice(weights)
def generate_conditional_pdf(self, min, max, ind_vars):
"""
Returns a random item according to pdf values of min and max
conditioned on the values for the independent vars in ind_var
"""
try:
underlying_dist = self._get_underlying_dist(ind_vars)
except KeyError:
# Oops. We have not seen these ind_vars before.
# Get a random underlying_dict.
ind_var_tuples = self._underlying_dists.keys()
random_tuple = spar_random.choice(ind_var_tuples)
underlying_dist = self._underlying_dists[random_tuple]
return underlying_dist.generate_pdf(min, max)
def create_insert_query(self, mid, new_row, out_dir):
'''Create an insert query and write to the query file'''
# Choose field randomly from HI_ENTROPY_FIELDS
rfield = sprandom.choice(tu.HI_ENTROPY_FIELDS)
where_clause = self.select_str + 'FROM main WHERE ' + rfield + ' = ' +\
"'" + new_row[rfield] + "'"
qid = self.qid_id_generator.get_id()
qfilename = os.path.join(out_dir, "insert_query_" + str(qid) + ".q")
qfile = open(qfilename, 'w')
self.write_query(qid, where_clause, qfile)
qfile.close()
self.fill_mod_query_table(mid, qid, where_clause)
return (qfilename, qid)
def generate(self, ind_vars):
"""
Returns a random item that relects the distribution of add
calls, conditioned on the values for the independent vars in
ind_vars.
"""
try:
underlying_dist = self._get_underlying_dist(ind_vars)
except KeyError:
# Oops. We have not seen these ind_vars before.
# Get a random underlying_dict.
ind_var_tuples = self._underlying_dists.keys()
random_tuple = spar_random.choice(ind_var_tuples)
underlying_dist = self._underlying_dists[random_tuple]
return underlying_dist.generate()
def generate_circuit_by_level(L, num_levels, W, gate_maker):
"""
This function this generates a random IBM circuit with num_levels instead
of depth specified.
It is called with the following inputs:
L, the number of bits per batch,
num_levels, the the number of levels in the circuit,
W, the number of input 'wires' (batch inputs taken) in the circuit, and
gate_maker, the function used to produce each gate.
"""
# Create the circuit:
circuit = ic.IBMCircuit(L)
# Create W input wires:
wires = [
iw.IBMInputWire("".join(("W", str(wire_ind))), circuit)
for wire_ind in xrange(W)
]
circuit.set_input_wires(wires)
ultimate_level = wires # the last level created
penultimate_level = [] # the second-to-last level created
# Initialize the global gate counter, which acts as the unique numerical id
# of each gate:
unique_gate_num = W
for level in xrange(num_levels):
new_level = []
for gate_index in xrange(W): # create W new gates
new_gate = gate_maker(L, ultimate_level, penultimate_level,
"".join(["G",
str(unique_gate_num)]), circuit)
# Add the new gate to the new level:
new_level.append(new_gate)
# Increment the unique gate number:
unique_gate_num += 1
# Update the ultimate_level and penultimate_level pointers:
penultimate_level = ultimate_level
ultimate_level = new_level
# Select the output gate from the last level:
output_gate = sr.choice(ultimate_level)
circuit.set_output_gate(output_gate)
return circuit
def generate_two_sided(self, record_set_size, range_size, db_size):
'''Returns a tuple containing a the lower and upper values of a
randomly generated range'''
#Get the desired density by dividing by the range of the bucket
desired_density = record_set_size / range_size
#Find all bit lengths that have that density
density = self.__bucket_density(db_size)
close_enoughs = []
min_bit_length = range_size.bit_length() + 1
for bit_len in xrange(min_bit_length, 65):
if self.__close_enough(density[bit_len], desired_density):
close_enoughs.append(bit_len)
if len(close_enoughs) == 0:
raise FooInputs
elif len(close_enoughs) == 1:
bit_length = close_enoughs[0]
else:
bit_length = spar_random.choice(close_enoughs)
#Generate a random range of bit length range_size
range = 1
for _ in xrange(range_size.bit_length() - 1):
range *= 2
range += spar_random.randbit()
assert range.bit_length() == range_size.bit_length()
#generate random starting point within the bucket for the range,
#the lower value is generated randomly and then multiplied by 2
#because randint cannot compute values larger than 2**63
lower_half = spar_random.randint(2**(bit_length - 1) / 2,
(2**(bit_length) - range_size) / 2)
lower = lower_half * 2
upper = lower + range
return (lower, upper)
def generate(self):
"""Populates the circuit, input and output files with a circuit, an
input, and the corresponding output with the appropriate parameters."""
# create the header and write it to the circuit file:
header_string = self._create_circuit_header()
# create the input wires and write the inputs to the input file:
input_wires = self._create_input_wires()
# set set of all circuit objects already created:
levels = [input_wires]
# initialize the global gate counter, which acts as the unique numerical
# id of each gate:
unique_gate_num_gen = itertools.count()
# for each level:
for level_index in xrange(self._D):
# Create the list of gates at this level:
this_level = [None] * self._W
for gate_ind in xrange(self._W):
displayname = "".join(["G",str(unique_gate_num_gen.next())])
# make the new gate:
new_gate = self._gate_maker(levels, displayname)
new_gate_output = sr.randbit()
new_gate.balance(new_gate_output)
# add this gate to our list of gates at this level:
this_level[gate_ind] = new_gate
# set things up for the next level:
ultimate_level = this_level
levels.append(this_level)
output_gate = sr.choice(levels[-1])
output_gate.set_name("output_gate")
# choose a random output, and write it to the output file:
output = sr.randbit()
self._output_file.write(str(output))
# balance the output gate with respect to the chosen output:
output_gate.balance(output)
circ = sc.StealthCircuit(input_wires, output_gate)
# write the circuit to the circuit file:
self._circuit_file.write(circ.display())
def make_random_gate(L, ultimate_level, penultimate_level, gate_name, circuit):
"""Creates a random gate with uniformly distributed type."""
gate_type = sr.choice(
[g_type for g_type in GATE_TYPES.numbers_generator()])
generate = TEST_TYPE_TO_GATE_MAKER[gate_type]
return generate(L, ultimate_level, penultimate_level, gate_name, circuit)
def create_update_queries(self, mid, mod_field, new_value, orig_value, new_row, \
out_dir):
'''Create update queries. One that matches before the update,
one after, and one both times'''
# Put 3 queries in one query file
# choose a HI_ENTROPY_FIELD other than mod_field
unmatched_entropy_field = list(tu.HI_ENTROPY_FIELDS)
try:
unmatched_entropy_field.remove(mod_field)
except ValueError:
# Value wasn't in the list of hi_entropy_fields
pass
other_field = sprandom.choice(unmatched_entropy_field)
other_field_value = new_row[other_field]
qid_list = list()
# (pre) matches the row before the change
qid = self.qid_id_generator.get_id()
qid_list.append(qid)
# name the query file based on the first qid of the 3 - should be ok
qfilename = os.path.join(out_dir, "update_query_" + str(qid) + ".q")
qfile = open(qfilename, 'w')
# put ZIP last since it will probably have more matches
if mod_field == sv.enum_to_sql_name(sv.VARS.ZIP_CODE):
first_pre_and_term = other_field + ' = ' + \
"'" + other_field_value + "'"
first_post_and_term = first_pre_and_term
second_pre_and_term = mod_field + ' = ' + "'" + orig_value + "'"
second_post_and_term = mod_field + ' = ' + "'" + new_value + "'"
else:
first_pre_and_term = mod_field + ' = ' + "'" + orig_value + "'"
first_post_and_term = mod_field + ' = ' + "'" + new_value + "'"
second_pre_and_term = other_field + ' = ' + \
"'" + other_field_value + "'"
second_post_and_term = second_pre_and_term
where_clause = self.select_str + ' FROM main WHERE ' + \
first_pre_and_term + ' AND ' + second_pre_and_term
self.write_query(qid, where_clause, qfile)
self.fill_mod_query_table(mid, qid, where_clause)
# (post) matches the row after the change
qid = self.qid_id_generator.get_id()
qid_list.append(qid)
where_clause = self.select_str + ' FROM main WHERE ' + \
first_post_and_term + ' AND ' + second_post_and_term
self.write_query(qid, where_clause, qfile)
self.fill_mod_query_table(mid, qid, where_clause)
# (both) matches the row before and after the change
qid = self.qid_id_generator.get_id()
qid_list.append(qid)
where_clause = self.select_str + ' FROM main WHERE ' + other_field +\
' = ' + "'" + other_field_value + "'"
self.write_query(qid, where_clause, qfile)
self.fill_mod_query_table(mid, qid, where_clause)
qfile.close()
return ([qfilename], qid_list)