def main():
parser = argparse.ArgumentParser()
parser.add_argument('--infile', type=str)
parser.add_argument('--outfile', type=str)
parser.add_argument('--nb_examples', type=int, default=5)
args = parser.parse_args()
with open(args.infile) as in_fh:
programs = [Program.parse(l.rstrip()) for l in in_fh]
datas = []
for program in programs:
data = {}
examples = constraint.get_input_output_examples(
program, args.nb_examples)
raw_examples = []
for inputs, output in examples:
raw_inputs = [x.val for x in inputs]
raw_output = output.val
raw_examples.append((raw_inputs, raw_output))
data = dict(program=program.prefix,
examples=raw_examples,
attribute=converter.get_attribute_vec(program))
datas.append(data)
with open(args.outfile, 'w') as out_fh:
for data in datas:
out_fh.write(json.dumps(data) + '\n')
def test_leader_election(solution, examples):
prefix = str(solution)
program = Program.parse(prefix)
consistent = True
for tuples in examples:
#print("tuples:", tuples)
inputs, output = tuples
#print("Inputs:", inputs)
#print("Output:", output)
#print("stripped:", inputs[0])
#print("TYPE:", type(inputs[0]))
#raw_input = []
#for i in range(len(inputs[0])):
# raw_input.append(inputs[0][i])
#raw_input = inputs[0]
#raw_input2 = raw_input.val
#print("Raw input:", raw_input2)
expected = IntValue(monarchical_leader_election(inputs[0].val))
actual = program(inputs[0])
#assertEqual(dfs_actual, dfs_expected)
#print("DFS expected:", dfs_expected)
#print("DFS actual:", dfs_actual)
if (actual != expected):
consistent = False
print("DeepCoder program inconsistent with ground truth program")
break
#assertEqual(dfs_program.toprefix(), prefix)
return consistent
def test_out_of_range(self):
prefix = 'LIST|LIST|LIST|SCAN1L,*,0|SCAN1L,*,3|MAP,/2,4'
inputs = [
ListValue([8, 155, -231, -115, -178, 115, -246, -93, 42, 237, -104, -92,-208, -15, -116, -144, -58, -66, -120]),
ListValue([-2, -16, -8, -4, 5, 6, 4, 4, 7, -5, 0, 8, 2, 10, 12, 10, -4, 14]),
ListValue([-7, -9, 3, -11, -6, 7, -3, -12, -7, -5, 10, 11, -8, -8, 10])
]
program = Program.parse(prefix)
self.assertRaises(OutputOutOfRangeError, program, *inputs)
def test_basic(self):
# takes the second highest negative number
prefix = 'LIST|INT|FILTER,<0,0|SORT,2|REVERSE,3|ACCESS,1,4'
program = Program.parse(prefix)
expected = IntValue(-2)
actual = program(ListValue([1, -5, -3, -4, -2, -1, 2, 3]), IntValue(1))
self.assertEqual(actual, expected)
self.assertEqual(program.toprefix(), prefix)
def test_maxrank(self):
# takes the second highest negative number
prefix = 'LIST|MAXIMUM,0'
program = Program.parse(prefix)
expected = IntValue(3)
actual = program(ListValue([1, -5, -3, -4, -2, -1, 2, 3]))
self.assertEqual(actual, expected)
self.assertEqual(program.toprefix(), prefix)
def test_null_allowed(self):
p = Program.parse('LIST|TAIL,0|ACCESS,1,0')
expected = [
constraint.ListConstraint(lmin=1,
int_constraints=[
constraint.IntConstraint(0, l - 1)
for l in range(constraint.L + 1)
]),
constraint.IntConstraint(0, 256),
constraint.IntConstraint()
]
output_constraint = constraint.IntConstraint()
actual = constraint.propagate_constraints(p, output_constraint)
self.assertEqual(expected, actual)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--infile', type=str)
parser.add_argument('--outfile', type=str)
parser.add_argument('--nb_examples', type=int, default=5)
args = parser.parse_args()
with open(args.infile) as in_fh:
programs = [Program.parse(l.rstrip()) for l in in_fh]
problems = []
for program in tqdm.tqdm(programs, total=len(programs)):
problem = {}
examples = None
# only try twice for speed
for _ in range(2):
try:
examples = constraint.get_input_output_examples(program, args.nb_examples)
# TODO: figure out why OutputOutOfRange happened here in T=4 generation
except (NullInputError, constraint.InvalidConstraintError, constraint.OutputOutOfRangeError):
continue
if not examples:
continue
raw_examples = []
for inputs, output in examples:
raw_inputs = [x.val for x in inputs]
raw_output = output.val
raw_examples.append(dict(inputs=raw_inputs, output=raw_output))
problem = dict(program=program.prefix, examples=raw_examples,
attribute=util.get_attribute_vec(program))
problems.append(problem)
with open(args.outfile, 'w') as out_fh:
print('[', file=out_fh)
for i, problem in enumerate(problems):
trailing_comma = i < len(problems) - 1
util.pretty_print_problem(problem, out_fh, trailing_comma)
print(']', file=out_fh)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--infile', type=str)
parser.add_argument('--outfile', type=str)
parser.add_argument('--nb_examples', type=int, default=5)
parser.add_argument('--nb_inputs', type=int, default=3)
args = parser.parse_args()
with open(args.infile) as in_fh:
line_count = sum([1 for _ in in_fh])
with open(args.infile) as in_fh:
xdata = []
ydata = []
programs = []
pbar = tqdm.tqdm(total=line_count)
for line in in_fh:
pbar.update(1)
program = Program.parse(line.rstrip())
try:
xdata.append(
get_program_row(program, args.nb_examples, args.nb_inputs))
ydata.append(get_attribute_vec(program))
except:
print('prog:', program)
print('constraint:')
for x in constraint.propagate_constraints(program):
print(x)
raise
programs.append(program)
x = collections.defaultdict(list)
for row in xdata:
for k, v in row.items():
x[k].append(v)
np.savez(args.outfile, y=ydata, **x)
def enumerate_programs(input_type_combinations, T, ctx, max_nb_programs):
"""Enumerates programs with T statements that have the same input types.
Each program is pruned and doesn't have any unused inputs or statements.
Arguments:
input_type_combinations (list): list of list of INT or LIST specifying all input types
to search over
T (int): number of statements in each outputted program
ctx (Context): search context
max_nb_programs (int): max number of programs to enumerate.
Returns:
set of programs with input types input_types and exactly T stmts.
"""
programs = []
workers = []
result_queue = multiprocessing.Queue()
stop_queue = multiprocessing.Queue()
for input_types in input_type_combinations:
worker = multiprocessing.Process(target=enumerate_helper,
args=(input_types, T, ctx,
result_queue, stop_queue))
worker.start()
workers.append(worker)
def join():
for worker in workers:
worker.join()
finished_cnt = 0
def all_done():
return finished_cnt == len(workers)
def stop_workers():
for _ in range(len(workers)):
stop_queue.put(1)
def wait_for_workers():
while True:
if not sum([worker.is_alive() for worker in workers]):
return
time.sleep(.1)
with tqdm.tqdm(total=max_nb_programs) as pbar:
stopped = False
while not all_done():
if not stopped and len(programs) >= max_nb_programs:
stopped = True
stop_workers()
try:
result = result_queue.get_nowait()
if result is None:
finished_cnt += 1
continue
program = Program.parse(result)
if len(programs) < max_nb_programs:
programs.append(program)
pbar.update(1)
except queue.Empty:
continue
if not stopped:
stop_workers()
join()
return programs
def test_get_unused_indices(self):
prefix = 'LIST|INT|MAP,*2,0|FILTER,>0,0|FILTER,<0,2'
program = Program.parse(prefix)
expected = {1, 3}
actual = get_unused_indices(program)
self.assertEqual(actual, expected)
def test_prune(self):
prefix = 'LIST|INT|MAP,*2,0|FILTER,>0,0|FILTER,<0,2'
p = Program.parse(prefix)
pp = prune(p)
expected = 'LIST|MAP,*2,0|FILTER,<0,1'
self.assertEqual(pp.toprefix(), expected)
def test_is_same(self):
lhs = Program.parse('LIST|MAXIMUM,0')
rhs = Program.parse('LIST|SCAN1L,max,0|MAXIMUM,1')
self.assertTrue(constraint.is_same(lhs, rhs))