def update_json(inp_data, mode):
updated_data = []
ctr = 0
for example in inp_data:
bmpmaker.clear()
if mode == "relative":
true_seq_relative = RelativeEventSequence.from_eval_str(example["y_ref"])
true_events_absolute = AbsoluteEventSequence.from_relative(true_seq_relative, width, width).events
pred_seq_relative = RelativeEventSequence.from_eval_str(example["y_pred"])
pred_events_absolute = AbsoluteEventSequence.from_relative(pred_seq_relative, width, width).events
elif mode == "cursor":
true_seq_cursor = CursorEventSequence.from_eval_str(example["y_ref"])
true_events_absolute = AbsoluteEventSequence.from_cursor(true_seq_cursor, width, width).events
pred_seq_cursor = CursorEventSequence.from_eval_str(example["y_pred"])
pred_events_absolute = AbsoluteEventSequence.from_cursor(pred_seq_cursor, width, width).events
else:
raise Exception("Unknown or unsupported mode: {}".format(mode))
bmpmaker.process_commands(true_events_absolute)
true_bitmap = np.array(bmpmaker.bitmap)
bmpmaker.clear()
bmpmaker.process_commands(pred_events_absolute)
pred_bitmap = np.array(bmpmaker.bitmap)
hamming = evaluate(true_bitmap, pred_bitmap)
example["hamming_distance"] = hamming
updated_data.append(example)
ctr += 1
if ctr % 500 == 0:
print "Progress: %d" % ctr
return updated_data
def get_event_sequence(action_str, seq_type, bitmap_dim):
if seq_type == "absolute":
event_sequence = AbsoluteEventSequence.from_string(action_str)
elif seq_type == "cursor":
relative_sequence = RelativeEventSequence.from_eval_str(action_str)
abs_sequence = AbsoluteEventSequence.from_relative(relative_sequence, bitmap_dim, bitmap_dim)
event_sequence = CursorEventSequence.from_absolute(abs_sequence)
else:
event_sequence = RelativeEventSequence.from_eval_str(action_str)
return event_sequence.events
def convert_commands_to_bitmap(bmpmaker, commands_str, mode):
bmpmaker.clear()
if mode == "relative":
rel_seq = RelativeEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_relative(rel_seq, bmpmaker.shape()[0], bmpmaker.shape()[1])
elif mode == "cursor":
cur_seq = CursorEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_cursor(cur_seq, bmpmaker.shape()[0], bmpmaker.shape()[1])
bmpmaker.process_commands(abs_seq.events)
return (bmpmaker.bitmap.copy(), bmpmaker.ordered_actions)
def get_event_sequence(action_str, seq_type, bitmap_dim):
if seq_type == "absolute":
event_sequence = AbsoluteEventSequence.from_string(action_str)
elif seq_type == "cursor":
relative_sequence = RelativeEventSequence.from_eval_str(action_str)
abs_sequence = AbsoluteEventSequence.from_relative(
relative_sequence, bitmap_dim, bitmap_dim)
event_sequence = CursorEventSequence.from_absolute(abs_sequence)
else:
event_sequence = RelativeEventSequence.from_eval_str(action_str)
return event_sequence.events
def update_json(inp_data, mode):
updated_data = []
ctr = 0
for example in inp_data:
bmpmaker.clear()
if mode == "relative":
true_seq_relative = RelativeEventSequence.from_eval_str(
example["y_ref"])
true_events_absolute = AbsoluteEventSequence.from_relative(
true_seq_relative, width, width).events
pred_seq_relative = RelativeEventSequence.from_eval_str(
example["y_pred"])
pred_events_absolute = AbsoluteEventSequence.from_relative(
pred_seq_relative, width, width).events
elif mode == "cursor":
true_seq_cursor = CursorEventSequence.from_eval_str(
example["y_ref"])
true_events_absolute = AbsoluteEventSequence.from_cursor(
true_seq_cursor, width, width).events
pred_seq_cursor = CursorEventSequence.from_eval_str(
example["y_pred"])
pred_events_absolute = AbsoluteEventSequence.from_cursor(
pred_seq_cursor, width, width).events
else:
raise Exception("Unknown or unsupported mode: {}".format(mode))
bmpmaker.process_commands(true_events_absolute)
true_bitmap = np.array(bmpmaker.bitmap)
bmpmaker.clear()
bmpmaker.process_commands(pred_events_absolute)
pred_bitmap = np.array(bmpmaker.bitmap)
hamming = evaluate(true_bitmap, pred_bitmap)
example["hamming_distance"] = hamming
updated_data.append(example)
ctr += 1
if ctr % 500 == 0:
print "Progress: %d" % ctr
return updated_data
def convert_commands_to_image_sequence(bmpmaker, imgmaker, commands_str, fprefix, mode):
bmpmaker.clear()
if mode == "relative":
rel_seq = RelativeEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_relative(rel_seq, bmpmaker.shape()[0], bmpmaker.shape()[1])
elif mode == "cursor":
cur_seq = CursorEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_cursor(cur_seq, bmpmaker.shape()[0], bmpmaker.shape()[1])
imgmaker.save_bitmap(bmpmaker.bitmap, "{}{:04d}.gif".format(fprefix,0))
for i,c in enumerate(abs_seq.events):
bmpmaker.process_command(c)
imgmaker.save_bitmap(bmpmaker.bitmap, "{}{:04d}.gif".format(fprefix,i+1))
def convert_commands_to_image_sequence(bmpmaker, imgmaker, commands_str,
fprefix, mode):
bmpmaker.clear()
if mode == "relative":
rel_seq = RelativeEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_relative(rel_seq,
bmpmaker.shape()[0],
bmpmaker.shape()[1])
elif mode == "cursor":
cur_seq = CursorEventSequence.from_eval_str(commands_str)
abs_seq = AbsoluteEventSequence.from_cursor(cur_seq,
bmpmaker.shape()[0],
bmpmaker.shape()[1])
imgmaker.save_bitmap(bmpmaker.bitmap, "{}{:04d}.gif".format(fprefix, 0))
for i, c in enumerate(abs_seq.events):
bmpmaker.process_command(c)
imgmaker.save_bitmap(bmpmaker.bitmap,
"{}{:04d}.gif".format(fprefix, i + 1))
def write_data(rows, key_attrs, image_idx, commands_idx, actions_idx, output_file, include_bitmaps=True, grid_dims=(25,25)):
tokenizer = TreebankWordTokenizer()
bmpmaker = BitmapMaker(grid_dims[0],grid_dims[1])
with open(output_file, "w") as fout:
objs = []
for i,row in enumerate(rows):
if i % 100 == 0:
print " {} ".format(i)
image_url = row[image_idx]
if include_bitmaps:
bitmap = retrieve_bitmap(image_url_to_text_bitmap_url(image_url))
else:
bitmap = None
image_key = image_url_to_key(image_url)
commands = tokenize_description(tokenizer, row[commands_idx])
actions = row[actions_idx]
abs_seq = AbsoluteEventSequence.from_mturk_string(actions).canonicalize()
rel_seq = RelativeEventSequence.from_absolute(abs_seq)
cur_seq = CursorEventSequence.from_absolute(abs_seq)
abs_str = str(abs_seq)
rel_str = str(rel_seq)
cur_str = str(cur_seq)
raw_txt = actions.replace("\r", "").replace("\n"," ")
rel_alignments,rel_align_info = align_strings(commands, rel_str, "relative", "clever", grid_dims[0], backup_using_naive=True)
abs_alignments,abs_align_info = align_strings(commands, abs_str, "absolute", "clever", grid_dims[0], backup_using_naive=True)
join_list_of_lists = lambda l: " ".join(str(i) for i in itertools.chain(*l))
str_rel_alignments = [(c,join_list_of_lists(r)) for c,r in rel_alignments]
str_abs_alignments = [(c,join_list_of_lists(r)) for c,r in abs_alignments]
obj = {}
obj["image_url"] = image_url
obj["image_id"] = image_key
obj["sample_id"] = i
obj["commands"] = commands
obj["actions.absolute"] = abs_str
obj["actions.relative"] = rel_str
obj["actions.cursor"] = cur_str
obj["alignments.relative"] = str_rel_alignments
obj["alignments.relative.info"] = rel_align_info
obj["alignments.absolute"] = str_abs_alignments
obj["alignments.absolute.info"] = abs_align_info
for key,val in key_attrs[image_key].iteritems():
obj[key] = val
if bitmap is not None:
abs_norm_seq = AbsoluteEventSequence.from_relative(rel_seq, grid_dims[0], grid_dims[1])
try:
bmpmaker.clear()
bmpmaker.process_commands(abs_norm_seq.events)
bitmap_mturk_norm = bmpmaker.bitmap.copy()
except:
bitmap_mturk_norm = None
try:
bmpmaker.clear()
bmpmaker.process_commands(abs_seq.events)
bitmap_mturk = bmpmaker.bitmap.copy()
except:
bitmap_mturk = None
obj["actions.absolute.normalized"] = str(abs_norm_seq)
obj["bitmap.orig_generated"] = bitmap_to_list(bitmap.astype(np.int_))
obj["bitmap.from_mturk"] = bitmap_to_list(bitmap_mturk.astype(np.int_))
obj["bitmap.normalized"] = bitmap_to_list(bitmap_mturk_norm)
objs.append(obj)
json.dump(objs, fout, indent=4, sort_keys=True)
print "Created dataset at %s" % output_file
