def save(self):
filename = config.hits_dir + '/' + remove_extension(self.show.from_file) + '.hits.json'
# we use a tmp file because json.dump may leave an unparsable file if an error occurs while encoding
with os.tmpfile() as tmp:
json.dump(self.pre_encode(self.json), tmp, indent = 4, sort_keys = True, cls = HitsEncoder)
tmp.seek(0)
open(filename, 'w').write(tmp.read())
def get_modules_as_dict(self,
module_path,
file_ext=".py",
sufix_filter=""):
"""
Finds all the modules int a path and import the module into the dict
:param: module_path:str
:param: file_ext:str
:param: sufix_filter:str
:rtype:dict
"""
size_to_remove = 0
if len(sufix_filter) > 0:
size_to_remove = len(sufix_filter) - len(file_ext)
module_path_abs = module_path
if not module_path_abs[0] == "/":
module_path_abs = os.path.join(self.base_path, module_path)
module_files = [
utils.remove_extension(os.path.basename(f))
for f in utils.file_list_by_extension(module_path_abs, 'py')
if f.endswith(sufix_filter)
]
modules_dict = {}
for m_file in module_files:
module_name = "{}.{}".format(module_path, m_file)
key_name = m_file if size_to_remove == 0 else m_file[:
-size_to_remove]
modules_dict[key_name] = {
"module": importlib.import_module(module_name),
"file": m_file
}
return modules_dict
def update_images_bBox(class_name,
bag,
topic="/dvs/image_raw",
data_folder="data_bBox",
annot_file="annotate.txt"):
bag_name = get_filename(bag.filename, char="/")
bag_name = remove_extension(bag_name)
save_folder = path.join(path.join(data_folder, class_name), bag_name)
save_folder = path.join(save_folder, IMG_FOLDER)
file_name = path.join(save_folder, annot_file)
df = pd.read_csv(file_name)
bridge = CvBridge()
for msg in bag.read_messages(topic):
cv_image = bridge.imgmsg_to_cv2(msg.message, "rgb8")
filename = str(msg.message.header.stamp) + ".png"
cols_of_interest = [
"x", "y", "w", "h", "proba", "class_name", "tagger"
]
for _, r in df.loc[df["boxImg_name"] == filename,
cols_of_interest].iterrows():
x, y, w, h, prob, cl, tagger = r
bbox_color = (0, 255, 0) if tagger == "yolo" else (0, 0, 255)
draw_bBox(cv_image, (x, y), (x + w, y + h),
cl,
prob,
bbox_color=bbox_color)
img_name = path.join(save_folder, filename)
cv2.imwrite(img_name, cv_image)
def plot_all_events_at_image_fr(bag,
topic="/dvs/events",
save_folder="data_bBox",
class_bag="chair",
img_w=346,
img_h=260,
img_annotate_file="annotate.txt"):
save_folder = os.path.join(save_folder, class_bag)
bag_name = get_filename(bag.filename, char="/")
bag_name = remove_extension(bag_name)
save_folder = os.path.join(save_folder, bag_name)
img_annot_file = os.path.join(save_folder, IMG_FOLDER)
img_annot_file = os.path.join(img_annot_file, img_annotate_file)
train_folder = os.path.join(save_folder, TRAIN_FOLDER)
if not os.path.exists(train_folder):
os.makedirs(train_folder)
df = pd.read_csv(img_annot_file)
if df.shape[0] == 0:
print("Class not present in file")
return
df.drop_duplicates(subset='boxImg_name', keep="first", inplace=True)
previous_data = df.iloc[0]
count = 0
events_count = -1
image = def_image(img_h, img_w)
for msg in bag.read_messages(topic):
current_data = df.iloc[count]
img_ts = genpy.Time.from_sec(current_data.timestamp * 1e-9)
for e in msg.message.events:
x, y, p, ts = e.x, e.y, e.polarity, e.ts
while img_ts < ts:
# we already have all events for consecutive images. Save and reset
img_name = str(current_data.timestamp) + ".png"
img_path = os.path.join(train_folder, img_name)
cv2.imwrite(img_path, image)
image = def_image(img_h, img_w)
# esto con un if deberia funcionar porque no debe haber mas de una
# imagen entre dos eventos
count += 1
if count >= df.shape[0]:
return
tmp = df.iloc[count]
previous_data = current_data
current_data = tmp
img_ts = genpy.Time.from_sec(current_data.timestamp * 1e-9)
# color = (0, 0, 255) if p > 0 else (255, 0, 0)
color = (255, 255, 255)
image[y][x] = color
# save last events
img_name = str(current_data.timestamp)
img_path = os.path.join(train_folder, img_name)
cv2.imwrite(img_path, image)
image = def_image(img_h, img_w)
def adapt_label_template(template_json, img_filepath):
img_json = template_json.copy()
img_json['imagePath'] = utils.get_filename(img_filepath)
img_json['imageData'] = None
im = Image.open(img_filepath)
img_width, img_height = im.size
img_json['imageWidth'] = img_width
img_json['imageHeight'] = img_height
# write new label .json file
new_label_fp = utils.remove_extension(img_filepath) + '.json'
utils.write_json(new_label_fp, img_json)
def load(show):
filename = config.hits_dir + '/' + remove_extension(show.from_file) + '.hits.json'
try:
j = json.load(open(filename, 'r'))
j['current'] = numbering_from_str(j['current'])
j['airdates'] = dict((j['current'].from_str(e), datetime.strptime(d, "%Y/%m/%d").date()) for e, d in j.get('airdates', {}).iteritems())
if 'torrents' not in j:
j['torrents'] = {}
for k, v in j['torrents'].iteritems():
j['torrents'][k] = [Torrent(**t) for t in v]
return j
except IOError as ex:
return Hits.default_content(show.start)
def get_label_img_fp_pairs(dir_):
label_img_fp_pairs = []
label_fps = get_label_fps(dir_)
for label_fp in label_fps:
label_fp_base = utils.remove_extension(label_fp)
matching_img_fp = label_fp_base + '.png'
# if image file exists (which it should if there is a corresponding .xml file)
if pathlib.Path(matching_img_fp).is_file():
label_img_fp_pairs.append((label_fp, matching_img_fp))
return label_img_fp_pairs
def get_ep_iter_from_res_name(resname, nbit, batch_size=4):
resname_no_ext = remove_extension(resname)
ffn = [ff[ff.find('_ep') + 3:] for ff in resname_no_ext]
key_list = []
for fff, fffn in zip(ffn, resname):
if '_it' in fff:
ind = fff.find('_it')
ep = int(fff[0:ind])
it = int(fff[ind + 3:])*batch_size
it = 4 if it==0 else it #hack to avoit 2 identical point (as val is done for it 0 and las of previous ep
else:
ep = int(fff)
it = nbit
key_list.append([fffn, ep, it])
aa = np.array(sorted(key_list, key=lambda x: (x[1], x[2])))
name_sorted, ep_sorted, it_sorted = aa[:, 0], aa[:, 1], aa[:, 2]
ep_sorted = np.array([int(ee) for ee in ep_sorted])
it_sorted = np.array([int(ee) for ee in it_sorted])
ep_sorted = ep_sorted - ep_sorted[0] #so that the first is 0
return name_sorted, ep_sorted, it_sorted
ftrain = gfile(oneres, 'res_train_ep01.csv')
rrt=pd.read_csv(ftrain[0])
nb_it = rrt.shape[0];
resdir, resname = get_parent_path(fres_valOn)
resname_sorted, b, c = get_ep_iter_from_res_name(resname, 0)
resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in resname_sorted]
resdic = {}
for ii, fres in enumerate(resname_sorted):
iind = [i for i, s in enumerate(data_name_list) if s in fres]
if len(iind) != 1: raise("bad size")
data_name = data_name_list[iind[0]]
iind = fres.find(data_name)
ddn = remove_extension( fres[iind+len(data_name)+1:] )
new_col_name = 'Mout_' + ddn
iind = ddn.find('model_ep')
transfo = ddn[:iind-1]
#data_name = data_name + '_' + transfo
if data_name not in resdic: resdic[data_name] = {}
if transfo not in resdic[data_name]:
resdic[data_name][transfo] = resV[ii]
else:
if np.any(resV[ii].L1-resdic[data_name][transfo].L1): raise('not consistent')
resdic[data_name][transfo][new_col_name] = resV[ii]['model_out']
for data_name, val_dic in resdic.items():
for transfo, df in val_dic.items():
width = 1440
height = 1080
framerate = 30
first_item_length = 10
item_length = 3
last_item_length = 10
transition_time = 0.5
output_name = "out.mp4"
pics = os.listdir(pics_dir)
pics.sort(key=str.lower)
sequence = []
for pic in pics:
removed_extension_pic = utils.remove_extension(pic)
pic_with_path = utils.join_path_segments(pics_dir, pic)
rotation_fixed_file_name = utils.join_path_segments(
pics_dir, removed_extension_pic + "_rotation_fixed.png")
long_file_name = utils.join_path_segments(
vids_dir, removed_extension_pic + "_long.mp4")
short_file_name = utils.join_path_segments(
vids_dir, removed_extension_pic + "_short.mp4")
merged_file_name = utils.join_path_segments(
vids_dir, removed_extension_pic + "_merged.mp4")
sequence.append([
pic_with_path, rotation_fixed_file_name, long_file_name,
short_file_name, merged_file_name
])
print(len(sequence))
def inline_buttons_handler(bot, update):
from app import app, db
query = update.callback_query
chat_id = query.message.chat_id
logger.debug("Got an inline button action: %s" % query.data)
bot.send_chat_action(chat_id=chat_id, action=telegram.ChatAction.TYPING)
# Try to get params
try:
params = json.loads(query.data)
action = params.get("action")
userfile_id = int(params.get("uf"))
except Exception as e:
logger.error(e)
bot.send_message(
chat_id=chat_id,
text="\n".join(
[
"Упс! Что-то пошло не так 😱",
"Передайте это администратору, чтобы он все исправил:",
"Query data: %s" % query.data,
"Exception: %s" % e,
]
),
)
raise
# Try to get info about file from db
file_info = get_file_info(bot, userfile_id)
if action in ACTIONS_MAPPING:
outfile = os.path.join(
app.config["PROCESSED_DIR"],
"%s %s %s.zip"
% (
remove_extension(file_info["filename"]),
file_info["userfile_id"],
action,
),
)
bot.send_message(text="Сейчас посмотрю...⏳", chat_id=chat_id)
try:
extract_file(bot, chat_id, file_info)
statuses = ACTIONS_MAPPING[action](file_info["extract_path"])
if any(statuses.values()):
zipdir(file_info["extract_path"], outfile)
bot.send_message(chat_id=chat_id, text="Готово!🚀")
bot.send_document(
chat_id=chat_id,
document=open(outfile, "rb"),
filename=os.path.basename(outfile),
reply_to_message_id=file_info["message_id"],
)
if not all(statuses.values()):
message = "⚠️ Следующие файлы не удалось обработать: ⚠️\n"
for file, status in statuses.items():
if not status:
file_path = os.path.relpath(
file, file_info["extract_path"]
)
# Telegram has limit for message length, so we
# split the message in case it is too long (> 4096)
if len(message) + len(file_path) + 10 < 4096:
message += f"\n ❌ {file_path}"
else:
bot.send_message(chat_id=chat_id, text=message)
message = f" ❌ {file_path}"
bot.send_message(chat_id=chat_id, text=message)
else:
bot.send_message(
chat_id=chat_id,
text="Не удалось обработать данные. Проверьте, что файлы предоставлены в нужном формате.",
)
except Exception as e:
logger.error(e)
bot.send_message(
chat_id=chat_id,
text="\n".join(
[
"Упс! Что-то пошло не так 😱",
"Передайте это администратору, чтобы он все исправил:",
"Query data: %s" % query.data,
"Exception: %s" % e,
]
),
)
raise
else:
bot.send_message(
chat_id=chat_id,
text="Данная команда в процессе реализации и пока не доступна 😞",
)
return "OK"
print(usage)
exit()
# build local variables from command line arguments
video_filepath = args[1]
ffmpeg_args = args[2]
print('\n')
print('VIDEO_FILEPATH=%s' % video_filepath)
print('FFMPEG_ARGS=%s' % ffmpeg_args)
print('\n\n')
# parse video filename
video_filename = utils.get_filename(video_filepath)
video_name = utils.remove_extension(video_filename)
# make output directory
output_dir = "output"
frames_dir = os.path.join(output_dir, video_name, 'frames')
os.makedirs(frames_dir, exist_ok=True)
# build ffmpeg command
ffmpeg_command = "ffmpeg -i '%s' %s '%s/frame-%%06d.png' -hide_banner" \
% (video_filepath, ffmpeg_args, frames_dir)
# log ffmpeg command
command_filepath = os.path.join(frames_dir, 'command')
print('\nLogging ffmpeg command in \'%s\'...' % command_filepath)
with open(command_filepath, 'w+') as command_file:
command_file.write(ffmpeg_command)
def anotate_events(bag,
topic="/dvs/events",
save_folder="data_bBox",
class_bag="chair",
start_indx=0,
how_many=-1,
img_annotate_file="annotate.txt"):
save_folder = os.path.join(save_folder, class_bag)
bag_name = get_filename(bag.filename, char="/")
bag_name = remove_extension(bag_name)
save_folder = os.path.join(save_folder, bag_name)
img_annot_file = os.path.join(save_folder, IMG_FOLDER)
img_annot_file = os.path.join(img_annot_file, img_annotate_file)
save_folder = os.path.join(save_folder, EVENT_FOLDER)
if not os.path.exists(save_folder):
os.makedirs(save_folder)
df = pd.read_csv(img_annot_file)
if df.empty:
print("Empty file")
return
df = df.sort_values(by=['timestamp'])
previous_data = df.loc[df["timestamp"] == df.iloc[0].timestamp]
count = 0
file_data = os.path.join(save_folder, "annotate_events.txt")
file_data_exist = os.path.exists(file_data)
with open(file_data, 'a') as fd:
if not file_data_exist:
# estamos creando el archivo... escribir cabeceras
fd.write(
"timestamp, x, y, p, img_name, class_name, box_x, box_y, box_w, box_h, box_prob, box_tagger\n"
)
events_count = -1
for msg in bag.read_messages(topic):
current_data = df.iloc[count]
img_ts = genpy.Time.from_sec(current_data.timestamp * 1e-9)
# use all boxes with the same timestamp
current_data = df.loc[df["timestamp"] == current_data.timestamp]
for e in msg.message.events:
x, y, p, ts = e.x, e.y, e.polarity, e.ts
while img_ts < ts:
count += 1
if count >= df.shape[0]:
return
tmp = df.iloc[count]
img_ts = genpy.Time.from_sec(tmp.timestamp * 1e-9)
tmp = df.loc[df["timestamp"] == tmp.timestamp]
previous_data = current_data
current_data = tmp
series_data = is_in_union_box(x, y, previous_data,
current_data, class_bag)
if series_data is not None:
# para solo procesar un numero determinado de eventos en cada corrida
events_count += 1
if events_count < start_indx: continue
if how_many > 0 and (start_indx + how_many <=
events_count):
return
fd.write(
'{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}\n'.
format(str(ts), x, y, p, series_data.boxImg_name,
series_data.class_name, series_data.x,
series_data.y, series_data.w, series_data.h,
series_data.proba, series_data.tagger))
def save_images(bag, topic="/dvs/image_raw", save_folder="data_bBox", class_bag="chair", start_indx_img=0, how_many=-1):
bridge = CvBridge()
save_folder = os.path.join(save_folder, class_bag)
bag_name = get_filename(bag.filename, char="/")
bag_name = remove_extension(bag_name)
save_folder = os.path.join(save_folder, bag_name)
save_folder = os.path.join(save_folder, IMG_FOLDER)
clean_img_folder = os.path.join(save_folder, CLEAN_IMG_FOLDER)
if not os.path.exists(save_folder):
os.makedirs(save_folder)
if not os.path.exists(clean_img_folder):
os.makedirs(clean_img_folder)
net = load_yolov3()
file_data = os.path.join(save_folder, "annotate.txt")
file_data_exist = os.path.exists(file_data)
with open(file_data, 'a') as fd:
if not file_data_exist:
# estamos creando el archivo... escribir cabeceras
fd.write("timestamp,boxImg_name,class_name,proba,x,y,w,h,tagger\n")
count = -1
for msg in bag.read_messages(topic):
count += 1
if count < start_indx_img: continue # esta imagen no es de interes (probablemente ya ha sido analizada)
if how_many > 0 and (start_indx_img + how_many <= count):
break # las imagenes de aqui en adelante no interesan, seguramente seran analizadas en otro momento
# timestamp = msg.timestamp
timestamp = msg.message.header.stamp
filename = str(timestamp) + ".png"
cv_image = bridge.imgmsg_to_cv2(msg.message, "rgb8")
# esto para los datos del ornitoptero en el que las imagenes estan rotadas
cv_image = cv2.rotate(cv_image, cv2.ROTATE_180)
# solo salvar de manera temporal para usar darknet
tmp_name = os.path.join(clean_img_folder, filename)
cv2.imwrite(tmp_name, cv_image)
pred, img = get_image_with_bb(net, tmp_name)
img_name = os.path.join(save_folder, filename)
cv2.imwrite(img_name, img)
for p in pred:
class_name, prob, bbox = p
x, y, w, h = bbox
fd.write("{0},{1},{2},{3},{4},{5},{6},{7},{8}\n".format(
str(timestamp),
filename,
class_name,
prob,
x, y, w, h, "yolo")
)
if not len(pred):
# esta imagen no tiene predicciones
# poner -1 y revisarla luego
fd.write("{0},{1},{2},{3},{4},{5},{6},{7},{8}\n".format(
str(timestamp),
filename,
"-1",
-1,
-1, -1, -1, -1, "yolo")
)
def generate_tasks_with_oracle_fixed_count_1_task_1_story(
world_paths, output_dir_path, n, noise=.1, train_noise=False):
"""Generates stories with guarantee that each task is seen n times."""
mkdir_p(output_dir_path)
n = n[0]
for world in world_paths:
w = World()
w.load(world)
world_name = remove_extension(world)
# ----------------------------- TRAINING ----------------------------- #
# Create folder to contain data
folder_name = '%s_nex_%d_%d' % (world_name, n, noise * 100)
logging.info("Creating New task in %s..." % folder_name)
mkdir_p(os.path.join(output_dir_path, folder_name))
train_file_path = os.path.join(output_dir_path, folder_name,
'qa21_task_AB_train.txt')
# Define task creator and task types
task = Specify_Tasks()
tasks = ['tb', 'fb', 'sofb']
questions = ['memory', 'reality', 'search', 'belief']
with open(train_file_path, 'w') as f:
# Generate all combinations of tasks and questions
task_questions = list(itertools.product(tasks, questions)) * n
random.shuffle(task_questions)
import pdb
pdb.set_trace()
# Create story for each task-question combo
stories = []
for ts, qs in task_questions:
if train_noise:
story = task.generate_story(w,
1,
tasks=[ts],
questions=[qs],
num_agents=4,
num_locations=6,
statement_noise=noise)
else:
story = task.generate_story(w,
1,
tasks=[ts],
questions=[qs],
num_agents=4,
num_locations=6)
f.write('\n'.join(stringify(story)))
f.write('\n')
# ---------------------------- VAL + TEST ---------------------------- #
# Iterate through all testing conditions
combo = itertools.product(tasks, questions, ['val', 'test'])
for task_type, question, data_set in combo:
fname = '%s_%s_%s_test.txt' % (task_type, question, data_set)
path = os.path.join(output_dir_path, folder_name, fname)
with open(path, 'w') as f:
stories = []
for i in range(n):
story = task.generate_story(w,
1, [task_type], [question],
num_agents=4,
num_locations=6,
statement_noise=noise)
f.write('\n'.join(stringify(story)))
f.write('\n')
def get_pandadf_from_res_valOn_csv(dres, resname, csv_regex='res_valOn', data_name_list=None,
select_last=None, target='ssim', target_scale=1):
if len(dres) != len(resname) : raise('length problem between dres and resname')
resdf_list = []
for oneres, resn in zip(dres, resname):
fres_valOn = gfile(oneres, csv_regex)
print('Found {} <{}> for {} '.format(len(fres_valOn), csv_regex, resn))
if len(fres_valOn) == 0:
continue
ftrain = gfile(oneres, 'res_train_ep01.csv')
rrt = pd.read_csv(ftrain[0])
nb_it = rrt.shape[0];
resdir, resname_val = get_parent_path(fres_valOn)
resname_sorted, b, c = get_ep_iter_from_res_name(resname_val, 0)
if select_last is not None:
if select_last<0:
resname_sorted = resname_sorted[select_last:]
else:
nb_iter = b*nb_it+c
resname_sorted = resname_sorted[np.argwhere(nb_iter > select_last)[1:8]]
resV = [pd.read_csv(resdir[0] + '/' + ff) for ff in resname_sorted]
resdf = pd.DataFrame()
for ii, fres in enumerate(resname_sorted):
iind = [i for i, s in enumerate(data_name_list) if s in fres]
if len(iind) ==1: #!= 1: raise ("bad size do not find which sample")
data_name = data_name_list[iind[0]]
else:
data_name = 'res_valds'
iind = fres.find(data_name)
ddn = remove_extension(fres[iind + len(data_name) + 1:])
new_col_name = 'Mout_' + ddn
iind = ddn.find('model_ep')
if iind==0:
transfo='raw'
else:
transfo = ddn[:iind - 1]
if transfo[0] == '_': #if start with _ no legend ... !
transfo = transfo[1:]
model_name = ddn[iind:]
aa, bb, cc = get_ep_iter_from_res_name([fres], nb_it)
nb_iter = bb[0] * nb_it + cc[0]
rr = resV[ii].copy()
rr['evalOn'], rr['transfo'] = data_name, transfo
rr['model_name'], rr['submodel_name'], rr['nb_iter'] = resn, model_name, str(nb_iter)
rr[target] = rr[target] * target_scale
resdf = pd.concat([resdf, rr], axis=0, sort=True)
resdf['error'] = resdf[target] - resdf['model_out']
resdf['error_abs'] = np.abs(resdf[target] - resdf['model_out'])
resdf_list.append(resdf)
return resdf_list
config = configure["main_configuration"]
logging.info('Config: ' + json.dumps(config, indent=2))
print("Data extraction\nConfiguration: ")
print(json.dumps(config, indent=2), end='\n')
print("Reading run pool files ...")
qrels = get_qrels(config["relevance_judgements"]) if bool(
config["relevance_judgements"]) else []
for run in os.listdir(config["run_pool"]):
print("Run : ", run)
relations = run2relations(join(config["run_pool"],
run), config["labels"],
config["scales"], config["ranks"], qrels,
config["max_rank"]) # ((q, doc), rel)
with open(
join(config["output_folder"], remove_extension(run)) +
".label", 'w') as out:
for r in tqdm(relations):
out.write("{q}\t{d}\t{rel}\n".format(q=r[0][0],
d=r[0][1],
rel=r[1]))
print("Parse queries ...")
write_queries_to_file(config["train_queries"],
join(config["out_queries"], "train_queries.txt"),
config["train_query_format"])
write_queries_to_file(config["test_queries"],
join(config["out_queries"], "test_queries.txt"),
config["test_query_format"])
print("Queries ok.")
print('Done.')
