def check_jobs(input_folder='BATCH'):
"""Checks all jobs found in the input folder or deeper"""
folders = []
statuses = {}
# Finding all input folders
print('### Walking through subfolders of: {0:s}'.format(input_folder))
for root, dirnames, filenames in os.walk(input_folder, topdown=True):
# Skipping folders with split job output
if os.path.split(root)[-1] in ['DONE', 'FAILED', 'OUTPUT']:
dirnames[:] = []
continue
# Checking each job
for dirname in fnmatch.filter(dirnames, 'job_*'):
folder = os.path.normpath(os.path.join(root, dirname))
folders.append(folder)
folders.sort()
# Checking each job
print('### Checking {0:d} jobs:'.format(len(folders)))
n_folders = len(folders)
n_checked = 0
for folder in folders:
print_progress(n_checked, n_folders)
status = check_job(folder)
statuses[folder] = status
n_checked += 1
print('- - - - - - - - - - - - -')
return statuses
def sliding_window_generator(image,
step=(15, 15),
window=(32, 32),
pyramid_firstscale=None,
pyramid_scale=0.5,
pyramid_max_layers=1):
"""
Generator of slices over the entire image in 2D
If pyramid_downscale!=0, will generate slices on reduced image size (maintaining slice size)
in layers (pyramid), until image size is less than window size (first layer is not reduced)
image: 2D image
step: x,y step sizes
window: w,h of the sliding window size
pyramid_ratio: reduction factor at each reduction iteration
returns: image slice data generator for items in the format (y, x, image_region_scaled, pyramid_scale)
"""
print(image.shape)
if (pyramid_firstscale != None):
image = transform.rescale(image, pyramid_firstscale)
print(image.shape)
for im_scaled, scale in pyramid_generator(image,
scale=pyramid_scale,
max_layers=pyramid_max_layers):
if im_scaled.shape[1] < window[1] or im_scaled.shape[1] < window[1]:
return
t = Timer('sliding_window')
# slide a window across the image
for y in range(0, im_scaled.shape[0], step[0]):
utils.print_progress(y,
im_scaled.shape[0],
elapsed_seconds=t.elapsed(),
status='sliding window',
show_remaining=True)
for x in range(0, im_scaled.shape[1], step[1]):
# yield the current window
yield (y, x, im_scaled[y:y + window[0],
x:x + window[1]], scale)
t.stop()
def package_dataset(out_file, data_dir):
unsorted_path = data_dir / "unsorted"
classes_file = data_dir / "classes.txt"
splits_file = data_dir / "splits.txt"
info_file = data_dir / "dataset.info"
changelog = data_dir / "changes.log"
manifest = data_dir / "MANIFEST"
files = [manifest, info_file, classes_file, splits_file, changelog]
files.extend(list(unsorted_path.glob("**/*.*")))
with open(manifest, 'w', encoding='utf-8') as f:
for item in files:
f.write("%s\n" % pathlib.Path(item).relative_to(data_dir))
archive = None
if out_file.suffix == '.zip':
archive = zipfile.ZipFile(out_file, 'w', zipfile.ZIP_LZMA)
for idx, name in enumerate(files):
archive.write(pathlib.Path(name),
arcname=pathlib.Path(name).relative_to(data_dir))
print_progress(idx, len(files) - 1)
elif '.tar' in out_file.suffix == '.tar':
archive = tarfile.open(out_file, "w")
for idx, name in enumerate(files):
archive.add(pathlib.Path(name),
arcname=pathlib.Path(name).relative_to(data_dir))
print_progress(idx, len(files) - 1)
elif out_file.suffixes == ['.tar', '.gz'] or out_file.suffix == '.tgz':
archive = tarfile.open(out_file, "w:gz")
for idx, name in enumerate(files):
archive.add(pathlib.Path(name),
arcname=pathlib.Path(name).relative_to(data_dir))
print_progress(idx, len(files) - 1)
else:
cprint("Unsupported archive extension: {}".format(''.join(
out_file.suffixes)),
color='red',
file=sys.stderr)
archive.close()
def sync_triplets(results, df_events):
"""Synchronise events from triplet results in different SLs that were processed in parallel"""
df_events['MEANTIMER_SL_MULT'] = -1
df_events['MEANTIMER_MIN'] = -1
df_events['MEANTIMER_MAX'] = -1
df_events['MEANTIMER_MEAN'] = -1
df_events['MEANTIMER_MULT'] = -1
df_events['HITS_MULT'] = -1
df_events['HITS_MULT_ACCEPTED'] = -1
if not results:
return
groups = pd.concat([result[2] for result in results]).groupby(EVT_COL)
print('### Performing triplets analysis on {0:d} events'.format(
len(groups)))
# Splitting event numbers into groups with different deviations from the trigger
deviations = [0, 2, 4, 6, 8, 10]
event_deviations = {}
for dev in deviations:
event_deviations[dev] = []
# Analysing each event
n_events = len(groups)
n_events_processed = 0
for event, df in groups:
n_events_processed += 1
print_progress(n_events_processed, n_events)
nHits = df.shape[0]
# Selecting only hits in the acceptance region
sel = pd.concat([(df['SL'] == sl) & (df['TDC_CHANNEL_NORM'].isin(ch))
for sl, ch in ACCEPTANCE_CHANNELS.items()],
axis=1).any(axis=1)
df = df[sel]
nHitsAcc = df.shape[0]
df_events.loc[event,
['HITS_MULT_ACCEPTED', 'HITS_MULT']] = (nHitsAcc, nHits)
# Checking TIME0 found in each chamber
tzeros = {}
# print(event)
# print(df[['SL', 'TDC_CHANNEL_NORM', 'LAYER', 'TIMENS']])
time0 = df_events.loc[event, 'TIME0']
for sl, df_sl in df.groupby('SL'):
# print('--- SL: {0:d}'.format(sl))
nLayers = len(df_sl.groupby('LAYER'))
# Skipping chambers that don't have 3 layers of hits
if nLayers < 3:
continue
tzeros_sl, angles_sl = meantimer_results(df_sl, verbose=False)
if sl not in tzeros:
tzeros[sl] = []
tzeros[sl].extend(tzeros_sl)
meantimers_info = results[sl][3]
for name in [
't0_dev', 't0_angle', 'hit_angles_diff', 'hit_means_diff'
]:
if name not in meantimers_info:
meantimers_info[name] = []
meantimers_info['t0_dev'].extend(
[time0 - tzero for tzero in tzeros_sl])
meantimers_info['t0_angle'].extend(angles_sl)
# print(tzeros_sl)
# if len(df_sl) < 4:
# continue
# # q = df_sl.sort_values('LAYER')['TIME_ABS'].values[:4]
# # tzero = (q[0] + q[3] + 3*q[1] + 3*q[2])/8.0 - 0.5*TDRIFT
# # Skipping chambers that don't have 4 layers of hits
# if nLayers < 4:
# continue
# # Calculating angles between pairs of hits in each chamber
# hits = df_sl.set_index('LAYER')
# br()
# hits = hits['TIMENS'].reindex(range(1,4+1), fill_value=0).values
# angles = np.array([hits[2] - hits[0], hits[1] - hits[3]]) / (ZCELL*2)
# means = [0.5 * (TDRIFT - hits[0] + hits[3]), 0.5 * (TDRIFT - hits[2] + hits[1])]
# meantimers_info['hit_angles_diff'].append(angles[1] - angles[0])
# meantimers_info['hit_means_diff'].append(means[1] - means[0])
# br()
# Calculating the mean of the t0 candidates excluding outliers
tzero, tzeros, nSLs = mean_tzero(tzeros)
if len(tzeros) < 1:
df_events.loc[event, 'MEANTIMER_MULT'] = 0
else:
df_events.loc[event, [
'MEANTIMER_MEAN', 'MEANTIMER_MIN', 'MEANTIMER_MAX',
'MEANTIMER_MULT', 'MEANTIMER_SL_MULT'
]] = (tzero, np.min(tzeros), np.max(tzeros), len(tzeros), nSLs)
deviation = abs(tzero - time0)
for dev in reversed(deviations):
if deviation > float(dev):
event_deviations[dev].append(event)
break
def select_accepted_events(allhits, events):
"""Removes events that don't pass acceptance cuts"""
print('### Removing events outside acceptance')
hits = allhits[allhits['TDC_CHANNEL_NORM'] <= NCHANNELS]
sel = pd.concat([(hits['SL'] == sl) & (hits['TDC_CHANNEL_NORM'].isin(ch))
for sl, ch in ACCEPTANCE_CHANNELS.items()],
axis=1).any(axis=1)
groups = hits[sel].groupby('EVENT_NR')
events_accepted = []
n_events = len(groups)
n_events_processed = 0
print('### Checking {0:d} events'.format(n_events))
events['CELL_HITS_MULT_MAX'] = 1
events['CELL_HITS_DT_MIN'] = -1
events['CELL_HITS_DT_MAX'] = -1
# sl_channels = {2: [3,4,5,6], 3: [1,2,3,4]}
# sl_channels = {2: [5,6,7,8], 3: [3,4,5,6]}
# sl_channels = {2: [7,8,9,10], 3: [5,6,7,8]}
sl_channels = None
for event, df in groups:
n_events_processed += 1
print_progress(n_events_processed, n_events)
# Accepting only specified events if provided
if args.events and event not in args.events:
continue
# Selecting only events that have 1+ hits in a single cell
if args.double_hits:
cellHits = df.groupby(['SL', 'TDC_CHANNEL_NORM'])['TIME_ABS']
nHits_max = cellHits.agg('size').max()
if nHits_max < 2:
continue
dt_min = 999
dt_max = -1
for cell, hits in cellHits:
if len(hits) < 2:
continue
hits_sorted = np.sort(hits.values)
dt_min = min(dt_min, hits_sorted[1] - hits_sorted[0])
dt_max = max(dt_max, hits_sorted[-1] - hits_sorted[0])
events.loc[
event,
['CELL_HITS_MULT_MAX', 'CELL_HITS_DT_MIN', 'CELL_HITS_DT_MAX'
]] = [nHits_max, dt_min, dt_max]
# Accepting the event
events_accepted.append(event)
# Skipping events that don't have hits exactly in the defined channels
channels_ok = True
if sl_channels:
for sl, chs in sl_channels.items():
if sorted(df[df['SL'] == sl]
['TDC_CHANNEL_NORM'].tolist()) == chs:
continue
channels_ok = False
break
if not channels_ok:
continue
tzero_result = event_accepted(df, cut_max_hits=args.event)
if not tzero_result:
continue
# Checking event for acceptance with different t0 candidates
tzero, tzeros, tzeros_all = tzero_result
df_clusters = tzero_clusters(tzeros_all)
nClusters = df_clusters['cluster'].nunique()
# Skipping event if no t0 cluster found
if nClusters < 1:
continue
tzero = -1.0
for cluster_id, df_cluster in df_clusters.groupby('cluster'):
if len(df_cluster) < MEANTIMER_CLUSTER_SIZE:
continue
cluster = df_cluster['t0'].values
tzero_cand = cluster.mean()
# Trying acceptance cuts with a subset of hits from this orbit within the event time window
window = ((df['TIME_ABS'] >= (tzero_cand + TIME_WINDOW[0])) &
(df['TIME_ABS'] <= (tzero_cand + TIME_WINDOW[1]))).index
tzero_result_cand = event_accepted(df.loc[window],
cut_max_hits=True)
if not tzero_result_cand:
continue
tzero = tzero_cand
break
if tzero < 0:
continue
# Accepting the event
events_accepted.append(event)
# Updating the TIME0 with meantimer result
if args.update_tzero or not args.event:
events.loc[event, 'TIME0'] = tzero
if args.event:
# Updating t0 of all hits directly if using external trigger
allhits.loc[allhits['EVENT_NR'] == event, 'TIME0'] = tzero
else:
# Removing the old event number and applying only to hits in the window
allhits.loc[allhits['EVENT_NR'] == event, 'EVENT_NR'] = -1
# Updating t0 of hits in the event time window
start = tzero + TIME_WINDOW[0]
end = tzero + TIME_WINDOW[1]
window = (allhits['TIME_ABS'] >= start) & (allhits['TIME_ABS']
<= end)
allhits.loc[window, ['TIME0', 'EVENT_NR']] = [tzero, event]
events.drop(events.index[~events.index.isin(events_accepted)],
inplace=True)
allhits.drop(allhits.index[~allhits['EVENT_NR'].isin(events_accepted)],
inplace=True)
print('### Selected {0:d}/{1:d} events in acceptance'.format(
len(events_accepted), n_events))
def calc_event_numbers(allhits, runnum, trigger_v1=False):
"""Calculates event number for groups of hits based on trigger hits"""
# Creating a dataframe to be filled with hits from found events (for better performance)
hits = allhits.loc[:1, ['EVENT_NR', 'TIME0']]
# Selecting only hits containing information about the event number or trigger signals
channels_trigger = CHANNELS_TRIGGER
sel = pd.Series(False, allhits.index)
for ch in channels_trigger:
sel = sel | ((allhits['FPGA'] == ch[0]) &
(allhits['TDC_CHANNEL'] == ch[1]))
# Changing selection for the new trigger encoding
if trigger_v1:
sel = allhits['HEAD'] == 3
sel = sel & (allhits['TDC_MEAS'] != 4095)
# Selecting hits that have to be grouped by time
ev_hits = allhits.loc[sel]
print('### Grouping hits by their time of arrival')
# Creating a sorted list of hits with 1 on each change of BX and splitting on each jump in BX number greater than given by EVENT_TIME_GAP
evt_group = (
ev_hits['ORBIT_CNT'].astype(np.uint64) * DURATION['orbit:bx'] +
ev_hits['BX_COUNTER']).sort_values().diff().fillna(0).astype(np.uint64)
evt_group[evt_group <= EVENT_TIME_GAP] = 0
evt_group[evt_group > EVENT_TIME_GAP] = 1
# Calculating cumulative sum to create group ids
evt_group = evt_group.cumsum()
# Adding column to be used for grouping hits with event number and trigger
allhits['EVENT_NR'] = evt_group
allhits['EVENT_NR'] = allhits['EVENT_NR'].fillna(-1).astype(int)
# Getting back rows with relevant channels with grouping column updated
ev_hits = allhits.loc[sel]
ev_hits.set_index(['FPGA', 'TDC_CHANNEL'], inplace=True)
# Checking each group to calculate event number for it
evt_groups = ev_hits.groupby('EVENT_NR')
n_groups = len(evt_groups)
n_groups_done = 0
# Creating a dataframe with 1 row per event
df_events = pd.DataFrame(data={'EVENT_ID': list(evt_groups.groups.keys())})
df_events['TIME0'] = -1
df_events['EVENT_NR'] = -1
df_events['TIMEDIFF_TRG_20'] = -1e9
df_events['TIMEDIFF_TRG_21'] = -1e9
df_events.set_index('EVENT_ID', inplace=True)
df_events.sort_index(inplace=True)
# Calculating event number for each group of hits
for grp, df in evt_groups:
print_progress(n_groups_done, n_groups)
n_groups_done += 1
df = df.sort_index()
evt_id = grp
# Skipping if only one specific event should be processed
if args.events and evt_id not in args.events:
continue
df_events.loc[grp, 'EVENT_NR'] = evt_id
# Getting time and orbit number of the event after duplicates were eliminated
time_event, orbit_event = None, None
if trigger_v1:
orbit_event = df.iloc[0]['ORBIT_CNT']
time_event = orbit_event * DURATION['orbit'] + df.iloc[0][
'TDC_MEAS'] * DURATION['bx']
else:
for ch in channels_trigger:
if ch in df.index:
time_event, orbit_event = df.loc[ch,
('TIME_ABS', 'ORBIT_CNT')]
break
# Looking for other hits within the time window of the event, taking into account latency
# set ttrig based on run number (HARDCODED)
time_offset = TIME_OFFSET
# Defining t0 as time of the trigger channel corrected by latency offset
tzero = time_event + time_offset
############################################
# # FIXME: Correcting TIME0 for this particular event
# tzero += 4.0
############################################
event_window = (tzero + TIME_WINDOW[0], tzero + TIME_WINDOW[1])
try:
window = allhits['TIME_ABS'].between(event_window[0],
event_window[1],
inclusive=False)
# print('event: {0:d} duration: {1:.3f}'.format(evt_id, clock() - start))
except Exception as e:
print('WARNING: Exception when calculating window')
df_events.loc[grp, ['EVENT_NR', 'TIME0']] = (evt_id, tzero)
# Storing hits of the event with corresponding event number and t0
idx = allhits.index[window | (allhits['EVENT_NR'] == grp)]
hits = hits.append(
pd.DataFrame(np.array([evt_id, tzero] * len(idx)).reshape([-1, 2]),
index=idx,
columns=['EVENT_NR', 'TIME0']))
# Updating hits in the main dataframe with EVENT_NR and TIME0 values from detected events
hits['EVENT_NR'] = hits['EVENT_NR'].astype(int)
allhits.loc[hits.index,
['EVENT_NR', 'TIME0']] = hits[['EVENT_NR', 'TIME0']]
# Creating a column with time passed since last event
df_events.set_index('EVENT_NR', inplace=True)
# Removing events that have no hits
if -1 in df_events.index:
df_events.drop(-1, inplace=True)
return df_events
def sort_dataset(splits, classes, unsorted_dir, out_dir):
# Create directories for splits
for split in splits["Names"]:
(out_path / split).mkdir(parents=True, exist_ok=True)
log_file = open(out_dir / "sorting.log", mode="w+")
classes_files = list(unsorted_path.glob("**/_classes.csv"))
manual_classes = [
k for k in classes_files
if str(k.relative_to(unsorted_path).parent) in splits
]
count = 0
copied = 0
skipped = 0
total = count_entries(manual_classes)
for csv in manual_classes:
data = pd.read_csv(csv, dtype={'id': np.int32})
# Directory with video segments corresponding to this csv file
seg_dir = csv.parent / 'segments'
# Set output root directory
curr = str(csv.relative_to(unsorted_path).parent)
if curr in splits:
out_dir = out_path / splits[curr] / curr
out_dir.mkdir(parents=True, exist_ok=True)
curr_splits = 0
# Create output directories for each class
for c in classes:
(out_dir / c).mkdir(parents=True, exist_ok=True)
# For each row of the manual classification file
for index, row in enumerate(data.itertuples()):
infile = seg_dir / "{:06d}.mp4".format(row.id)
print_progress(count, total, end="")
# If second row contains an error type, skip
if pd.notnull(row.notes) and any(
[x in row.notes for x in errors]):
cprint(" Skipped ({:06d}, {}, {}) ".format(
row.id, row.tag, row.notes, row.validated),
'red',
end="\r",
flush=True)
log_file.write(
"Skipped copying file {} with entry ({:06d}, {}, {}). Reason: Matched a known error subclass.\n"
.format(infile, row.id, row.tag, row.notes))
skipped += 1
# If second row contains a speaker change, split video at change and skip
elif pd.notnull(row.notes) and "Speaker Change" in row.notes:
print("", end="\r")
cprint(
"Found Speaker Change in file {} at {:06d}. Splitting data into new set."
.format(csv, row.id),
'blue',
flush=True)
curr_splits += 1
curr = str(csv.relative_to(
unsorted_path).parent) + "-" + str(curr_splits)
out_dir = out_dir.parent / curr
out_dir.mkdir(parents=True, exist_ok=True)
for c in classes:
(out_dir / c).mkdir(parents=True, exist_ok=True)
log_file.write(
"Split video dataset defined in {} at {} due to speaker change. New ID: {}\n"
.format(csv, row.id, curr))
skipped += 1
# If validation failed, skip
elif pd.isnull(row.validated) or row.validated == False:
cprint(" Skipped ({:06d}, {}, {}) ".format(
row.id, row.tag, row.notes, row.validated),
'red',
end="\r",
flush=True)
log_file.write(
"Skipped copying file {} with entry ({:06d}, {}, {}). Reason: Validation failure.\n"
.format(infile, row.id, row.tag, row.notes))
skipped += 1
# If second row is not null, show a message but copy anyways
elif pd.notnull(row.notes):
cprint(" Copying ({:06d}, {}, {}) ".format(
row.id, row.tag, row.notes, row.validated),
'yellow',
end="\r",
flush=True)
outfile = out_dir / row.tag / "{:06d}.mp4".format(row.id)
shutil.copy(str(infile), str(outfile))
copied += 1
# If first row is a valid class name, copy file to output directory
elif row.tag in classes:
print(" Copying ({:06d}, {})... ".format(
row.id, row.tag),
end="\r",
flush=True)
outfile = out_dir / row.tag / "{:06d}.mp4".format(row.id)
shutil.copy(str(infile), str(outfile))
copied += 1
# Otherwise, show an error and skip
else:
print("", end="\r")
cprint(
"Failed to copy ({:06d}, {}, {}): Unknown class '{}'".
format(row.id, row.tag, row.notes, row.tag),
'red',
flush=True)
log_file.write(
"Skipped copying file {} with entry ({:06d}, {}, {}). Reason: Unknown class {}.\n"
.format(infile, row.id, row.tag, row.notes, row.tag))
skipped += 1
count += 1
sys.stdout.flush()
print(
"\rDone. Copied {} files with {} classes. ({} skipped, see log file for details) "
.format(copied, len(classes), skipped))
log_file.close()
def main(args):
vidpath = pathlib.Path(args.video.name).resolve()
diarpath = pathlib.Path(args.diarization.name).resolve()
if args.output is None:
outbase = vidpath.parent / vidpath.stem
args.output = outbase.with_suffix('.diar.mp4')
else:
args.output = pathlib.Path(args.output).resolve()
if (args.output.exists() and args.overwrite != True):
cprint("ERROR: Output file {} already exists.".format(args.output),
'red',
file=sys.stderr)
exit(0)
else:
fd, tmpfile = tempfile.mkstemp(".mp4")
os.close(fd)
capture = cv2.VideoCapture(str(vidpath.absolute()))
if args.chip:
outwidth = 150
outheight = 150
else:
outwidth = int(capture.get(cv2.CAP_PROP_FRAME_WIDTH))
outheight = int(capture.get(cv2.CAP_PROP_FRAME_HEIGHT))
tempout = cv2.VideoWriter(tmpfile, cv2.VideoWriter_fourcc(*'mp4v'),
capture.get(cv2.CAP_PROP_FPS),
(outwidth, outheight))
framecount = capture.get(cv2.CAP_PROP_FRAME_COUNT)
start_time = time.time()
args.diarization.readline()
currframe = 0
for line in args.diarization:
# [ prediction, frame_start, frame_dur, time_start, time_dur, mean_confidence, frame_confidences... ]
params = line.split(',')
for i in range(int(params[2])):
ret, frame = capture.read()
if ret:
color = (0, 0, 0)
if params[0] == 'Idle':
color = (0, 0, 255)
elif params[0] == 'Speak':
color = (0, 255, 0)
if args.chip:
draw_face_chip(frame, color=color)
elif args.overlay:
draw_facial_reco(frame, color=color)
draw_dot(frame, color=color)
tempout.write(frame)
currframe += 1
if not args.silent:
print_progress(currframe, framecount)
else:
raise ("Reached end of video before end of data.")
capture.release()
tempout.release()
if args.audio:
try:
cmd = [
'ffmpeg', '-i', tmpfile, '-i', args.video, '-y',
'-hide_banner', '-loglevel', 'error', '-nostats', '-map',
'0:0', '-map', '1:1?', '-c:v', 'copy', '-c:a', 'copy',
'-shortest', args.output
]
returned_output = subprocess.call(cmd)
os.remove(tmpfile)
except:
shutil.move(tmpfile, args.output)
else:
shutil.move(tmpfile, args.output)
if not args.silent:
print("--- Took {} s --- ".format(time.time() - start_time))