def intersect_and_copy_upsampling(self, ref_all, relevant_entry, epsilon,
log_file, feature_info):
#ref_all: reference_entries[other_key][entry_key]['intervals'] e.g. [COVAREP][some video id]['intervals']
#relevant_entry: relevant_entries[other_key][entry_key] e.g. [COVAREP][some video id]
#epsilon: error allowed in alignment
#ref_time < one interval in relevant_entry
pbar_small = log.progress_bar(total=ref_all.shape[0],
unit=" Segments",
leave=False)
pbar_small.set_description("Aligning: " + feature_info)
sub = relevant_entry["intervals"]
features = relevant_entry["features"]
#finding where intersect happens
pointer_b = 0 # for relevant_entry
aligned_sub = []
aligned_feature = []
for i, inter in enumerate(ref_all):
#print(pointer_b)
if (abs(inter[0] - inter[1]) < epsilon):
pbar_small.update(1)
continue
pointer_c = pointer_b
while (pointer_c < sub.shape[0]):
if (inter[0] - sub[pointer_c][0]) > (-epsilon) and (
sub[pointer_c][1] - inter[0]) > (-epsilon):
aligned_sub.append(sub[pointer_c])
aligned_feature.append(features[pointer_c])
break
else:
pointer_c += 1
if pointer_c == sub.shape[0]:
diff = list(map(lambda x: abs(inter[0] - x), sub[:, 0]))
min_diff = min(diff)
pointer_c = diff.index(min_diff)
with open(log_file, 'w+') as fi:
fi.write(
'no corresponding frame, find the closest one, {}, difference: {}\n'
.format(feature_info, min_diff))
aligned_sub.append(sub[pointer_c])
aligned_feature.append(features[pointer_c])
else:
pointer_b = pointer_c
pbar_small.update(1)
aligned_sub = np.array(aligned_sub)
aligned_feature = np.array(aligned_feature)
zero_idx = np.where(np.isinf(aligned_feature))
aligned_feature[zero_idx] = 0
pbar_small.close()
return aligned_sub, aligned_feature
def upsampling(self, relevant_entry, video_id):
interval = relevant_entry['intervals']
feature = relevant_entry['features']
shape = relevant_entry['features'].shape[1]
pbar_small = log.progress_bar(total=interval.shape[0],
unit=" Segments",
leave=False)
pbar_small.set_description("Aligning: " + video_id)
last_frame = math.floor(np.amax(interval) * 100)
#first_frame = math.floor(np.amin(interval) * 100)
#length = max(last_frame, last_frame - first_frame)
upsampled_feature = np.zeros((last_frame + 1, shape))
for i, inter in enumerate(interval):
for idx in range(math.floor(inter[0] * 100),
math.floor(inter[1] * 100)):
upsampled_feature[idx] = feature[i]
pbar_small.update(1)
pbar_small.close()
return upsampled_feature
def read_URL(url, destination):
if destination is None:
log.error("Destination is not specified when downloading data",
error=True)
if os.path.isdir(destination.rsplit(os.sep, 1)[-2]) is False:
os.mkdir(destination.rsplit(os.sep, 1)[-2])
if (os.path.isfile(destination)):
log.error("%s file already exists ..." % destination, error=True)
r = requests.get(url, stream=True)
if r.status_code != 200:
log.error('URL: %s does not exist' % url, error=True)
# Total size in bytes.
total_size = int(r.headers.get('content-length', 0))
block_size = 1024
unit = total_size / block_size
wrote = 0
with open(destination, 'wb') as f:
log.status("Downloading from %s to %s..." % (url, destination))
pbar = log.progress_bar(total=math.ceil(total_size // block_size),
data=r.iter_content(block_size),
postfix="Total in kBs",
unit='kB',
leave=False)
for data in pbar: #unit_scale=True,
wrote = wrote + len(data)
f.write(data)
pbar.close()
if total_size != 0 and wrote != total_size:
log.error("Error downloading the data to %s ..." % destination,
error=True)
log.success("Download complete!")
return True
def write_CSD(data,metadata,rootName,destination,compression,compression_opts,full_chunk_shape):
log.status("Writing the <%s> computational sequence data to %s"%(rootName,destination))
if compression is not None:
log.advise("Compression with %s and opts -%d"%(compression,compression_opts))
#opening the file
writeh5Handle=h5py.File(destination,'w')
#creating the root handle
rootHandle=writeh5Handle.create_group(rootName)
#writing the data
dataHandle=rootHandle.create_group("data")
pbar = log.progress_bar(total=len(data.keys()),unit=" Computational Sequence Entries",leave=False)
for vid in data:
vidHandle=dataHandle.create_group(vid)
if compression is not None:
vidHandle.create_dataset("features",data=data[vid]["features"],compression=compression,compression_opts=compression_opts)
vidHandle.create_dataset("intervals",data=data[vid]["intervals"],compression=compression,compression_opts=compression_opts)
else:
vidHandle.create_dataset("features",data=data[vid]["features"])
vidHandle.create_dataset("intervals",data=data[vid]["intervals"])
pbar.update(1)
pbar.close()
log.success("<%s> computational sequence data successfully wrote to %s"%(rootName,destination))
log.status("Writing the <%s> computational sequence metadata to %s"%(rootName,destination))
#writing the metadata
metadataHandle=rootHandle.create_group("metadata")
for metadataKey in metadata.keys():
metadataHandle.create_dataset(metadataKey,(1,),dtype=h5py.special_dtype(vlen=unicode) if sys.version_info.major is 2 else h5py.special_dtype(vlen=str))
cast_operator=unicode if sys.version_info.major is 2 else str
metadataHandle[metadataKey][0]=cast_operator(json.dumps(metadata[metadataKey]))
writeh5Handle.close()
log.success("<%s> computational sequence metadata successfully wrote to %s"%(rootName,destination))
log.success("<%s> computational sequence successfully wrote to %s ..."%(rootName,destination))
def align(self, reference, collapse_functions=None, replace=True):
aligned_output = {}
for sequence_name in self.computational_sequences.keys():
aligned_output[sequence_name] = {}
if reference not in self.computational_sequences.keys():
log.error("Computational sequence <%s> does not exist in dataset" %
reference,
error=True)
refseq = self.computational_sequences[reference].data
#unifying the dataset, removing any entries that are not in the reference computational sequence
self.unify()
#building the relevant entries to the reference - what we do in this section is simply removing all the [] from the entry ids and populating them into a new dictionary
log.status(
"Pre-alignment based on <%s> computational sequence started ..." %
reference)
relevant_entries = self.__get_relevant_entries(reference)
log.status("Alignment starting ...")
pbar = log.progress_bar(total=len(refseq.keys()),
unit=" Computational Sequence Entries",
leave=False)
pbar.set_description("Overall Progress")
for entry_key in list(refseq.keys()):
pbar_small = log.progress_bar(
total=refseq[entry_key]['intervals'].shape[0],
unit=" Segments",
leave=False)
pbar_small.set_description("Aligning %s" % entry_key)
for i in range(refseq[entry_key]['intervals'].shape[0]):
#interval for the reference sequence
ref_time = refseq[entry_key]['intervals'][i, :]
#we drop zero or very small sequence lengths - no align for those
if (abs(ref_time[0] - ref_time[1]) < epsilon):
pbar_small.update(1)
continue
#aligning all sequences (including ref sequence) to ref sequence
for otherseq_key in list(self.computational_sequences.keys()):
if otherseq_key != reference:
intersects, intersects_features = self.__intersect_and_copy(
ref_time,
relevant_entries[otherseq_key][entry_key], epsilon)
else:
intersects, intersects_features = refseq[entry_key][
'intervals'][i, :][None, :], refseq[entry_key][
'features'][i, :][None, :]
#there were no intersections between reference and subject computational sequences for the entry
if intersects.shape[0] == 0:
continue
#collapsing according to the provided functions
if type(collapse_functions) is list:
intersects, intersects_features = self.__collapse(
intersects, intersects_features,
collapse_functions)
if (intersects.shape[0] != intersects_features.shape[0]):
log.error(
"Dimension mismatch between intervals and features when aligning <%s> computational sequences to <%s> computational sequence"
% (otherseq_key, reference),
error=True)
aligned_output[otherseq_key][entry_key + "[%d]" % i] = {}
aligned_output[otherseq_key][entry_key + "[%d]" %
i]["intervals"] = intersects
aligned_output[otherseq_key][
entry_key +
"[%d]" % i]["features"] = intersects_features
pbar_small.update(1)
pbar_small.close()
pbar.update(1)
pbar.close()
log.success("Alignment to <%s> complete." % reference)
if replace is True:
log.status(
"Replacing dataset content with aligned computational sequences"
)
self.__set_computational_sequences(aligned_output)
return None
else:
log.status(
"Creating new dataset with aligned computational sequences")
newdataset = mmdataset({})
newdataset.__set_computational_sequences(aligned_output,
metadata_copy=False)
return newdataset
def align_upsampling_and_save(self,
reference,
id_idx,
collapse_function=None,
epsilon=10e-6):
folder = '/data/mifs_scratch/yw454/cmumosei_aligned'
log_file = './mosei_alignment_log.txt'
#aligned_output = {}
count = 0
##self.computational_sequences.keys are COVERAP, OpenFace, WordVec, etc
#for sequence_name in self.computational_sequences.keys():
# #init a dictionary to store different featues seperately
# aligned_output[sequence_name]={}
if reference not in self.computational_sequences.keys():
log.error("Computational sequence <%s> does not exist in dataset" %
reference,
error=True)
#get data of reference feature
refseq = self.computational_sequences[reference].data
#unifying the dataset, removing any entries that are not in the reference computational sequence
self.unify()
#building the relevant entries to the reference - what we do in this section is simply removing all the [] from the entry ids and populating them into a new dictionary
log.status(
"Pre-alignment based on <%s> computational sequence started ..." %
reference)
relevant_entries = self.get_relevant_entries(reference)
log.status("Alignment starting ...")
pbar = log.progress_bar(total=len(refseq.keys()),
unit=" Computational Sequence Entries",
leave=False)
pbar.set_description("Overall Progress")
# for some_id in all video ids
for entry_key in list(refseq.keys()):
if entry_key in id_idx:
stored_idx = id_idx.index(entry_key)
#if stored_idx < 104 or (stored_idx > 104 and stored_idx < 1781):
if stored_idx < 1781 or stored_idx == 1815:
continue
all_intersects = {}
all_intersect_features = {}
#for sequence_name in self.computational_sequences.keys():
# all_intersects[sequence_name] = []
# all_intersect_features[sequence_name] = []
ref_all = refseq[entry_key]['intervals']
#aligning all sequences to ref sequence (previous: align refer to refer as well, now: not include refer)
#otherseq_key: other features; entry_key: some video id
for otherseq_key in list(self.computational_sequences.keys()):
if otherseq_key != reference:
feature_info = 'reference: {}, other feature {}, video id: {}'.format(
reference, otherseq_key, entry_key)
intersects, intersects_features = self.intersect_and_copy_upsampling(
ref_all, relevant_entries[otherseq_key][entry_key],
epsilon, log_file, feature_info)
else:
intersects, intersects_features = refseq[entry_key][
'intervals'][:, :], refseq[entry_key]['features'][:, :]
#print(type(intersects[0]))
#print(type(intersects_features[0]))
#print(len(intersects[0]))
#print(len(intersects_features[0]))
all_intersects[otherseq_key] = intersects
all_intersect_features[otherseq_key] = intersects_features
#save features per video
for sequence_name in self.computational_sequences.keys():
video_code = id_idx.index(entry_key)
video_code = str(video_code).zfill(6)
save_htk_format(all_intersect_features[sequence_name],
sequence_name, folder, video_code)
save_intervals(all_intersects[sequence_name], sequence_name,
folder, video_code)
print('alignment saved for video {} feature {}.'.format(
video_code, sequence_name))
pbar.update(1)
pbar.close()
def upsampling_and_save(self,
reference,
id_idx,
collapse_function=None,
epsilon=10e-6):
folder = '/data/mifs_scratch/yw454/cmumosei_aligned'
#not_enough_label_file = './mosei_notenough_lable_videos.txt'
##self.computational_sequences.keys are COVERAP, OpenFace, WordVec, etc
#for sequence_name in self.computational_sequences.keys():
# #init a dictionary to store different featues seperately
# aligned_output[sequence_name]={}
if reference not in self.computational_sequences.keys():
log.error("Computational sequence <%s> does not exist in dataset" %
reference,
error=True)
modality = list(self.computational_sequences.keys())
support = ['COVAREP', 'WordVec']
for m in modality:
if m not in support:
raise ValueError('feature type not supported {}'.format(m))
#get data of reference feature
refseq = self.computational_sequences[reference].data
#unifying the dataset, removing any entries that are not in the reference computational sequence
self.unify()
#building the relevant entries to the reference - what we do in this section is simply removing all the [] from the entry ids and populating them into a new dictionary
log.status(
"Pre-alignment based on <%s> computational sequence started ..." %
reference)
relevant_entries = self.get_relevant_entries(reference)
log.status("Alignment starting ...")
pbar = log.progress_bar(total=len(refseq.keys()),
unit=" Computational Sequence Entries",
leave=False)
pbar.set_description("Overall Progress")
# for some_id in all video ids
for entry_key in list(refseq.keys()):
not_enough_label = False
if entry_key not in ALL_VIDEO:
continue
if entry_key in id_idx:
stored_idx = id_idx.index(entry_key)
if stored_idx <= 2132:
#if stored_idx != 1781:
continue
video_code = id_idx.index(entry_key)
video_code = str(video_code).zfill(6)
for otherseq_key in list(self.computational_sequences.keys()):
if otherseq_key == reference:
# save reference (COVAREP) data
processed_feature = refseq[entry_key]['features'][:, :]
else:
#save upsampled (wordvec) data
processed_feature = self.upsampling(
relevant_entries[otherseq_key][entry_key], entry_key)
save_htk_format(processed_feature, otherseq_key, folder,
video_code)
print('alignment saved for video {} feature {}.'.format(
video_code, otherseq_key))
pbar.update(1)
pbar.close()
def validate_data_format(data, root_name, verbose=True):
log.status(
"Checking the format of the data in <%s> computational sequence ..." %
root_name)
pbar = log.progress_bar(total=len(data.keys()),
unit=" Computational Sequence Entries",
leave=False)
failure = False
if (type(data) is not dict):
#this will cause the rest of the pipeline to crash - RuntimeError
log.error(
"%s computational sequence data is not in heirarchy format ...",
error=True)
try:
#for each video check the shapes of the intervals and features
for vid in data.keys():
#check the intervals first - if failure simply show a warning - no exit since we want to identify all the cases
if len(data[vid]["intervals"].shape) != 2:
if verbose:
log.error(
"Video <%s> in <%s> computational sequence has wrong intervals array shape. "
% (vid, root_name),
error=False)
failure = True
#check the features next
if len(data[vid]["features"].shape) < 2:
if verbose:
log.error(
"Video <%s> in <%s> computational sequence has wrong features array shape. "
% (vid, root_name),
error=False)
failure = True
#if the first dimension of intervals and features doesn't match
if data[vid]["features"].shape[0] != data[vid]["intervals"].shape[
0]:
if verbose:
log.error(
"Video <%s> in <%s> computational sequence - features and intervals have different first dimensions. "
% (vid, root_name),
error=False)
failure = True
pbar.update(1)
#some other thing has happened! - RuntimeError
except:
if verbose:
log.error(
"<%s> computational sequence data format could not be checked. "
% root_name,
error=True)
pbar.close()
pbar.close()
#failure during intervals and features check
if failure:
log.error(
"<%s> computational sequence data integrity check failed due to inconsistency in intervals and features. "
% root_name,
error=True)
else:
log.success("<%s> computational sequence data in correct format." %
root_name)
return True