def get_by_time(self, start, end):
"""
This expects ints and will attempt to convert strings to int,
:param start: int(1473803814)
:param end: int(1473803824)
:return:
"""
if isinstance(start, str):
start = file_name_to_epoch_time(start)
if isinstance(end, str):
end = file_name_to_epoch_time(end)
shelve_paths = list()
for root, _, potential_files in os.walk(self.label_dir):
for potential_file in potential_files:
if not potential_file.endswith('.pickle.gz'):
continue
potential_file_path = os.path.join(root, potential_file)
if end >= file_name_to_epoch_time(potential_file) >= start:
# out.append((label, potential_file_path))
shelve_paths.append(potential_file_path)
return self.unpack_shelves(shelve_paths)
def main(glob_string, family, label, data_sets, api):
"""
This function assumes that the name of the file is a string representation
of the capture time. This will not work for ANYTHING else.
:param glob_string: './data/*.h5'
:param label: 'all'
:param api: ExampleApi
:return:
"""
start_time = time.time()
file_paths = glob.glob(glob_string)
lg.debug('file_paths={0}'.format(file_paths))
for file_path in file_paths:
file_name = os.path.basename(file_path)
# Here we attempt to determine if the file_name corresponds to the insertion date
# else, we just assume that it happened now and create the timestamp accordingly.
try:
epoch_time = file_name_to_epoch_time(file_name)
except ValueError:
epoch_time = None
possible_label = file_path.split('/')[-2]
tmp_dict = {
'label_name': label or possible_label,
'label_family': family,
'file_path': file_path,
'indices': [-1],
'epoch_start_time': epoch_time,
'data_sets': data_sets,
}
api.insert_by_label(Label(tmp_dict))
# api.insert(epoch_time, family, label or possible_label, file_path, [-1])
lg.info("delta={0} n_files_upserted={1}".format(time.time() - start_time,
len(file_paths)))
return len(file_paths)
def test_str_date_time_to_epoch(self):
result = file_name_to_epoch_time('2016_09_13_16_53_24')
self.assertEqual(result, 1473803604)