def runFullPutMany(src_list, cleanUp=False):
if cleanUp:
if os.path.exists('./videos_full'):
shutil.rmtree('./videos_full')
manager = FullStorageManager(None, CropSplitter(), 'videos_full')
now = timer()
targets = [os.path.basename(src) for src in src_list]
logs = manager.put_many(src_list,
targets,
log=True,
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'batch_size': 30,
'num_processes': os.cpu_count()
})
put_time = timer() - now
logrecord('full', ({
'file': src_list
}), 'put', str({'elapsed': put_time}), 's')
for i, log in enumerate(logs):
logrecord('fullMany', i, 'put', str({'elapsed': log}), 's')
def runFullSequential(src, cleanUp=False):
if cleanUp:
if os.path.exists('./videos_full'):
shutil.rmtree('./videos_full')
manager = FullStorageManager(None, CropSplitter(), 'videos_full')
now = timer()
manager.put(src,
os.path.basename(src),
parallel=False,
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'batch_size': 30,
'num_processes': os.cpu_count()
})
put_time = timer() - now
logrecord('full', ({'file': src}), 'put', str({'elapsed': put_time}), 's')
clips = manager.get(os.path.basename(src), Condition())
pipelines = []
for c in clips:
pipelines.append(c[KeyPoints()])
result = counts(pipelines, ['one'], stats=True)
logrecord('full', ({'file': src}), 'get', str(result), 's')
def runFullPutMany(src_list, batch_size=30, cleanUp=False):
if cleanUp:
if os.path.exists('./videos_full'):
shutil.rmtree('./videos_full')
manager = FullStorageManager(None, CropSplitter(), 'videos_full')
now = timer()
targets = [os.path.basename(src) for src in src_list]
logs = manager.put_many(src_list,
targets,
log=True,
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'batch_size': batch_size,
'num_processes': os.cpu_count()
})
put_time = timer() - now
logrecord('full', ({
'file': src_list
}), 'put', str({'elapsed': put_time}), 's')
for i, log in enumerate(logs):
logrecord('fullMany', i, 'put', str({'elapsed': log}), 's')
# Don't call get() for now
for src in src_list:
clips = manager.get(os.path.basename(src), Condition())
pipelines = []
for c in clips:
pipelines.append(c[KeyPoints()])
result = counts(pipelines, ['one'], stats=True)
logrecord('full', ({'file': src}), 'get', str(result), 's')
def test_put(src, cleanUp=False):
if cleanUp:
if os.path.exists('./videos'):
shutil.rmtree('./videos')
manager = FullStorageManager(None, CropSplitter(), 'videos')
start = time.time()
manager.put(src,
os.path.basename(src),
parallel=True,
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'batch_size': 50,
'num_processes': 6
})
print("Put time:", time.time() - start)
clips = manager.get(os.path.basename(src), Condition(label='person'))
pipelines = []
for c in clips:
pipelines.append(c[KeyPoints()])
result = counts(pipelines, ['one'], stats=True)
logrecord('full', ({'file': src}), 'get', str(result), 's')
def runFull(src, cleanUp = False):
if cleanUp:
if os.path.exists('./videos_full'):
shutil.rmtree('./videos_full')
manager = FullStorageManager(None, CropSplitter(), 'videos_full')
now = timer()
manager.put(src, os.path.basename(src), parallel = True, args={'encoding': XVID, 'size': -1, 'sample': 1.0, 'offset': 0, 'limit': -1, 'batch_size': 50, 'num_processes': os.cpu_count()})
put_time = timer() - now
logrecord('full', ({'file': src}), 'put', str({'elapsed': put_time}), 's')
def runFullPut(src, batch_size=20):
# local_folder = '/var/www/html/videos'
# ip_addr = get_local_ip()
# remote_folder = 'http://' + ip_addr + '/videos'
local_folder = '/shared/db/'
remote_folder = '/shared/db/'
manager = FullStorageManager(
CustomTagger(fixed_tagger, batch_size=batch_size),
NullSplitter(),
local_folder,
remote_folder,
dsn='dbname=header user=postgres password=deeplens host=10.0.0.4')
def put():
now = timer()
manager.put(src,
'test',
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'batch_size': batch_size,
'num_processes': 4,
'background_scale': 1
})
put_time = timer() - now
print("Put time for full:", put_time)
# print("Batch size:", batch_size, "Folder size:", get_size(local_folder))
put()
def runFullPut(src):
local_folder = '/var/www/html/videos'
ip_addr = get_local_ip()
remote_folder = 'http://' + ip_addr + '/videos'
manager = FullStorageManager(
None,
NullSplitter(),
local_folder,
remote_folder,
dsn='dbname=header user=postgres password=deeplens host=10.0.0.5')
def put():
now = timer()
manager.put(src,
os.path.basename(src),
args={
'encoding': XVID,
'size': -1,
'sample': 1.0,
'offset': 0,
'limit': -1,
'background_scale': 1
})
put_time = timer() - now
logrecord('full', ({
'file': src
}), 'put', str({'elapsed': put_time}), 's')
def get():
clips = manager.get('test',
Condition(label='foreground', custom_filter=None))
now = timer()
frame_count = 0
for c in clips:
for frame in c:
frame_count += 1
result = timer() - now
logrecord('full', ({
'file': src,
'frames': frame_count
}), 'get', str(result), 's')
put()