def create_yolo_test_data():
import shutil
import numpy as np
import os
from PIL import Image
setup_django()
from dvaapp.view_shared import handle_uploaded_file
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.models import Region,TEvent,Frame,Label,RegionLabel
from dvaapp.tasks import perform_dataset_extraction,perform_export
try:
shutil.rmtree('/Users/aub3/tests/yolo_test')
except:
pass
try:
os.mkdir('/Users/aub3/tests/yolo_test')
except:
pass
data = np.load('shared/underwater_data.npz')
json_test = {}
json_test['anchors'] = [(0.57273, 0.677385), (1.87446, 2.06253), (3.33843, 5.47434), (7.88282, 3.52778), (9.77052, 9.16828)]
id_2_boxes = {}
class_names = {
0:"red_buoy",
1:"green_buoy",
2:"yellow_buoy",
3:"path_marker",
4:"start_gate",
5:"channel"
}
labels = {k: Label.objects.create(name=v, set="test") for k, v in class_names}
for i,image in enumerate(data['images'][:500]):
path = "/Users/aub3/tests/yolo_test/{}.jpg".format(i)
Image.fromarray(image).save(path)
id_2_boxes[path.split('/')[-1]] = data['boxes'][i].tolist()
local('zip /Users/aub3/tests/yolo_test.zip -r /Users/aub3/tests/yolo_test/* ')
fname = "/Users/aub3/tests/yolo_test.zip"
name = "yolo_test"
f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip")
dv = handle_uploaded_file(f, name)
perform_dataset_extraction(TEvent.objects.create(video=dv).pk)
for df in Frame.objects.filter(video=dv):
for box in id_2_boxes[df.name]:
r = Region()
r.video = dv
r.frame = df
c , top_x, top_y, bottom_x, bottom_y = box
r.object_name = class_names[c]
r.region_type = Region.ANNOTATION
r.x = top_x
r.y = top_y
r.w = bottom_x - top_x
r.h = bottom_y - top_y
r.save()
l = RegionLabel()
l.frame = df
l.video = dv
l.label = labels[c]
l.region = r
l.save()
perform_export(TEvent.objects.create(video=dv,arguments={'destination':'FILE'}).pk)
try:
shutil.rmtree('/Users/aub3/tests/yolo_test')
except:
pass
def ci():
"""
Perform Continuous Integration testing using Travis
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
import base64
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaui.view_shared import handle_uploaded_file, pull_vdn_list \
, import_vdn_dataset_url
from dvaapp.models import Video, TEvent, VDNServer, DVAPQL, Retriever, DeepModel
from django.conf import settings
from dvaapp.processing import DVAPQLProcess
from dvaapp.tasks import perform_dataset_extraction, perform_indexing, perform_export, perform_import, \
perform_retriever_creation, perform_detection, \
perform_video_segmentation, perform_transformation
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
if sys.platform != 'darwin':
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
for i, v in enumerate(Video.objects.all()):
if v.dataset:
arguments = {'sync': True}
perform_dataset_extraction(
TEvent.objects.create(video=v, arguments=arguments).pk)
else:
arguments = {'sync': True}
perform_video_segmentation(
TEvent.objects.create(video=v, arguments=arguments).pk)
arguments = {'index': 'inception', 'target': 'frames'}
perform_indexing(
TEvent.objects.create(video=v, arguments=arguments).pk)
if i == 0: # save travis time by just running detection on first video
# face_mtcnn
arguments = {'detector': 'face'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
arguments = {
'filters': {
'event_id': dt.pk
},
}
perform_transformation(
TEvent.objects.create(video=v, arguments=arguments).pk)
# coco_mobilenet
arguments = {'detector': 'coco'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
arguments = {
'filters': {
'event_id': dt.pk
},
}
perform_transformation(
TEvent.objects.create(video=v, arguments=arguments).pk)
# inception on crops from detector
arguments = {
'index': 'inception',
'target': 'regions',
'filters': {
'event_id': dt.pk,
'w__gte': 50,
'h__gte': 50
}
}
perform_indexing(
TEvent.objects.create(video=v, arguments=arguments).pk)
# assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk)
temp = TEvent.objects.create(video=v,
arguments={'destination': "FILE"})
perform_export(temp.pk)
temp.refresh_from_db()
fname = temp.arguments['file_name']
f = SimpleUploadedFile(fname,
file("{}/exports/{}".format(
settings.MEDIA_ROOT, fname)).read(),
content_type="application/zip")
vimported = handle_uploaded_file(f, fname)
perform_import(
TEvent.objects.create(video=vimported,
arguments={
"source": "LOCAL"
}).pk)
dc = Retriever()
args = {}
args['components'] = 32
args['m'] = 8
args['v'] = 8
args['sub'] = 64
dc.algorithm = Retriever.LOPQ
dc.source_filters = {
'indexer_shasum':
DeepModel.objects.get(name="inception",
model_type=DeepModel.INDEXER).shasum
}
dc.arguments = args
dc.save()
clustering_task = TEvent()
clustering_task.arguments = {'retriever_pk': dc.pk}
clustering_task.operation = 'perform_retriever_creation'
clustering_task.save()
perform_retriever_creation(clustering_task.pk)
query_dict = {
'process_type':
DVAPQL.QUERY,
'image_data_b64':
base64.encodestring(file('tests/query.png').read()),
'tasks': [{
'operation': 'perform_indexing',
'arguments': {
'index':
'inception',
'target':
'query',
'next_tasks': [{
'operation': 'perform_retrieval',
'arguments': {
'count': 20,
'retriever_pk':
Retriever.objects.get(name='inception').pk
}
}]
}
}]
}
launch_workers_and_scheduler_from_environment()
qp = DVAPQLProcess()
qp.create_from_json(query_dict)
qp.launch()
qp.wait()
server, datasets, detectors = pull_vdn_list(1)
for k in datasets:
if k['name'] == 'MSCOCO_Sample_500':
print 'FOUND MSCOCO SAMPLE'
import_vdn_dataset_url(VDNServer.objects.get(pk=1), k['url'], None,
k)
def ci():
"""
Perform Continuous Integration testing using Travis
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
import base64
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, pull_vdn_list \
, import_vdn_dataset_url
from dvaapp.models import Video, IndexEntries, TEvent, VDNServer, DVAPQL, Retriever
from django.conf import settings
from dvaapp.operations.processing import DVAPQLProcess
from dvaapp.tasks import perform_dataset_extraction, perform_indexing, perform_export, perform_import, \
perform_retriever_creation, perform_detection, \
perform_video_segmentation, perform_transformation
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
if sys.platform != 'darwin':
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
for i, v in enumerate(Video.objects.all()):
if v.dataset:
arguments = {'sync': True}
perform_dataset_extraction(
TEvent.objects.create(video=v, arguments=arguments).pk)
else:
arguments = {'sync': True}
perform_video_segmentation(
TEvent.objects.create(video=v, arguments=arguments).pk)
arguments = {'index': 'inception', 'target': 'frames'}
perform_indexing(
TEvent.objects.create(video=v, arguments=arguments).pk)
if i == 0: # save travis time by just running detection on first video
# face_mtcnn
arguments = {'detector': 'face'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
arguments = {
'filters': {
'event_id': dt.pk
},
}
perform_transformation(
TEvent.objects.create(video=v, arguments=arguments).pk)
# coco_mobilenet
arguments = {'detector': 'coco'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
arguments = {
'filters': {
'event_id': dt.pk
},
}
perform_transformation(
TEvent.objects.create(video=v, arguments=arguments).pk)
# inception on crops from detector
arguments = {
'index': 'inception',
'target': 'regions',
'filters': {
'event_id': dt.pk,
'w__gte': 50,
'h__gte': 50
}
}
perform_indexing(
TEvent.objects.create(video=v, arguments=arguments).pk)
# assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk)
temp = TEvent.objects.create(video=v,
arguments={'destination': "FILE"})
perform_export(temp.pk)
temp.refresh_from_db()
fname = temp.arguments['file_name']
f = SimpleUploadedFile(fname,
file("{}/exports/{}".format(
settings.MEDIA_ROOT, fname)).read(),
content_type="application/zip")
vimported = handle_uploaded_file(f, fname)
perform_import(
TEvent.objects.create(video=vimported,
arguments={
"source": "LOCAL"
}).pk)
# dc = Retriever()
# args = {}
#
# dc.included_index_entries_pk = [k.pk for k in IndexEntries.objects.all().filter(algorithm=dc.indexer_algorithm)]
# dc.components = 32
# dc.save()
# clustering_task = TEvent()
# clustering_task.arguments = {'clusters_id': dc.pk}
# clustering_task.operation = 'perform_clustering'
# clustering_task.save()
# perform_clustering(clustering_task.pk)
# query_dict = {
# 'process_type': DVAPQL.QUERY,
# 'image_data_b64': base64.encodestring(file('tests/query.png').read()),
# 'indexer_queries': [
# {
# 'algorithm': 'inception',
# 'count': 10,
# 'approximate': False
# }
# ]
# }
# qp = DVAPQLProcess()
# qp.create_from_json(query_dict)
# # execute_index_subquery(qp.indexer_queries[0].pk)
# query_dict = {
# 'process_type': DVAPQL.QUERY,
# 'image_data_b64': base64.encodestring(file('tests/query.png').read()),
# 'indexer_queries': [
# {
# 'algorithm': 'inception',
# 'count': 10,
# 'approximate': True
# }
# ]
# }
# qp = DVAPQLProcess()
# qp.create_from_json(query_dict)
# # execute_index_subquery(qp.indexer_queries[0].pk)
server, datasets, detectors = pull_vdn_list(1)
for k in datasets:
if k['name'] == 'MSCOCO_Sample_500':
print 'FOUND MSCOCO SAMPLE'
import_vdn_dataset_url(VDNServer.objects.get(pk=1), k['url'], None,
k)
test_backup()
if i == 1: # save travis time by just running detection on first video
# face_mtcnn
arguments = {'detector': 'face'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
print "done perform_detection"
arguments = {'filters': {'event_id': dt.pk}, }
perform_transformation(TEvent.objects.create(video=v, arguments=arguments).pk)
print "done perform_transformation"
# coco_mobilenet
arguments = {'detector': 'coco'}
dt = TEvent.objects.create(video=v, arguments=arguments)
perform_detection(dt.pk)
print "done perform_detection"
arguments = {'filters': {'event_id': dt.pk}, }
perform_transformation(TEvent.objects.create(video=v, arguments=arguments).pk)
print "done perform_transformation"
# inception on crops from detector
arguments = {'index': 'inception', 'target': 'regions',
'filters': {'event_id': dt.pk, 'w__gte': 50, 'h__gte': 50}}
perform_indexing(TEvent.objects.create(video=v, arguments=arguments).pk)
print "done perform_indexing"
temp = TEvent.objects.create(arguments={'video_selector':{'pk':v.pk}})
perform_export(temp.pk)
fname = Export.objects.get(event=temp).url
f = SimpleUploadedFile(fname, file(fname.replace(settings.MEDIA_URL,settings.MEDIA_ROOT)).read(),
content_type="application/zip")
print fname
vimported = handle_uploaded_file(f, fname)
perform_import(TEvent.objects.get(video=vimported, operation='perform_import').pk)
fname = "/Users/aub3/tests/yolo_test.zip"
name = "yolo_test"
f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip")
dv = handle_uploaded_file(f, name)
perform_dataset_extraction(TEvent.objects.create(video=dv).pk)
for df in Frame.objects.filter(video=dv):
for box in id_2_boxes[df.name]:
r = Region()
r.video = dv
r.frame = df
c, top_x, top_y, bottom_x, bottom_y = box
r.object_name = class_names[c]
r.region_type = Region.ANNOTATION
r.x = top_x
r.y = top_y
r.w = bottom_x - top_x
r.h = bottom_y - top_y
r.save()
l = RegionLabel()
l.frame = df
l.video = dv
l.label = labels[c]
l.region = r
l.save()
perform_export(TEvent.objects.create(video=dv, arguments={'destination': 'FILE'}).pk)
try:
shutil.rmtree('/Users/aub3/tests/yolo_test')
except:
pass
