def test():
"""
Run tests by launching tasks
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies',
'https://www.youtube.com/watch?v=gYtz5sw98Bc')
def ci():
"""
Used in conjunction with travis for Continuous Integration testing
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
from dvaapp.models import Video
from dvaapp.tasks import extract_frames, perform_face_indexing, inpcetion_index_by_id, perform_yolo_ssd_detection_by_id
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc')
for i,v in enumerate(Video.objects.all()):
extract_frames(v.pk)
inpcetion_index_by_id(v.pk)
if i ==0: # save travis time by just running detection on first video
perform_yolo_ssd_detection_by_id(v.pk)
perform_face_indexing(v.pk)
test_backup()
def ci():
"""
Used in conjunction with travis for Continuous Integration testing
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
from dvaapp.models import Video
from dvaapp.tasks import extract_frames, perform_indexing, perform_detection
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies',
'https://www.youtube.com/watch?v=gYtz5sw98Bc')
for v in Video.objects.all():
extract_frames(v.pk)
perform_indexing(v.pk)
# perform_detection(v.pk) detection is not performed in CI since it take long time on CPU
test_backup()
def ci():
"""
Used in conjunction with travis for Continuous Integration testing
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
import base64
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video, create_query
from dvaapp.models import Video, Clusters,IndexEntries,TEvent
from django.conf import settings
from dvaapp.tasks import extract_frames, perform_face_indexing, inception_index_by_id, perform_ssd_detection_by_id,\
perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, export_video_by_id, import_video_by_id,\
inception_query_by_image, perform_clustering, assign_open_images_text_tags_by_id
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc')
for i,v in enumerate(Video.objects.all()):
extract_frames(TEvent.objects.create(video=v).pk)
inception_index_by_id(TEvent.objects.create(video=v).pk)
if i ==0: # save travis time by just running detection on first video
perform_ssd_detection_by_id(TEvent.objects.create(video=v).pk)
perform_yolo_detection_by_id(TEvent.objects.create(video=v).pk)
perform_face_indexing(v.pk)
inception_index_ssd_detection_by_id(TEvent.objects.create(video=v).pk)
assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk)
fname = export_video_by_id(TEvent.objects.create(video=v,event_type=TEvent.EXPORT).pk)
f = SimpleUploadedFile(fname, file("{}/exports/{}".format(settings.MEDIA_ROOT,fname)).read(), content_type="application/zip")
vimported = handle_uploaded_file(f, fname)
import_video_by_id(TEvent.objects.create(video=vimported).pk)
dc = Clusters()
dc.indexer_algorithm = 'inception'
dc.included_index_entries_pk = [k.pk for k in IndexEntries.objects.all().filter(algorithm=dc.indexer_algorithm)]
dc.components = 32
dc.save()
clustering_task = TEvent()
clustering_task.clustering = dc
clustering_task.event_type = TEvent.CLUSTERING
clustering_task.operation = 'perform_clustering'
clustering_task.save()
perform_clustering(clustering_task.pk)
query,dv = create_query(10,False,['inception',],[],'data:image/png;base64,'+base64.encodestring(file('tests/query.png').read()))
inception_query_by_image(query.pk)
query,dv = create_query(10,True,['inception',],[],'data:image/png;base64,'+base64.encodestring(file('tests/query.png').read()))
inception_query_by_image(query.pk)
test_backup()
def ci():
"""
Used in conjunction with travis for Continuous Integration testing
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
from dvaapp.models import Video, Clusters, IndexEntries
from django.conf import settings
from dvaapp.tasks import extract_frames, perform_face_indexing, inception_index_by_id, perform_ssd_detection_by_id,\
perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, export_video_by_id, import_video_by_id,\
perform_clustering
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies',
'https://www.youtube.com/watch?v=gYtz5sw98Bc')
for i, v in enumerate(Video.objects.all()):
extract_frames(v.pk)
inception_index_by_id(v.pk)
if i == 0: # save travis time by just running detection on first video
perform_ssd_detection_by_id(v.pk)
perform_yolo_detection_by_id(v.pk)
perform_face_indexing(v.pk)
inception_index_ssd_detection_by_id(v.pk)
fname = export_video_by_id(v.pk)
f = SimpleUploadedFile(fname,
file("{}/exports/{}".format(
settings.MEDIA_ROOT, fname)).read(),
content_type="application/zip")
vimported = handle_uploaded_file(f, fname)
import_video_by_id(vimported.pk)
dc = Clusters()
dc.indexer_algorithm = 'inception'
dc.included_index_entries_pk = [
k.pk for k in IndexEntries.objects.all().filter(
algorithm=dc.indexer_algorithm)
]
dc.components = 32
dc.save()
perform_clustering(dc.pk)
test_backup()
def process_video_list(filename):
"""
submit multiple videos from a json file
"""
import django, json
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.views import handle_youtube_video
vlist = json.load(file(filename))
for video in vlist:
handle_youtube_video(video['name'], video['url'])
def process_video_list(filename):
"""
submit multiple videos from a json file
"""
import django,json
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.views import handle_youtube_video
vlist = json.load(file(filename))
for video in vlist:
handle_youtube_video(video['name'],video['url'])
def test(ci=False):
"""
Run tests
:param ci: if True (fab test:1) tests are run on Travis this option skips creating tasks and directly calls
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
from dvaapp.models import Video
from dvaapp.tasks import extract_frames, perform_indexing, perform_detection
if ci:
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('jungle book',
'https://www.youtube.com/watch?v=C4qgAaxB_pc')
for v in Video.objects.all():
extract_frames(v.pk)
perform_indexing(v.pk)
perform_detection(v.pk)
test_backup()
else:
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('jungle book',
'https://www.youtube.com/watch?v=C4qgAaxB_pc')
def test():
"""
Run tests by launching tasks
:return:
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4")
handle_uploaded_file(f, name)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc')
def generate_vdn(fast=False):
kill()
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.core.files.uploadedfile import SimpleUploadedFile
from dvaapp.views import handle_uploaded_file, handle_youtube_video
from dvaapp import models
from dvaapp.tasks import extract_frames, perform_face_detection_indexing_by_id, inception_index_by_id, \
perform_ssd_detection_by_id, perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, \
export_video_by_id
dirname = get_coco_dirname()
local(
'wget https://www.dropbox.com/s/2dq085iu34y0hdv/coco_input.zip?dl=1 -O coco.zip'
)
local('unzip coco.zip')
with lcd(dirname):
local("zip coco_input.zip -r *.jpg")
fname = '{}/coco_input.zip'.format(dirname)
with open('{}/coco_sample_metadata.json'.format(dirname)) as datafile:
data = json.load(datafile)
f = SimpleUploadedFile("coco_input.zip",
file(fname).read(),
content_type="application/zip")
v = handle_uploaded_file(f, 'mscoco_sample_500')
extract_frames(v.pk)
video = v
models.Annotation.objects.all().filter(video=video).delete()
for frame in models.Frame.objects.all().filter(video=video):
frame_id = str(int(frame.name.split('_')[-1].split('.')[0]))
annotation = models.Annotation()
annotation.video = v
annotation.frame = frame
annotation.full_frame = True
annotation.metadata_json = json.dumps(data[frame_id]['image'])
annotation.label = 'metadata'
annotation.save()
for frame in models.Frame.objects.all().filter(video=video):
frame_id = str(int(frame.name.split('_')[-1].split('.')[0]))
for a in data[frame_id][u'annotations']:
annotation = models.Annotation()
annotation.video = v
annotation.frame = frame
annotation.metadata_json = json.dumps(a)
annotation.full_frame = False
annotation.x = a['bbox'][0]
annotation.y = a['bbox'][1]
annotation.w = a['bbox'][2]
annotation.h = a['bbox'][3]
label, _ = models.VLabel.objects.get_or_create(
video=video,
label_name='coco_instance/{}/{}'.format(
a[u'category'][u'supercategory'], a[u'category'][u'name']))
annotation.label = label.label_name
annotation.label_parent = label
annotation.save()
for a in data[frame_id][u'keypoints']:
annotation = models.Annotation()
annotation.video = v
annotation.frame = frame
annotation.metadata_json = json.dumps(a)
annotation.x = a['bbox'][0]
annotation.y = a['bbox'][1]
annotation.w = a['bbox'][2]
annotation.h = a['bbox'][3]
label, _ = models.VLabel.objects.get_or_create(
video=video,
label_name='coco_keypoints/{}/{}'.format(
a[u'category'][u'supercategory'], a[u'category'][u'name']))
annotation.label = label.label_name
annotation.label_parent = label
annotation.save()
for caption in data[frame_id][u'captions']:
annotation = models.Annotation()
annotation.video = v
annotation.frame = frame
annotation.metadata_text = caption['caption']
annotation.full_frame = True
label, _ = models.VLabel.objects.get_or_create(
video=video, label_name='coco_caption')
annotation.label = label.label_name
annotation.label_parent = label
annotation.save()
if not fast:
inception_index_by_id(v.pk)
perform_ssd_detection_by_id(v.pk)
perform_face_detection_indexing_by_id(v.pk)
inception_index_ssd_detection_by_id(v.pk)
export_video_by_id(v.pk)
v = handle_youtube_video("Zelda",
"https://www.youtube.com/watch?v=vHiTxNrbB4M")
extract_frames(v.pk)
if not fast:
inception_index_by_id(v.pk)
perform_ssd_detection_by_id(v.pk)
perform_face_detection_indexing_by_id(v.pk)
inception_index_ssd_detection_by_id(v.pk)
export_video_by_id(v.pk)
v = handle_youtube_video("Paris",
"https://www.youtube.com/watch?v=zEAqJmS6ajk")
extract_frames(v.pk)
if not fast:
inception_index_by_id(v.pk)
perform_ssd_detection_by_id(v.pk)
perform_face_detection_indexing_by_id(v.pk)
inception_index_ssd_detection_by_id(v.pk)
export_video_by_id(v.pk)
local(
'wget https://www.dropbox.com/s/g8dv5yeh9bmflec/lfw_funneled.zip?dl=1 -O lfw.zip'
)
f = SimpleUploadedFile("lfw.zip",
file("lfw.zip").read(),
content_type="application/zip")
v = handle_uploaded_file(f, 'LFW subset')
extract_frames(v.pk)
if not fast:
inception_index_by_id(v.pk)
perform_face_detection_indexing_by_id(v.pk)
export_video_by_id(v.pk)
print command
os.system(command)
elif sys.argv[1] == 'test':
for fname in glob.glob('tests/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('jungle book',
'https://www.youtube.com/watch?v=C4qgAaxB_pc')
elif sys.argv[1] == 'ci_test':
for fname in glob.glob('tests/ci/*.mp4'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="video/mp4")
handle_uploaded_file(f, name, False)
for fname in glob.glob('tests/*.zip'):
name = fname.split('/')[-1].split('.')[0]
f = SimpleUploadedFile(fname,
file(fname).read(),
content_type="application/zip")
handle_uploaded_file(f, name)
handle_youtube_video('jungle book',
'https://www.youtube.com/watch?v=C4qgAaxB_pc')
