def test(): """ Run tests by launching tasks :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video for fname in glob.glob('tests/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc')
def ci(): """ Used in conjunction with travis for Continuous Integration testing :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video from dvaapp.models import Video from dvaapp.tasks import extract_frames, perform_face_indexing, inpcetion_index_by_id, perform_yolo_ssd_detection_by_id for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc') for i,v in enumerate(Video.objects.all()): extract_frames(v.pk) inpcetion_index_by_id(v.pk) if i ==0: # save travis time by just running detection on first video perform_yolo_ssd_detection_by_id(v.pk) perform_face_indexing(v.pk) test_backup()
def ci(): """ Used in conjunction with travis for Continuous Integration testing :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video from dvaapp.models import Video from dvaapp.tasks import extract_frames, perform_indexing, perform_detection for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc') for v in Video.objects.all(): extract_frames(v.pk) perform_indexing(v.pk) # perform_detection(v.pk) detection is not performed in CI since it take long time on CPU test_backup()
def ci(): """ Used in conjunction with travis for Continuous Integration testing :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() import base64 from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video, create_query from dvaapp.models import Video, Clusters,IndexEntries,TEvent from django.conf import settings from dvaapp.tasks import extract_frames, perform_face_indexing, inception_index_by_id, perform_ssd_detection_by_id,\ perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, export_video_by_id, import_video_by_id,\ inception_query_by_image, perform_clustering, assign_open_images_text_tags_by_id for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc') for i,v in enumerate(Video.objects.all()): extract_frames(TEvent.objects.create(video=v).pk) inception_index_by_id(TEvent.objects.create(video=v).pk) if i ==0: # save travis time by just running detection on first video perform_ssd_detection_by_id(TEvent.objects.create(video=v).pk) perform_yolo_detection_by_id(TEvent.objects.create(video=v).pk) perform_face_indexing(v.pk) inception_index_ssd_detection_by_id(TEvent.objects.create(video=v).pk) assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk) fname = export_video_by_id(TEvent.objects.create(video=v,event_type=TEvent.EXPORT).pk) f = SimpleUploadedFile(fname, file("{}/exports/{}".format(settings.MEDIA_ROOT,fname)).read(), content_type="application/zip") vimported = handle_uploaded_file(f, fname) import_video_by_id(TEvent.objects.create(video=vimported).pk) dc = Clusters() dc.indexer_algorithm = 'inception' dc.included_index_entries_pk = [k.pk for k in IndexEntries.objects.all().filter(algorithm=dc.indexer_algorithm)] dc.components = 32 dc.save() clustering_task = TEvent() clustering_task.clustering = dc clustering_task.event_type = TEvent.CLUSTERING clustering_task.operation = 'perform_clustering' clustering_task.save() perform_clustering(clustering_task.pk) query,dv = create_query(10,False,['inception',],[],'data:image/png;base64,'+base64.encodestring(file('tests/query.png').read())) inception_query_by_image(query.pk) query,dv = create_query(10,True,['inception',],[],'data:image/png;base64,'+base64.encodestring(file('tests/query.png').read())) inception_query_by_image(query.pk) test_backup()
def ci(): """ Used in conjunction with travis for Continuous Integration testing :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video from dvaapp.models import Video, Clusters, IndexEntries from django.conf import settings from dvaapp.tasks import extract_frames, perform_face_indexing, inception_index_by_id, perform_ssd_detection_by_id,\ perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, export_video_by_id, import_video_by_id,\ perform_clustering for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('tomorrow never dies', 'https://www.youtube.com/watch?v=gYtz5sw98Bc') for i, v in enumerate(Video.objects.all()): extract_frames(v.pk) inception_index_by_id(v.pk) if i == 0: # save travis time by just running detection on first video perform_ssd_detection_by_id(v.pk) perform_yolo_detection_by_id(v.pk) perform_face_indexing(v.pk) inception_index_ssd_detection_by_id(v.pk) fname = export_video_by_id(v.pk) f = SimpleUploadedFile(fname, file("{}/exports/{}".format( settings.MEDIA_ROOT, fname)).read(), content_type="application/zip") vimported = handle_uploaded_file(f, fname) import_video_by_id(vimported.pk) dc = Clusters() dc.indexer_algorithm = 'inception' dc.included_index_entries_pk = [ k.pk for k in IndexEntries.objects.all().filter( algorithm=dc.indexer_algorithm) ] dc.components = 32 dc.save() perform_clustering(dc.pk) test_backup()
def process_video_list(filename): """ submit multiple videos from a json file """ import django, json sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from dvaapp.views import handle_youtube_video vlist = json.load(file(filename)) for video in vlist: handle_youtube_video(video['name'], video['url'])
def process_video_list(filename): """ submit multiple videos from a json file """ import django,json sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from dvaapp.views import handle_youtube_video vlist = json.load(file(filename)) for video in vlist: handle_youtube_video(video['name'],video['url'])
def test(ci=False): """ Run tests :param ci: if True (fab test:1) tests are run on Travis this option skips creating tasks and directly calls :return: """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video from dvaapp.models import Video from dvaapp.tasks import extract_frames, perform_indexing, perform_detection if ci: for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('jungle book', 'https://www.youtube.com/watch?v=C4qgAaxB_pc') for v in Video.objects.all(): extract_frames(v.pk) perform_indexing(v.pk) perform_detection(v.pk) test_backup() else: for fname in glob.glob('tests/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('jungle book', 'https://www.youtube.com/watch?v=C4qgAaxB_pc')
def generate_vdn(fast=False): kill() import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, handle_youtube_video from dvaapp import models from dvaapp.tasks import extract_frames, perform_face_detection_indexing_by_id, inception_index_by_id, \ perform_ssd_detection_by_id, perform_yolo_detection_by_id, inception_index_ssd_detection_by_id, \ export_video_by_id dirname = get_coco_dirname() local( 'wget https://www.dropbox.com/s/2dq085iu34y0hdv/coco_input.zip?dl=1 -O coco.zip' ) local('unzip coco.zip') with lcd(dirname): local("zip coco_input.zip -r *.jpg") fname = '{}/coco_input.zip'.format(dirname) with open('{}/coco_sample_metadata.json'.format(dirname)) as datafile: data = json.load(datafile) f = SimpleUploadedFile("coco_input.zip", file(fname).read(), content_type="application/zip") v = handle_uploaded_file(f, 'mscoco_sample_500') extract_frames(v.pk) video = v models.Annotation.objects.all().filter(video=video).delete() for frame in models.Frame.objects.all().filter(video=video): frame_id = str(int(frame.name.split('_')[-1].split('.')[0])) annotation = models.Annotation() annotation.video = v annotation.frame = frame annotation.full_frame = True annotation.metadata_json = json.dumps(data[frame_id]['image']) annotation.label = 'metadata' annotation.save() for frame in models.Frame.objects.all().filter(video=video): frame_id = str(int(frame.name.split('_')[-1].split('.')[0])) for a in data[frame_id][u'annotations']: annotation = models.Annotation() annotation.video = v annotation.frame = frame annotation.metadata_json = json.dumps(a) annotation.full_frame = False annotation.x = a['bbox'][0] annotation.y = a['bbox'][1] annotation.w = a['bbox'][2] annotation.h = a['bbox'][3] label, _ = models.VLabel.objects.get_or_create( video=video, label_name='coco_instance/{}/{}'.format( a[u'category'][u'supercategory'], a[u'category'][u'name'])) annotation.label = label.label_name annotation.label_parent = label annotation.save() for a in data[frame_id][u'keypoints']: annotation = models.Annotation() annotation.video = v annotation.frame = frame annotation.metadata_json = json.dumps(a) annotation.x = a['bbox'][0] annotation.y = a['bbox'][1] annotation.w = a['bbox'][2] annotation.h = a['bbox'][3] label, _ = models.VLabel.objects.get_or_create( video=video, label_name='coco_keypoints/{}/{}'.format( a[u'category'][u'supercategory'], a[u'category'][u'name'])) annotation.label = label.label_name annotation.label_parent = label annotation.save() for caption in data[frame_id][u'captions']: annotation = models.Annotation() annotation.video = v annotation.frame = frame annotation.metadata_text = caption['caption'] annotation.full_frame = True label, _ = models.VLabel.objects.get_or_create( video=video, label_name='coco_caption') annotation.label = label.label_name annotation.label_parent = label annotation.save() if not fast: inception_index_by_id(v.pk) perform_ssd_detection_by_id(v.pk) perform_face_detection_indexing_by_id(v.pk) inception_index_ssd_detection_by_id(v.pk) export_video_by_id(v.pk) v = handle_youtube_video("Zelda", "https://www.youtube.com/watch?v=vHiTxNrbB4M") extract_frames(v.pk) if not fast: inception_index_by_id(v.pk) perform_ssd_detection_by_id(v.pk) perform_face_detection_indexing_by_id(v.pk) inception_index_ssd_detection_by_id(v.pk) export_video_by_id(v.pk) v = handle_youtube_video("Paris", "https://www.youtube.com/watch?v=zEAqJmS6ajk") extract_frames(v.pk) if not fast: inception_index_by_id(v.pk) perform_ssd_detection_by_id(v.pk) perform_face_detection_indexing_by_id(v.pk) inception_index_ssd_detection_by_id(v.pk) export_video_by_id(v.pk) local( 'wget https://www.dropbox.com/s/g8dv5yeh9bmflec/lfw_funneled.zip?dl=1 -O lfw.zip' ) f = SimpleUploadedFile("lfw.zip", file("lfw.zip").read(), content_type="application/zip") v = handle_uploaded_file(f, 'LFW subset') extract_frames(v.pk) if not fast: inception_index_by_id(v.pk) perform_face_detection_indexing_by_id(v.pk) export_video_by_id(v.pk)
print command os.system(command) elif sys.argv[1] == 'test': for fname in glob.glob('tests/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('jungle book', 'https://www.youtube.com/watch?v=C4qgAaxB_pc') elif sys.argv[1] == 'ci_test': for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) handle_youtube_video('jungle book', 'https://www.youtube.com/watch?v=C4qgAaxB_pc')