def test_coco(): import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from dvaui.view_shared import handle_uploaded_file from dvaapp.tasks import perform_import, perform_region_import, perform_dataset_extraction from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.models import TEvent for fname in glob.glob('tests/ci/coco*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") v = handle_uploaded_file(f, name) dt = TEvent.objects.get(video=v, operation='perform_import') perform_import(dt.pk) dt = TEvent(video=v, operation='perform_dataset_extraction') dt.save() perform_dataset_extraction(dt.pk) shutil.copy("tests/ci/coco_regions/coco_ci_regions.json", "dva/media/ingest/coco_ci_regions.json") args = {"path": "/ingest/coco_ci_regions.json"} dt = TEvent(video=v, operation='perform_region_import', arguments=args) dt.save() perform_region_import(dt.pk)
def create_dataset(): name = "coco_train" fname = "coco_train.zip" f = SimpleUploadedFile(fname, "", content_type="application/zip") v = handle_uploaded_file(f, name) outpath = "/root/DVA/dva/media/{}/video/{}.zip".format(v.pk, v.pk) os.system('rm {}'.format(outpath)) command = 'aws s3api get-object --request-payer "requester" --bucket visualdatanetwork --key coco/train2014.zip {}'.format(outpath) print command os.system(command) perform_dataset_extraction(TEvent.objects.create(video=v).pk)
def create_yolo_test_data(): import shutil import numpy as np import os from PIL import Image setup_django() from dvaapp.view_shared import handle_uploaded_file from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.models import Region,TEvent,Frame,Label,RegionLabel from dvaapp.tasks import perform_dataset_extraction,perform_export try: shutil.rmtree('/Users/aub3/tests/yolo_test') except: pass try: os.mkdir('/Users/aub3/tests/yolo_test') except: pass data = np.load('shared/underwater_data.npz') json_test = {} json_test['anchors'] = [(0.57273, 0.677385), (1.87446, 2.06253), (3.33843, 5.47434), (7.88282, 3.52778), (9.77052, 9.16828)] id_2_boxes = {} class_names = { 0:"red_buoy", 1:"green_buoy", 2:"yellow_buoy", 3:"path_marker", 4:"start_gate", 5:"channel" } labels = {k: Label.objects.create(name=v, set="test") for k, v in class_names} for i,image in enumerate(data['images'][:500]): path = "/Users/aub3/tests/yolo_test/{}.jpg".format(i) Image.fromarray(image).save(path) id_2_boxes[path.split('/')[-1]] = data['boxes'][i].tolist() local('zip /Users/aub3/tests/yolo_test.zip -r /Users/aub3/tests/yolo_test/* ') fname = "/Users/aub3/tests/yolo_test.zip" name = "yolo_test" f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") dv = handle_uploaded_file(f, name) perform_dataset_extraction(TEvent.objects.create(video=dv).pk) for df in Frame.objects.filter(video=dv): for box in id_2_boxes[df.name]: r = Region() r.video = dv r.frame = df c , top_x, top_y, bottom_x, bottom_y = box r.object_name = class_names[c] r.region_type = Region.ANNOTATION r.x = top_x r.y = top_y r.w = bottom_x - top_x r.h = bottom_y - top_y r.save() l = RegionLabel() l.frame = df l.video = dv l.label = labels[c] l.region = r l.save() perform_export(TEvent.objects.create(video=dv,arguments={'destination':'FILE'}).pk) try: shutil.rmtree('/Users/aub3/tests/yolo_test') except: pass
def process_visual_genome(): setup_django() import os, shutil, gzip, json from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.view_shared import handle_uploaded_file from dvaapp import models from dvaapp.models import TEvent from dvaapp.tasks import perform_dataset_extraction, export_video from collections import defaultdict os.system('aws s3api get-object --request-payer "requester" --bucket visualdatanetwork --key visual_genome_objects.txt.gz /root/DVA/visual_genome_objects.txt.gz') data = defaultdict(list) with gzip.open('/root/DVA/visual_genome_objects.txt.gz') as metadata: for line in metadata: entries = line.strip().split('\t') data[entries[1]].append({ 'x': int(entries[2]), 'y': int(entries[3]), 'w': int(entries[4]), 'h': int(entries[5]), 'object_id': entries[0], 'object_name': entries[6], 'text': ' '.join(entries[6:]), }) name = "visual_genome" fname = "visual_genome.zip" f = SimpleUploadedFile(fname, "", content_type="application/zip") v = handle_uploaded_file(f, name) outpath = "/root/DVA/dva/media/{}/video/{}.zip".format(v.pk, v.pk) os.system('rm {}'.format(outpath)) os.system( 'aws s3api get-object --request-payer "requester" --bucket visualdatanetwork --key visual_genome.zip {}'.format( outpath)) perform_dataset_extraction(TEvent.objects.create(video=v).pk) video = v models.Region.objects.all().filter(video=video).delete() buffer = [] batch_count = 0 for frame in models.Frame.objects.all().filter(video=video): frame_id = str(int(frame.name.split('/')[-1].split('.')[0])) for o in data[frame_id]: annotation = models.Region() annotation.region_type = models.Region.ANNOTATION annotation.video = v annotation.frame = frame annotation.x = o['x'] annotation.y = o['y'] annotation.h = o['h'] annotation.w = o['w'] annotation.object_name = o['object_name'] annotation.metadata = o annotation.text = o['text'] buffer.append(annotation) if len(buffer) == 1000: try: models.Region.objects.bulk_create(buffer) batch_count += 1 print "saved {}".format(batch_count) except: print "encountered an error doing one by one" for k in buffer: try: k.save() except: print "skipping" print k.object_name buffer = [] try: models.Region.objects.bulk_create(buffer) print "saved {}".format(batch_count) except: print "encountered an error doing one by one" for k in buffer: try: k.save() except: print "skipping" print k.object_name print "exporting" export_video(TEvent.objects.create(video=v).pk)
def ci(): """ Perform Continuous Integration testing using Travis """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() import base64 from django.core.files.uploadedfile import SimpleUploadedFile from dvaui.view_shared import handle_uploaded_file, pull_vdn_list \ , import_vdn_dataset_url from dvaapp.models import Video, TEvent, VDNServer, DVAPQL, Retriever, DeepModel from django.conf import settings from dvaapp.processing import DVAPQLProcess from dvaapp.tasks import perform_dataset_extraction, perform_indexing, perform_export, perform_import, \ perform_retriever_creation, perform_detection, \ perform_video_segmentation, perform_transformation for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) if sys.platform != 'darwin': for fname in glob.glob('tests/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) for i, v in enumerate(Video.objects.all()): if v.dataset: arguments = {'sync': True} perform_dataset_extraction( TEvent.objects.create(video=v, arguments=arguments).pk) else: arguments = {'sync': True} perform_video_segmentation( TEvent.objects.create(video=v, arguments=arguments).pk) arguments = {'index': 'inception', 'target': 'frames'} perform_indexing( TEvent.objects.create(video=v, arguments=arguments).pk) if i == 0: # save travis time by just running detection on first video # face_mtcnn arguments = {'detector': 'face'} dt = TEvent.objects.create(video=v, arguments=arguments) perform_detection(dt.pk) arguments = { 'filters': { 'event_id': dt.pk }, } perform_transformation( TEvent.objects.create(video=v, arguments=arguments).pk) # coco_mobilenet arguments = {'detector': 'coco'} dt = TEvent.objects.create(video=v, arguments=arguments) perform_detection(dt.pk) arguments = { 'filters': { 'event_id': dt.pk }, } perform_transformation( TEvent.objects.create(video=v, arguments=arguments).pk) # inception on crops from detector arguments = { 'index': 'inception', 'target': 'regions', 'filters': { 'event_id': dt.pk, 'w__gte': 50, 'h__gte': 50 } } perform_indexing( TEvent.objects.create(video=v, arguments=arguments).pk) # assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk) temp = TEvent.objects.create(video=v, arguments={'destination': "FILE"}) perform_export(temp.pk) temp.refresh_from_db() fname = temp.arguments['file_name'] f = SimpleUploadedFile(fname, file("{}/exports/{}".format( settings.MEDIA_ROOT, fname)).read(), content_type="application/zip") vimported = handle_uploaded_file(f, fname) perform_import( TEvent.objects.create(video=vimported, arguments={ "source": "LOCAL" }).pk) dc = Retriever() args = {} args['components'] = 32 args['m'] = 8 args['v'] = 8 args['sub'] = 64 dc.algorithm = Retriever.LOPQ dc.source_filters = { 'indexer_shasum': DeepModel.objects.get(name="inception", model_type=DeepModel.INDEXER).shasum } dc.arguments = args dc.save() clustering_task = TEvent() clustering_task.arguments = {'retriever_pk': dc.pk} clustering_task.operation = 'perform_retriever_creation' clustering_task.save() perform_retriever_creation(clustering_task.pk) query_dict = { 'process_type': DVAPQL.QUERY, 'image_data_b64': base64.encodestring(file('tests/query.png').read()), 'tasks': [{ 'operation': 'perform_indexing', 'arguments': { 'index': 'inception', 'target': 'query', 'next_tasks': [{ 'operation': 'perform_retrieval', 'arguments': { 'count': 20, 'retriever_pk': Retriever.objects.get(name='inception').pk } }] } }] } launch_workers_and_scheduler_from_environment() qp = DVAPQLProcess() qp.create_from_json(query_dict) qp.launch() qp.wait() server, datasets, detectors = pull_vdn_list(1) for k in datasets: if k['name'] == 'MSCOCO_Sample_500': print 'FOUND MSCOCO SAMPLE' import_vdn_dataset_url(VDNServer.objects.get(pk=1), k['url'], None, k)
if __name__ == '__main__': for fname in glob.glob('data/citest*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name) if settings.DEBUG: for fname in glob.glob('data/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) for i, v in enumerate(Video.objects.all()): perform_import(TEvent.objects.get(video=v, operation='perform_import').pk) if v.dataset: arguments = {'sync': True} perform_dataset_extraction(TEvent.objects.create(video=v, arguments=arguments).pk) else: arguments = {'sync': True} perform_video_segmentation(TEvent.objects.create(video=v, arguments=arguments).pk) arguments = {'index': 'inception', 'target': 'frames'} perform_indexing(TEvent.objects.create(video=v, arguments=arguments).pk) if i == 1: # save travis time by just running detection on first video # face_mtcnn arguments = {'detector': 'face'} dt = TEvent.objects.create(video=v, arguments=arguments) perform_detection(dt.pk) print "done perform_detection" arguments = {'filters': {'event_id': dt.pk}, } perform_transformation(TEvent.objects.create(video=v, arguments=arguments).pk) print "done perform_transformation" # coco_mobilenet
def ci(): """ Perform Continuous Integration testing using Travis """ import django sys.path.append(os.path.dirname(__file__)) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() import base64 from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.views import handle_uploaded_file, pull_vdn_list \ , import_vdn_dataset_url from dvaapp.models import Video, IndexEntries, TEvent, VDNServer, DVAPQL, Retriever from django.conf import settings from dvaapp.operations.processing import DVAPQLProcess from dvaapp.tasks import perform_dataset_extraction, perform_indexing, perform_export, perform_import, \ perform_retriever_creation, perform_detection, \ perform_video_segmentation, perform_transformation for fname in glob.glob('tests/ci/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) if sys.platform != 'darwin': for fname in glob.glob('tests/*.mp4'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="video/mp4") handle_uploaded_file(f, name, False) for fname in glob.glob('tests/*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") handle_uploaded_file(f, name) for i, v in enumerate(Video.objects.all()): if v.dataset: arguments = {'sync': True} perform_dataset_extraction( TEvent.objects.create(video=v, arguments=arguments).pk) else: arguments = {'sync': True} perform_video_segmentation( TEvent.objects.create(video=v, arguments=arguments).pk) arguments = {'index': 'inception', 'target': 'frames'} perform_indexing( TEvent.objects.create(video=v, arguments=arguments).pk) if i == 0: # save travis time by just running detection on first video # face_mtcnn arguments = {'detector': 'face'} dt = TEvent.objects.create(video=v, arguments=arguments) perform_detection(dt.pk) arguments = { 'filters': { 'event_id': dt.pk }, } perform_transformation( TEvent.objects.create(video=v, arguments=arguments).pk) # coco_mobilenet arguments = {'detector': 'coco'} dt = TEvent.objects.create(video=v, arguments=arguments) perform_detection(dt.pk) arguments = { 'filters': { 'event_id': dt.pk }, } perform_transformation( TEvent.objects.create(video=v, arguments=arguments).pk) # inception on crops from detector arguments = { 'index': 'inception', 'target': 'regions', 'filters': { 'event_id': dt.pk, 'w__gte': 50, 'h__gte': 50 } } perform_indexing( TEvent.objects.create(video=v, arguments=arguments).pk) # assign_open_images_text_tags_by_id(TEvent.objects.create(video=v).pk) temp = TEvent.objects.create(video=v, arguments={'destination': "FILE"}) perform_export(temp.pk) temp.refresh_from_db() fname = temp.arguments['file_name'] f = SimpleUploadedFile(fname, file("{}/exports/{}".format( settings.MEDIA_ROOT, fname)).read(), content_type="application/zip") vimported = handle_uploaded_file(f, fname) perform_import( TEvent.objects.create(video=vimported, arguments={ "source": "LOCAL" }).pk) # dc = Retriever() # args = {} # # dc.included_index_entries_pk = [k.pk for k in IndexEntries.objects.all().filter(algorithm=dc.indexer_algorithm)] # dc.components = 32 # dc.save() # clustering_task = TEvent() # clustering_task.arguments = {'clusters_id': dc.pk} # clustering_task.operation = 'perform_clustering' # clustering_task.save() # perform_clustering(clustering_task.pk) # query_dict = { # 'process_type': DVAPQL.QUERY, # 'image_data_b64': base64.encodestring(file('tests/query.png').read()), # 'indexer_queries': [ # { # 'algorithm': 'inception', # 'count': 10, # 'approximate': False # } # ] # } # qp = DVAPQLProcess() # qp.create_from_json(query_dict) # # execute_index_subquery(qp.indexer_queries[0].pk) # query_dict = { # 'process_type': DVAPQL.QUERY, # 'image_data_b64': base64.encodestring(file('tests/query.png').read()), # 'indexer_queries': [ # { # 'algorithm': 'inception', # 'count': 10, # 'approximate': True # } # ] # } # qp = DVAPQLProcess() # qp.create_from_json(query_dict) # # execute_index_subquery(qp.indexer_queries[0].pk) server, datasets, detectors = pull_vdn_list(1) for k in datasets: if k['name'] == 'MSCOCO_Sample_500': print 'FOUND MSCOCO SAMPLE' import_vdn_dataset_url(VDNServer.objects.get(pk=1), k['url'], None, k) test_backup()
#!/usr/bin/env python import django, os, sys, glob, shutil sys.path.append('../server/') os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings") django.setup() from dvaui.view_shared import handle_uploaded_file from dvaapp.tasks import perform_import, perform_region_import, perform_dataset_extraction from django.core.files.uploadedfile import SimpleUploadedFile from dvaapp.models import TEvent from django.conf import settings if __name__ == '__main__': for fname in glob.glob('ci/coco*.zip'): name = fname.split('/')[-1].split('.')[0] f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") v = handle_uploaded_file(f, name) dt = TEvent.objects.get(video=v, operation='perform_import') perform_import(dt.pk) dt = TEvent(video=v, operation='perform_dataset_extraction', arguments={}) dt.save() perform_dataset_extraction(dt.pk) shutil.copy("ci/coco_regions/coco_ci_regions.json", "{}/ingest/coco_ci_regions.json".format(settings.MEDIA_ROOT)) args = {"path": "/ingest/coco_ci_regions.json"} dt = TEvent(video=v, operation='perform_region_import', arguments=args) dt.save() perform_region_import(dt.pk)
2: "yellow_buoy", 3: "path_marker", 4: "start_gate", 5: "channel" } labels = {k: Label.objects.create(name=v, set="test") for k, v in class_names} for i, image in enumerate(data['images'][:500]): path = "/Users/aub3/tests/yolo_test/{}.jpg".format(i) Image.fromarray(image).save(path) id_2_boxes[path.split('/')[-1]] = data['boxes'][i].tolist() local('zip /Users/aub3/tests/yolo_test.zip -r /Users/aub3/tests/yolo_test/* ') fname = "/Users/aub3/tests/yolo_test.zip" name = "yolo_test" f = SimpleUploadedFile(fname, file(fname).read(), content_type="application/zip") dv = handle_uploaded_file(f, name) perform_dataset_extraction(TEvent.objects.create(video=dv).pk) for df in Frame.objects.filter(video=dv): for box in id_2_boxes[df.name]: r = Region() r.video = dv r.frame = df c, top_x, top_y, bottom_x, bottom_y = box r.object_name = class_names[c] r.region_type = Region.ANNOTATION r.x = top_x r.y = top_y r.w = bottom_x - top_x r.h = bottom_y - top_y r.save() l = RegionLabel() l.frame = df