def launch_queues_env():
"""
Launch workers for each queue
:param detection: use fab launch_queues:1 to lauch detector queue in addition to all others
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video,VDNServer
for k in os.environ:
if k.startswith('LAUNCH_Q_'):
if k.strip() == 'LAUNCH_Q_qextract':
queue_name = k.split('_')[-1]
local('fab startq:{},{} &'.format(queue_name,os.environ['LAUNCH_Q_qextract']))
else:
queue_name = k.split('_')[-1]
local('fab startq:{} &'.format(queue_name))
if not ('DISABLE_VDN' in os.environ):
if VDNServer.objects.count() == 0:
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
if 'TEST' in os.environ and Video.objects.count() == 0:
test()
def clean():
"""
Reset database, migrate, clear media folder, and (only on my dev machine) kill workers/clear all queues.
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.conf import settings
from dvaapp.models import VDNServer
if sys.platform == 'darwin':
for qname in set(settings.TASK_NAMES_TO_QUEUE.values()):
try:
local('rabbitmqadmin purge queue name={}'.format(qname))
except:
logging.warning("coudnt clear queue {}".format(qname))
# TODO: wait for Celery bug fix https://github.com/celery/celery/issues/3620
# local('celery amqp exchange.delete broadcast_tasks')
migrate()
local('python manage.py flush --no-input')
migrate()
local("rm -rf {}/*".format(settings.MEDIA_ROOT))
local("mkdir {}/queries".format(settings.MEDIA_ROOT))
if sys.platform == 'darwin':
local("rm logs/*.log")
try:
local("ps auxww | grep 'celery -A dva worker' | awk '{print $2}' | xargs kill -9")
except:
pass
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
def init_server():
"""
Initialize server database by adding default VDN server and DVAPQL templates
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video, VDNServer, StoredDVAPQL
if StoredDVAPQL.objects.count() == 0:
for fname in glob.glob('configs/templates/*.json'):
StoredDVAPQL.objects.create(name=fname,
process_type=StoredDVAPQL.PROCESS,
script=json.loads(file(fname).read()))
if not ('DISABLE_VDN' in os.environ):
if VDNServer.objects.count() == 0:
servers = json.loads(file('configs/vdn_servers.json').read())
for s in servers:
server = VDNServer()
server.url = s['url']
server.name = s['name']
server.last_response_datasets = s['last_response_datasets']
server.last_response_detectors = s['last_response_detectors']
server.save()
if 'TEST' in os.environ and Video.objects.count() == 0:
test()
def init_server():
"""
Initialize server database by adding default VDN server and DVAPQL templates
"""
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video, VDNServer, StoredDVAPQL
from django_celery_beat.models import PeriodicTask,IntervalSchedule
di,created = IntervalSchedule.objects.get_or_create(every=os.environ.get('REFRESH_MINUTES',3),
period=IntervalSchedule.MINUTES)
_ = PeriodicTask.objects.get_or_create(name="monitoring",task="monitor_system",
interval=di,queue='qextract')
if StoredDVAPQL.objects.count() == 0:
for fname in glob.glob('configs/templates/*.json'):
StoredDVAPQL.objects.create(name=fname,
process_type=StoredDVAPQL.PROCESS,
script=json.loads(file(fname).read()))
if not ('DISABLE_VDN' in os.environ):
if VDNServer.objects.count() == 0:
servers = json.loads(file('configs/vdn_servers.json').read())
for s in servers:
server = VDNServer()
server.url = s['url']
server.name = s['name']
server.last_response_datasets = s['last_response_datasets']
server.last_response_detectors = s['last_response_detectors']
server.save()
if 'TEST' in os.environ and Video.objects.count() == 0:
test()
def add_default_vdn_server():
"""
Add http://www.visualdata.network/ as default VDN server
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video,VDNServer
if VDNServer.objects.count() == 0:
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
def launch_queues_env():
"""
Launch workers for each queue
:param detection: use fab launch_queues:1 to lauch detector queue in addition to all others
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video, VDNServer
for k in os.environ:
if k.startswith('LAUNCH_Q_'):
if k.strip() == 'LAUNCH_Q_qextract':
queue_name = k.split('_')[-1]
local('fab startq:{},{} &'.format(
queue_name, os.environ['LAUNCH_Q_qextract']))
else:
queue_name = k.split('_')[-1]
local('fab startq:{} &'.format(queue_name))
if not ('DISABLE_VDN' in os.environ):
if VDNServer.objects.count() == 0:
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
if 'TEST' in os.environ and Video.objects.count() == 0:
test()
def clean():
"""
Reset database, migrate, clear media folder, and (only on my dev machine) kill workers/clear all queues.
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from django.conf import settings
from dvaapp.models import VDNServer
if sys.platform == 'darwin':
for qname in set(settings.TASK_NAMES_TO_QUEUE.values()):
try:
local('rabbitmqadmin purge queue name={}'.format(qname))
except:
logging.warning("coudnt clear queue {}".format(qname))
migrate()
local('python manage.py flush --no-input')
migrate()
local("rm -rf {}/*".format(settings.MEDIA_ROOT))
local("mkdir {}/queries".format(settings.MEDIA_ROOT))
if sys.platform == 'darwin':
local("rm logs/*.log")
try:
local(
"ps auxww | grep 'celery -A dva worker' | awk '{print $2}' | xargs kill -9"
)
except:
pass
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
def add_default_vdn_server():
"""
Add http://www.visualdata.network/ as default VDN server
:return:
"""
import django, os
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video, VDNServer
if VDNServer.objects.count() == 0:
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
def init_server():
import django
sys.path.append(os.path.dirname(__file__))
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dva.settings")
django.setup()
from dvaapp.models import Video,VDNServer,StoredDVAPQL
if StoredDVAPQL.objects.count() == 0:
for fname in glob.glob('dvaapp/test_scripts/*.json'):
StoredDVAPQL.objects.create(name=fname,
process_type=StoredDVAPQL.PROCESS,
script=json.loads(file(fname).read()))
if not ('DISABLE_VDN' in os.environ):
if VDNServer.objects.count() == 0:
server = VDNServer()
server.url = "http://www.visualdata.network/"
server.name = "VisualData.Network"
server.save()
if 'TEST' in os.environ and Video.objects.count() == 0:
test()