def test_csv2elastic(self):
""" Ensure that we can properly index a CSV into Elasticsearch """
fname = os.path.join(os.path.dirname(__file__), "data/sample.csv")
user_data = {"password" : "admin", "email" : "*****@*****.**"}
dataset_desc = { "title" : "sometitle", "dataset" : open(fname, 'rb'), "source_column" : "uid", "text_column" : "text", "time_column" : "created_at", "time_pattern": "%Y-%m-%d %H:%M:%S"}
with self.client :
# login
login = self.client.post('/api/v1/sessions',data=user_data)
# create a dataset
resp = self.client.post("/api/v1/datasets", data=dataset_desc)
self.assertEquals(resp.json["index_state"], "raw")
dataset_id = str(resp.json["id"])
# add additional columns
dataset_desc.pop("dataset", None)
dataset_desc["additional_columns"] = ["deleted_last_seen,permission_denied"]
url = "/api/v1/datasets/"+dataset_id
resp = self.client.put(url, data=dataset_desc)
self.assertEquals(resp.status_code, 200)
# get dataset again
resp = self.client.get("/api/v1/datasets/"+dataset_id)
dataset = resp.json
# index dataset
resp = csv2elastic(dataset)
# make sure the index exists
self.assertEquals(resp["created"], True)
self.assertEquals(resp["_index"], dataset["index_name"])
self.assertRaises(Exception, lambda : create_index(dataset["index_name"]))
# make sure that the job have been added to the queue
self.assertEqual(len(get_queue("taf").jobs), 1)
get_worker("taf").work(True)
results = search("*", dataset["index_name"])
self.assertEquals(results["hits"]["total"],121)
#!/usr/bin/env python3
import os
from flask_rq import get_worker
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
from server import create_app
if __name__ == '__main__':
# default to dev config
env = os.getenv('OK_ENV', 'dev')
app = create_app(env)
with app.app_context():
worker = get_worker()
sentry_dsn = os.getenv('SENTRY_DSN')
if sentry_dsn:
client = Client(sentry_dsn, transport=HTTPTransport)
# disable sentry for now (causes worker CrashLoopBackOff in kubernetes)
# register_sentry(client, worker)
worker.work()
def worker():
""" Run RQ workers. """
get_worker().work()
def test_get_worker_low(self):
worker = get_worker('low')
self.assertEqual(worker.queues[0].name, 'low')
def run(self, **kwargs):
app.logger.info("Running {} with arguments {}".format(
self.__class__.__name__, kwargs))
self.__dict__.update(**kwargs) # update self's with kwargs
get_worker().work()
def run_jobs(self):
get_worker().work(burst=True)
db.session.expire_all()
def test_get_worker_default(self):
worker = get_worker()
self.assertEqual(worker.queues[0].name, 'default')
def test_ctx_job(self):
simple.ctx_delay(0)
self.assertEqual(len(get_queue().jobs), 1)
get_worker().work(True)
app.config['RQ_DEFAULT_HOST'] = '127.0.0.1'
app.config['RQ_DEFAULT_PORT'] = 6379
app.config['RQ_DEFAULT_PASSWORD'] = ''
app.config['RQ_DEFAULT_DB'] = 1
RQ(app=app)
@job
def process(i):
print i
# process.delay(3)
@job('low')
def process2(i):
print i
# process2.delay(2)
# job = get_queue().enqueue(process)
# job = get_queue('low').enqueue(process)
get_worker().work(True)
get_worker('default', 'low').work(True)
if __name__ == '__main__':
pass
def test_job_queue(self):
""" Ensure we can access job queue """
self.assertEqual(len(get_queue().jobs), 0)
get_worker("taf").work(True)
#!/usr/bin/env python3
import os
from flask_rq import get_worker
from raven import Client
from raven.transport.http import HTTPTransport
from rq.contrib.sentry import register_sentry
from server import create_app
if __name__ == '__main__':
# default to dev config
env = os.environ.get('OK_ENV', 'dev')
app = create_app('settings/{0!s}.py'.format(env))
with app.app_context():
worker = get_worker()
sentry_dsn = os.getenv('SENTRY_DSN')
if sentry_dsn:
client = Client(sentry_dsn, transport=HTTPTransport)
register_sentry(client, worker)
worker.work()
def worker():
get_worker().work()
def run(self, **kwargs):
app.logger.info("Running {} with arguments {}".format(self.__class__.__name__, kwargs))
self.__dict__.update(**kwargs) # update self's with kwargs
get_worker().work()
def worker():
""" Run RQ workers. """
get_worker().work()
def work():
get_worker().work()
def test_job_default(self):
simple.delay(0)
self.assertEqual(len(get_queue().jobs), 1)
get_worker().work(True)
def test_job_default(self):
simple.delay(0)
self.assertEqual(len(get_queue().jobs), 1)
get_worker().work(True)
def test_job_specified_queue(self):
specified.delay(3)
self.assertEqual(len(get_queue('low').jobs), 1)
get_worker('low').work(True)
def test_job_specified_queue(self):
specified.delay(3)
self.assertEqual(len(get_queue('low').jobs), 1)
get_worker('low').work(True)
def test_get_worker_low(self):
worker = get_worker('low')
self.assertEqual(worker.queues[0].name, 'low')
def test_get_worker_default(self):
worker = get_worker()
self.assertEqual(worker.queues[0].name, 'default')
def run_jobs(self):
get_worker().work(burst=True)
db.session.expire_all()
#!/usr/bin/python3
# -*- coding: utf-8 -*-
# @Created on : 18/1/5 下午2:29
# @Author : zpy
# @Software: PyCharm
import os
from flask_rq import get_worker
from server import create_app
if __name__ == '__main__':
env = os.environ.get('yspider_env', 'dev')
app = create_app(env)
with app.app_context():
get_worker().work()
