def test_csv2elastic(self): """ Ensure that we can properly index a CSV into Elasticsearch """ fname = os.path.join(os.path.dirname(__file__), "data/sample.csv") user_data = {"password" : "admin", "email" : "*****@*****.**"} dataset_desc = { "title" : "sometitle", "dataset" : open(fname, 'rb'), "source_column" : "uid", "text_column" : "text", "time_column" : "created_at", "time_pattern": "%Y-%m-%d %H:%M:%S"} with self.client : # login login = self.client.post('/api/v1/sessions',data=user_data) # create a dataset resp = self.client.post("/api/v1/datasets", data=dataset_desc) self.assertEquals(resp.json["index_state"], "raw") dataset_id = str(resp.json["id"]) # add additional columns dataset_desc.pop("dataset", None) dataset_desc["additional_columns"] = ["deleted_last_seen,permission_denied"] url = "/api/v1/datasets/"+dataset_id resp = self.client.put(url, data=dataset_desc) self.assertEquals(resp.status_code, 200) # get dataset again resp = self.client.get("/api/v1/datasets/"+dataset_id) dataset = resp.json # index dataset resp = csv2elastic(dataset) # make sure the index exists self.assertEquals(resp["created"], True) self.assertEquals(resp["_index"], dataset["index_name"]) self.assertRaises(Exception, lambda : create_index(dataset["index_name"])) # make sure that the job have been added to the queue self.assertEqual(len(get_queue("taf").jobs), 1) get_worker("taf").work(True) results = search("*", dataset["index_name"]) self.assertEquals(results["hits"]["total"],121)
#!/usr/bin/env python3 import os from flask_rq import get_worker from raven import Client from raven.transport.http import HTTPTransport from rq.contrib.sentry import register_sentry from server import create_app if __name__ == '__main__': # default to dev config env = os.getenv('OK_ENV', 'dev') app = create_app(env) with app.app_context(): worker = get_worker() sentry_dsn = os.getenv('SENTRY_DSN') if sentry_dsn: client = Client(sentry_dsn, transport=HTTPTransport) # disable sentry for now (causes worker CrashLoopBackOff in kubernetes) # register_sentry(client, worker) worker.work()
def worker(): """ Run RQ workers. """ get_worker().work()
def test_get_worker_low(self): worker = get_worker('low') self.assertEqual(worker.queues[0].name, 'low')
def run(self, **kwargs): app.logger.info("Running {} with arguments {}".format( self.__class__.__name__, kwargs)) self.__dict__.update(**kwargs) # update self's with kwargs get_worker().work()
def run_jobs(self): get_worker().work(burst=True) db.session.expire_all()
def test_get_worker_default(self): worker = get_worker() self.assertEqual(worker.queues[0].name, 'default')
def test_ctx_job(self): simple.ctx_delay(0) self.assertEqual(len(get_queue().jobs), 1) get_worker().work(True)
app.config['RQ_DEFAULT_HOST'] = '127.0.0.1' app.config['RQ_DEFAULT_PORT'] = 6379 app.config['RQ_DEFAULT_PASSWORD'] = '' app.config['RQ_DEFAULT_DB'] = 1 RQ(app=app) @job def process(i): print i # process.delay(3) @job('low') def process2(i): print i # process2.delay(2) # job = get_queue().enqueue(process) # job = get_queue('low').enqueue(process) get_worker().work(True) get_worker('default', 'low').work(True) if __name__ == '__main__': pass
def test_job_queue(self): """ Ensure we can access job queue """ self.assertEqual(len(get_queue().jobs), 0) get_worker("taf").work(True)
#!/usr/bin/env python3 import os from flask_rq import get_worker from raven import Client from raven.transport.http import HTTPTransport from rq.contrib.sentry import register_sentry from server import create_app if __name__ == '__main__': # default to dev config env = os.environ.get('OK_ENV', 'dev') app = create_app('settings/{0!s}.py'.format(env)) with app.app_context(): worker = get_worker() sentry_dsn = os.getenv('SENTRY_DSN') if sentry_dsn: client = Client(sentry_dsn, transport=HTTPTransport) register_sentry(client, worker) worker.work()
def worker(): get_worker().work()
def run(self, **kwargs): app.logger.info("Running {} with arguments {}".format(self.__class__.__name__, kwargs)) self.__dict__.update(**kwargs) # update self's with kwargs get_worker().work()
def work(): get_worker().work()
def test_job_default(self): simple.delay(0) self.assertEqual(len(get_queue().jobs), 1) get_worker().work(True)
def test_job_specified_queue(self): specified.delay(3) self.assertEqual(len(get_queue('low').jobs), 1) get_worker('low').work(True)
#!/usr/bin/python3 # -*- coding: utf-8 -*- # @Created on : 18/1/5 下午2:29 # @Author : zpy # @Software: PyCharm import os from flask_rq import get_worker from server import create_app if __name__ == '__main__': env = os.environ.get('yspider_env', 'dev') app = create_app(env) with app.app_context(): get_worker().work()