def worker(args): env = os.environ.copy() env["AIRFLOW_HOME"] = settings.AIRFLOW_HOME # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker worker = worker.worker(app=celery_app) options = {"optimization": "fair", "O": "fair", "queues": args.queues, "concurrency": args.concurrency} if args.daemon: pid, stdout, stderr, log_file = setup_locations("worker", args.pid, args.stdout, args.stderr, args.log_file) handle = setup_logging(log_file) stdout = open(stdout, "w+") stderr = open(stderr, "w+") ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(pid, -1), files_preserve=[handle], stdout=stdout, stderr=stderr ) with ctx: sp = subprocess.Popen(["airflow", "serve_logs"], env=env) worker.run(**options) sp.kill() stdout.close() stderr.close() else: signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) sp = subprocess.Popen(["airflow", "serve_logs"], env=env) worker.run(**options) sp.kill()
def celery(): """ Run celery worker. """ from project.extensions import celery from celery.bin import worker worker = worker.worker(app=celery) worker.run()
def run(self): # pylint: disable=E0202 from flask import current_app from celery.bin import worker from async.celery_helpers import CeleryFactory celery = CeleryFactory(current_app).celery worker = worker.worker(app=celery) worker.run(loglevel=logging.INFO, state_db="async/celery_state", autoreload=True)
def start_celery_worker(): from celery import current_app from celery.bin import worker celery_app = current_app._get_current_object() worker = worker.worker(app=celery_app) options = { 'broker': app.config['CELERY_BROKER_URL'], 'loglevel': 'INFO', 'traceback': True } worker.run(**options)
def worker(args): # Worker to serve static log files through this simple flask app env = os.environ.copy() env["AIRFLOW_HOME"] = settings.AIRFLOW_HOME sp = subprocess.Popen(["airflow", "serve_logs"], env=env) # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker worker = worker.worker(app=celery_app) options = {"optimization": "fair", "O": "fair", "queues": args.queues, "concurrency": args.concurrency} worker.run(**options) sp.kill()
def worker(args): # Worker to serve static log files through this simple flask app env = os.environ.copy() env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME sp = subprocess.Popen(['airflow', 'serve_logs'], env=env) # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker worker = worker.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, } worker.run(**options) sp.kill()
def worker(args): env = os.environ.copy() env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker worker = worker.worker(app=celery_app) options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, } if not args.foreground: pid, stdout, stderr, log_file = setup_locations("worker", args.pid, args.stdout, args.stderr, args.log_file) handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(pid, -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: sp = subprocess.Popen(['airflow', 'serve_logs'], env=env) worker.run(**options) sp.kill() stdout.close() stderr.close() else: signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) sp = subprocess.Popen(['airflow', 'serve_logs'], env=env) worker.run(**options) sp.kill()
meta={'msg': 'Uploading all files to aws...'}) item = store.save(item, data) item['paid'] = 2 self.update_state(state='PROGRESS', meta={'msg': 'Saving into database...'}) g = sync(item) if g: g.join() store.redis.hset('app_record', appid, item['version_code']) else: self.update_state(state='PROGRESS', meta={'msg': 'This app has been up to date...'}) except socket.error, e: self.update_state(state='PROGRESS', meta={'msg': 'Have some error happened...'}) self.retry(exc=e) return item['appid'] if __name__ == "__main__": from celery.bin import worker worker = worker.worker(app=c) options = { 'concurrency': 4, 'loglevel': 'INFO', 'traceback': True, } worker.run(**options)
def handle(self, *args, **options): worker.run(*args, **options)
def handle(self, *args, **options): worker.check_args(args) worker.run(**options)
#/usr/bin/env python from celery import current_app from celery.bin import worker from run import app from settings import * application = current_app._get_current_object() application.config_from_object('settings') worker = worker.worker(app=application) options = { 'broker': CELERY_BROKER_URL, 'loglevel': 'INFO', 'traceback': True, } worker.run(**options)
translation=os.path.join(dali_dir, transform_basename), transformed=os.path.join(dali_dir, translated_basename), pdb1=pdb1, pdb2=pdb2) os.symlink(pdb1, os.path.join(dali_dir, "{0}.pdb".format(pdbid1))) os.symlink(pdb2, os.path.join(dali_dir, "{0}.pdb".format(pdbid2))) phi = get_electrostatics_grid(pdbid1, chain=chain1, alignment_id=dali_dir) get_vanderderwaals_grids(pdbid1, chain=chain1, alignment_id=dali_dir, box=phi, scale=1 / phi.scale) phi = get_electrostatics_grid(pdbid2, chain=chain2, alignment_id=dali_dir) get_vanderderwaals_grids(pdbid2, chain=chain2, alignment_id=dali_dir, box=phi, scale=1 / phi.scale) with open(progress_file, 'w') as f: print("done", file=f) print("Finished.") if __name__ == '__main__': app = current_app._get_current_object() worker = worker.worker(app=app) worker.run(**queue.conf)
def worker(args): """Starts Airflow Celery worker""" env = os.environ.copy() env['AIRFLOW_HOME'] = settings.AIRFLOW_HOME if not settings.validate_session(): print("Worker exiting... database connection precheck failed! ") sys.exit(1) # Celery worker from airflow.executors.celery_executor import app as celery_app from celery.bin import worker # pylint: disable=redefined-outer-name autoscale = args.autoscale if autoscale is None and conf.has_option("celery", "worker_autoscale"): autoscale = conf.get("celery", "worker_autoscale") worker = worker.worker(app=celery_app) # pylint: disable=redefined-outer-name options = { 'optimization': 'fair', 'O': 'fair', 'queues': args.queues, 'concurrency': args.concurrency, 'autoscale': autoscale, 'hostname': args.celery_hostname, 'loglevel': conf.get('core', 'LOGGING_LEVEL'), } if conf.has_option("celery", "pool"): options["pool"] = conf.get("celery", "pool") if args.daemon: pid, stdout, stderr, log_file = setup_locations( "worker", args.pid, args.stdout, args.stderr, args.log_file) handle = setup_logging(log_file) stdout = open(stdout, 'w+') stderr = open(stderr, 'w+') ctx = daemon.DaemonContext( pidfile=TimeoutPIDLockFile(pid, -1), files_preserve=[handle], stdout=stdout, stderr=stderr, ) with ctx: sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill() stdout.close() stderr.close() else: signal.signal(signal.SIGINT, sigint_handler) signal.signal(signal.SIGTERM, sigint_handler) sub_proc = subprocess.Popen(['airflow', 'serve_logs'], env=env, close_fds=True) worker.run(**options) sub_proc.kill()
def handle(self, *args, **options): worker.run()