def run_script(self):
from django.conf import settings
from celery.bin.celery import main as celery_main
if settings.DEBUG and 'worker' in sys.argv and '-h' not in sys.argv:
python_reloader(celery_main, sys.argv)
else:
celery_main(sys.argv)
def run_from_argv(self, argv):
os.environ.setdefault("CELERY_APP", ws_settings.CELERY_APP)
if settings.DEBUG and "-h" not in sys.argv:
if not any(x in sys.argv
for x in ("-c", "--concurrency", "-P", "--pool")):
sys.argv += ["-c", "1", "--pool", "solo"]
return run_with_reloader(celery_main)
celery_main()
def main(argv=sys.argv):
"""Run Celery with application environment."""
settings = settings_from_env()
setup_logger(settings)
config = make_config(settings)
config.make_wsgi_app()
celery_main(argv)
def run_script(self):
from django.conf import settings
from celery.bin.celery import main as celery_main
if settings.DEBUG and "worker" in sys.argv and "-h" not in sys.argv:
python_reloader(celery_main, (sys.argv,), {})
else:
celery_main(sys.argv)
def runcelery(*args, **options):
#CeleryCommand().handle_argv(['worker', '--app=desktop.celery', '--concurrency=1', '--loglevel=DEBUG'])
opts = ['runcelery', 'worker', '--app=' + options['app'], '--concurrency=' + str(options['concurrency']), '--loglevel=' + options['loglevel']]
drop_privileges_if_necessary(CELERY_OPTIONS)
if conf.DEV.get():
autoreload.main(celery_main, (opts,))
else:
celery_main(opts)
LOG.error("Failed to exec '%s' with argument '%s'" % args)
sys.exit(-1)
def celery():
set_env()
from celery.bin.celery import main as celery_main
from django.conf import settings
parser = ArgumentParser(usage="%(prog)s subcommand [options] [args]", add_help=False)
parser.add_argument('-A', '--app', action='store', default='djangofloor', help=settings.BIND_ADDRESS_HELP)
options, extra_args = parser.parse_known_args()
sys.argv[1:] = extra_args
__set_default_option(options, 'app')
celery_main(sys.argv)
def celery():
set_env()
from celery.bin.celery import main as celery_main
parser = ArgumentParser(usage="%(prog)s subcommand [options] [args]",
add_help=False)
parser.add_argument('-A', '--app', action='store', default='easydjango')
options, extra_args = parser.parse_known_args()
sys.argv[1:] = extra_args
__set_default_option(options, 'app')
celery_main(sys.argv)
def run_script(self):
from django.conf import settings
from celery.bin.celery import main as celery_main
if settings.DEBUG and "worker" in sys.argv and "-h" not in sys.argv:
if callable(run_with_reloader):
run_with_reloader(celery_main, sys.argv)
elif callable(python_reloader):
python_reloader(celery_main, (sys.argv, ), {})
else:
celery_main(sys.argv)
def main(argv=sys.argv):
# these must be done before importing celery.
talisker.initialise()
os.environ['CELERYD_REDIRECT_STDOUTS'] = 'False'
# techincally we don't need this, as we disable celery's logging
# altogether, but it doesn't hurt
os.environ['CELERYD_HIJACK_ROOT_LOGGER'] = 'False'
ensure_extra_versions_supported('celery')
from celery.bin.celery import main as celery_main
enable_signals()
celery_main(argv)
def runcelery(queue=None, threads=3):
from celery.bin.celery import main as celery_main
#import any modules containing celery tasks or classes that have
#not already been imported
from app import celery, app
from app.jobs import celery_tasks
celery_args = ['celery', 'worker']
if queue:
celery_args.append('-Q%s' % queue)
celery_args.append('-n {}'.format(queue))
celery_args.append('--concurrency=%s' % str(threads))
celery_main(celery_args)
def runcelery(*args, **options):
#CeleryCommand().handle_argv(['worker', '--app=desktop.celery', '--concurrency=1', '--loglevel=DEBUG'])
opts = [
'runcelery', 'worker', '--app=' + options['app'],
'--concurrency=' + str(options['concurrency']),
'--loglevel=' + options['loglevel']
]
drop_privileges_if_necessary(CELERY_OPTIONS)
if conf.DEV.get():
autoreload.main(celery_main, (opts, ))
else:
celery_main(opts)
LOG.error("Failed to exec '%s' with argument '%s'" % args)
sys.exit(-1)
def celerydev():
setup_logging('celerydev')
app = create_app(parse_options(), no_sql=True)
Logging._setup = True # Disable Celery from setting up logging, already done in setup_logging().
celery_args = ['celery', 'worker', '-B', '-s', '/tmp/celery.db', '--concurrency=5']
with app.app_context():
return celery_main(celery_args)
def celerybeat():
setup_logging('celerybeat')
app = create_app(parse_options(), no_sql=True)
Logging._setup = True
celery_args = ['celery', 'beat', '-C', '--pidfile', OPTIONS['--pid'], '-s', OPTIONS['--schedule']]
with app.app_context():
return celery_main(celery_args)
def celeryworker():
setup_logging('celeryworker{}'.format(OPTIONS['--name']))
app = create_app(parse_options(), no_sql=True)
Logging._setup = True
celery_args = ['celery', 'worker', '-n', OPTIONS['--name'], '-C', '--autoscale=10,1', '--without-gossip']
with app.app_context():
return celery_main(celery_args)
def purge_tasks():
with manager.app.app_context():
celery_args = [
'celery', '-A', 'croplands_api', 'purge',
'--broker=' + manager.app.config['CELERY_BROKER_URL']
]
return celery_main(celery_args)
def worker(ctx):
"""
Run the celery worker service
"""
app = factory(ctx.obj['CONFIG'])
celery_args = ['celery', 'worker']
with app.app_context():
return celery_main(celery_args)
def tearDown(self):
"""Tear down the test and remove local state."""
try:
celery_main([
'celery', '-A', 'receiver_test', 'control', '-b',
'redis://127.0.0.1:6379/10', 'shutdown'
])
except SystemExit:
pass
self.celery_thread.join()
try:
celery_main([
'celery', '-A', 'receiver_test', '-b',
'redis://127.0.0.1:6379/10', '--force', 'purge'
])
except SystemExit:
pass
def runcelery(*args, **options):
# Native does not load Hue's config
# CeleryCommand().handle_argv(['worker', '--app=desktop.celery', '--concurrency=1', '--loglevel=DEBUG'])
opts = [
'runcelery', 'worker', '--app=' + options['app'],
'--concurrency=' + str(options['concurrency']),
'--loglevel=' + options['loglevel']
]
drop_privileges_if_necessary(CELERY_OPTIONS)
if conf.DEV.get():
autoreload.main(celery_main, (opts, ))
else:
celery_main(opts)
LOG.info("Stopping command '%s'" % ' '.join(opts))
sys.exit(-1)
def celerydev():
env = OPTIONS['--env'] or 'dev'
app = create_app(env, parse_options())
celery_args = [
'celery', 'worker', '-B', '-E', '-s', '/tmp/celery.db',
'--concurrency=5', '--loglevel=DEBUG'
] # nosec
with app.app_context():
return celery_main(celery_args)
def celerydev():
setup_logging('celerydev')
app = create_app(parse_options(), no_sql=True)
Logging._setup = True # Disable Celery from setting up logging, already done in setup_logging().
celery_args = [
'celery', 'worker', '-B', '-s', '/tmp/celery.db', '--concurrency=5'
]
with app.app_context():
return celery_main(celery_args)
def run_celery(request, args: List[str], add_ini_option=True):
app = getattr(request.registry, 'celery', None)
if not app:
app = get_celery(request.registry)
os.environ['CELERY_LOADER'] = 'pcelery.loader.PyramidCeleryLoader'
app.set_default()
if add_ini_option:
app.user_options['preload'].add(add_preload_arguments)
return celery_main(args, auto_envvar_prefix='CELERY')
def celeryworker():
setup_logging('celeryworker{}'.format(OPTIONS['--name']))
app = create_app(parse_options(), no_sql=True)
Logging._setup = True
celery_args = [
'celery', 'worker', '-n', OPTIONS['--name'], '-C', '--autoscale=10,1',
'--without-gossip'
]
with app.app_context():
return celery_main(celery_args)
def celerybeat():
setup_logging('celerybeat')
app = create_app(get_config('restify.config.Config'), no_sql=True)
Logging._setup = True
celery_args = [
'celery', 'beat', '-C', '--pidfile', OPTIONS['--pid'], '-s',
OPTIONS['--schedule']
]
with app.app_context():
return celery_main(celery_args)
def celerybeat():
setup_logging('celerybeat')
app = create_app(parse_options(), no_sql=True)
Logging._setup = True
celery_args = [
'celery', 'beat', '-C', '--pidfile', OPTIONS['--pid'], '-s',
OPTIONS['--schedule']
]
with app.app_context():
return celery_main(celery_args)
def celerybeat():
env = OPTIONS['--env'] or 'dev'
pidfile = OPTIONS['--pid'] or './celery.pid'
schedule = OPTIONS['--schedule'] or './celery-beat.db'
app = create_app(env, parse_options())
celery_args = [
'celery', 'beat', '-C', '--pidfile', pidfile, '-s', schedule
]
with app.app_context():
return celery_main(celery_args)
def worker(Q="croplands_api"):
from random import randint
celery_args = [
'celery', 'worker', '-l', 'warning', '-n',
str(uuid.uuid4()).replace("-", "")[0:12], '-Q', Q, '--concurrency', '3'
]
print " ".join(celery_args)
with manager.app.app_context():
return celery_main(celery_args)
def celeryworker():
env = OPTIONS['--env'] or 'dev'
cname = OPTIONS['--name'] or 123
app = create_app(env, parse_options())
celery_args = [
'celery', 'worker', '-n', cname, '-C', '--autoscale=10,1',
'--without-gossip'
]
with app.app_context():
return celery_main(celery_args)
def celerybeat():
setup_logging('celerybeat')
app = create_app(parse_options())
Logging._setup = True
celery_args = [
'celery', 'beat', '-S',
'gitlab_tools.beat.schedulers.DatabaseScheduler', '-C', '--pidfile',
OPTIONS['--pid'], '-s', OPTIONS['--schedule']
]
with app.app_context():
return celery_main(celery_args)
def tearDownClass(cls):
"""Teardown and remove all messages."""
try:
celery_main([
'celery', '-A', 'pacifica.elasticsearch.tasks', 'control',
'-b', 'redis://127.0.0.1:6379/0', 'shutdown'
])
except SystemExit:
pass
cls.celery_thread.communicate()
cls.celery_thread.wait()
try:
celery_main([
'celery', '-A', 'pacifica.elasticsearch.tasks', '-b',
'redis://127.0.0.1:6379/0', '--force', 'purge'
])
except SystemExit:
pass
for key, _value in cls.env_hash.items():
del os.environ[key]
def celerybeat():
options = parse_options()
setup_logging('celerybeat',
logging.DEBUG if options.DEBUG else logging.WARNING)
app = create_app(options)
Logging._setup = True
celery_args = [
'celery', 'beat', '-C', '--pidfile', OPTIONS['--pid'], '-s',
OPTIONS['--schedule']
]
with app.app_context():
return celery_main(celery_args)
def tearDown(self):
"""Tear down the test and remove local state."""
try:
celery_main([
'celery', '-A', 'pacifica.cartd.tasks', 'control', '-b',
'redis://127.0.0.1:6379/0', 'shutdown'
])
except SystemExit:
pass
self.celery_thread.join()
try:
celery_main([
'celery', '-A', 'pacifica.cartd.tasks', '-b',
'redis://127.0.0.1:6379/0', '--force', 'purge'
])
except SystemExit:
pass
for filename, _content in self.test_files.items():
full_path = os.path.join(self.archive_dir, filename)
if os.path.isfile(full_path):
os.unlink(full_path)
def celeryd():
celery_args = [
"celery",
"worker",
"-l",
"info",
"-E",
"-c",
"2",
"--without-gossip",
"--without-mingle",
"--without-heartbeat",
]
if os.name == "nt":
# Run "solo" in Windows
celery_args += ["-P", "solo"]
with app.app_context():
return celery_main(celery_args)
def run_from_argv(self, argv):
os.environ.setdefault("CELERY_APP", ws_settings.CELERY_APP)
sys.argv.pop(0)
celery_main()
def celeryworker(celery_args):
base_celery_args = ['celery', 'worker', '-n', 'ar_worker', '-C',
'--autoscale=10,1', '--without-gossip', '--loglevel', 'INFO']
base_celery_args.extend(celery_args)
with current_app.app_context():
return celery_main(base_celery_args)
def handle(self, *args, **options):
celery_main(argv=['celery', '-A', 'frontdesk', 'worker', '-l', 'info'])
def start(self, args):
#哎嘿，咋还有个--
import sys
print(["celery"] + args.celery_args[1:])
# sys.setrecursionlimit(200)
celery_main(["celery"] + args.celery_args[1:])
def celerydev():
app = create_app(get_config('app.config.Config'))
celery_args = ['celery', 'worker', '-B', '-s', '/tmp/celery.db', '--concurrency=5']
with app.app_context():
return celery_main(celery_args)
def run(self):
celery_args = ["celery", "worker", "-C", "--autoscale=10,1", "--without-gossip"]
with app.app_context():
return celery_main(celery_args)
def run_celery_worker():
"""Run the main solo worker."""
return celery_main([
'celery', '-A', 'receiver_test', 'worker', '--pool', 'solo',
'--quiet', '-b', 'redis://127.0.0.1:6379/10'
])
