def local():
# NOTE: in theory a local environment doesn't need any roledef, because
# we use Profile.development for running processes and all fabric
# operations should be run via local(). BUT, in fact, they are run via
# run() or sudo() (sparks doesn't make any difference, all tasks are
# "remote"), that's why we populate roledefs, to benefit from all sparks
# tasks.
set_roledefs_and_parallel({
'db': ['localhost'],
'web': ['localhost'],
'lang': ['localhost'],
'beat': ['localhost'],
'shell': ['localhost'],
'flower': ['localhost'],
'worker': ['localhost'],
})
# As of sparks 2.2+, it should not be needed to set env.host_string anymore,
# but it's still comfortable to have it when lanching tasks that are not
# fully roledefs-compatible (eg. sdf.compilemessages ran alone).
env.host_string = 'localhost'
if USE_JENKINS:
env.root = JENKINS_ROOT
else:
env.root = os.path.expanduser('~/sources/1flow')
env.env_was_set = True
def local():
""" The local-development task. """
# NOTE: in theory a local environment doesn't need any roledef, because
# we use Profile.development for running processes and all fabric
# operations should be run via local(). BUT, in fact, they are run via
# run() or sudo() (sparks doesn't make any difference, all tasks are
# "remote"), that's why we populate roledefs, to benefit from all sparks
# tasks.
set_roledefs_and_parallel({
'db': ['localhost'],
'web': ['localhost'],
'lang': ['localhost'],
'beat': ['localhost'],
'shell': ['localhost'],
'flower': ['localhost'],
'worker': ['localhost'],
})
# As of sparks 2.2+, it should not be needed to set env.host_string anymore,
# but it's still comfortable to have it when lanching tasks that are not
# fully roledefs-compatible (eg. sdf.compilemessages ran alone).
env.host_string = 'localhost'
env.environment = 'development'
if USE_JENKINS:
env.root = JENKINS_ROOT
else:
env.root = os.path.expanduser('~/sources/1flow')
env.env_was_set = True
def zero(branch=None):
""" A master clone, restarted from scratch everytime to test migrations. """
set_roledefs_and_parallel({
'db': ['zero.1flow.io'],
'web': ['zero.1flow.io'],
'beat': ['zero.1flow.io'],
'shell': ['zero.1flow.io'],
'worker': ['zero.1flow.io'],
'flower': ['zero.1flow.io'],
})
if branch is None:
env.branch = 'develop'
else:
env.branch = get_current_git_branch()
env.host_string = 'zero.1flow.io'
env.sparks_options = {
'worker_concurrency': {
'__all__': 2,
}
}
# env.user is set via .ssh/config
env.env_was_set = True
def zero(branch=None):
""" A master clone, restarted from scratch everytime to test migrations. """
set_roledefs_and_parallel({
'db': ['zero.1flow.io'],
'web': ['zero.1flow.io'],
'beat': ['zero.1flow.io'],
'shell': ['zero.1flow.io'],
'worker': ['zero.1flow.io'],
'flower': ['zero.1flow.io'],
})
if branch is None:
env.branch = 'develop'
else:
env.branch = get_current_git_branch()
env.host_string = 'zero.1flow.io'
env.sparks_options = {
'worker_concurrency': {
'__all__': 2,
}
}
# env.user is set via .ssh/config
env.env_was_set = True
def preview(branch=None):
""" This is the default config, we don't need to set anything more.
To create a new test environment:
adapt .ssh/config
ssh duncan
lxc-clone
in the LXC rootfs, clean /etc/supervisor/conf.d/*
edit firewall / iptables
start LXC
"""
#
# WARNING: when adding preview machine(s),
# don't forget to add django settings files for them…
#
set_roledefs_and_parallel({
'db': ['obi.1flow.io'],
'web': ['obi.1flow.io'],
'lang': ['obi.1flow.io'],
'beat': ['obi.1flow.io'],
'shell': ['worbi.1flow.io'],
'flower': ['worbi.1flow.io'],
'worker_high': ['obi.1flow.io'],
'worker_medium': ['worbi.1flow.io'],
'worker_low': ['worbi.1flow.io'],
})
if branch is None:
env.branch = get_current_git_branch()
# implicit: else: branch will be 'develop',
# set directly from the sparks defaults.
env.sparks_options = {
'nice_arguments': {
'worker_low': '-n 5',
'worker_medium': '-n 1',
'worker_high': '-n -3',
'worker_medium': '-n 1',
},
'worker_concurrency': {
'worker_low': 4,
'worker_medium': 4,
'worker_high': 6,
}
}
# we force the user because we can login as standard user there
env.user = '******'
env.env_was_set = True
def preview(branch=None):
""" Default config. We don't need to set anything more.
To create a new test environment:
adapt .ssh/config
ssh duncan
lxc-clone
in the LXC rootfs, clean /etc/supervisor/conf.d/*
edit firewall / iptables
start LXC
"""
#
# WARNING: when adding preview machine(s),
# don't forget to add django settings files for them…
#
set_roledefs_and_parallel({
'db': ['obi.1flow.io'],
'web': ['obi.1flow.io'],
'lang': ['obi.1flow.io'],
'beat': ['obi.1flow.io'],
'shell': ['worbi.1flow.io'],
'flower': ['worbi.1flow.io'],
'worker_high': ['obi.1flow.io'],
'worker_medium': ['worbi.1flow.io'],
'worker_low': ['worbi.1flow.io'],
})
if branch is None:
env.branch = get_current_git_branch()
# implicit: else: branch will be 'develop',
# set directly from the sparks defaults.
env.sparks_options = {
'nice_arguments': {
# 'worker_low': '-n 5',
# 'worker_medium': '-n 1',
'worker_high': '-n -3',
},
'worker_concurrency': {
'worker_low': 2,
'worker_medium': 2,
'worker_high': 4,
}
}
# we force the user because we can login as standard user there
env.user = '******'
env.env_was_set = True
def production():
# we force the user because we can login as standard user there
env.user = '******'
env.environment = 'production'
set_roledefs_and_parallel({
'db': ['1flow.io'],
'web': ['1flow.io'],
'beat': ['worker-01.1flow.io', ],
'shell': ['worker-03.1flow.io', ],
'flower': ['worker-01.1flow.io', ],
'worker_high': ['worker-01.1flow.io',
'worker-02.1flow.io',
'worker-03.1flow.io',
'worker-04.1flow.io', ],
'worker_medium': ['worker-02.1flow.io',
'worker-03.1flow.io',
'worker-04.1flow.io', ],
'worker_low': ['worker-03.1flow.io',
'worker-04.1flow.io', ],
'worker_fetch': ['worker-02.1flow.io',
'worker-03.1flow.io',
'worker-04.1flow.io', ],
'worker_swarm': ['worker-03.1flow.io',
'worker-04.1flow.io', ],
'worker_clean': ['worker-04.1flow.io', ],
'worker_background': ['worker-04.1flow.io', ],
})
env.sparks_options = {
'nice_arguments': {
'worker_low': '-n 3',
'worker_fetch': '-n 5',
'worker_background': '-n 10',
'worker_swarm': '-n 2',
'worker_medium': '-n 1',
'worker_high': '-n -3',
'shell': '-n -1',
},
'ionice_arguments': {
#'shell': '-n -1',
},
'repository': {
# We need to patch this because worker-04 is an LXC on the same
# physical host than dev.1flow.net, and getting from it directly
# doesn't work because of my NAT configuration.
'__all__': '[email protected]:1flow.git',
},
'autoscale': {
'worker_swarm': '32,2',
'worker_fetch': '24,1',
'worker_background': '4,0',
'worker_high': '8,1',
# Maximum one worker to avoid hammering
# the database with huge requests.
'worker_clean': '1,0',
'__all__': '8,0',
},
'max_tasks_per_child': {
# 2014-03-10: whereas many things have improved with celery 3.1,
# we still face the problem of slowly leaking workers. Surely it
# comes from our code, but I didn't find an easy way to find out
# exactly where. Thus, we relaunch workers every now and then.
#'worker_swarm': '16',
# Fetchers can literally eat memory. RECYCLE.
'worker_fetch': '8',
# Cleaning tasks are long; worker consumes ~500Mb after first run.
'worker_clean': '1',
'__all__': '32',
},
# Time-limit is useless because there is already the socket timeout.
# And anyway, it's leaking memory in celery 3.0.x.
'worker_soft_time_limit': {
'worker_swarm': '120',
# I consider that 5 minutes is enough to convert an article to
# markdown. If it doesn't acheive the conversion in this time
# frame, there is probably a more serious problem. Note that it
# can take time because of high niceness of worker processes,
# eg. they run at low priority, and a bunch of them on only a
# few cpu cores. So we have to let them a fair amount of time.
'worker_fetch': '60',
},
}
env.env_was_set = True
def production():
""" The 1flow.io production configuration task. """
# we force the user because we can login as standard user there
env.user = '******'
env.environment = 'production'
# Switch to develop to avoid needing to release a new version
# for every little patch. 1flow.io is our new testbed. In fact,
# It's a continuous delivery platform :-D
env.branch = 'develop'
env.sparks_options = {
# 'shell_arguments': {
# 'command_post_args': "--NotebookApp.ip='*'",
# },
'worker_information': {
'worker_mongo': (
'MongoDB worker (transient)',
'high,medium,low',
),
'worker_sync': (
'Inter-node synchronization worker',
'sync,background',
),
'worker_permanent': (
'Inter-node synchronization worker',
'permanent',
),
'worker_net': (
'Network-related worker',
'swarm,refresh',
),
'worker_default': (
'Default celery queue worker',
'default,create',
),
'worker_articles': (
'Articles parsing worker',
'fetch',
),
'worker_longtasks': (
'Long tasks worker',
'check,clean',
),
},
'custom_arguments': {
'worker_articles': (
'--without-heartbeat --without-mingle --without-gossip '
'--time-limit 210'),
'__all__': '--without-heartbeat --without-mingle --without-gossip',
},
'nice_arguments': {
# Lower priority
'worker_articles': '-n 5',
'worker_mongo': '-n 3',
'worker_longtasks': '-n 1',
# Higher priority
'flower': '-n -1',
'shell': '-n -5',
# Others are default.
},
'ionice_arguments': {
'shell': '-c 2 -n 1',
},
# 'worker_pool': {
# 'worker_permanent': 'prefork',
# '__all__': 'gevent',
# },
# 'repository': {
# '__all__': '[email protected]:1flow.git',
# },
'autoscale': { # queues:
'worker_mongo': '144,2', # 'high,medium,low',
'worker_net': '27,9', # 'swarm,refresh',
# 'worker_default': '16,4', # 'default,create',
# 'worker_sync': '16,4', # 'sync,background',
'worker_articles': '12,4', # 'fetch',
'worker_longtasks': '2,1', # 'check,clean',
'__all__': '9,3',
},
'max_tasks_per_child': {
# Cleaning tasks are long; worker
# consumes ~500Mb after first run.
'worker_longtasks': '1',
# Clean often to avoid eventual memory leaking
# (instanciating processors external parsers, mainly).
'worker_articles': '32',
'__all__': '128',
},
'worker_soft_time_limit': {
'worker_net': '120',
# I consider that a few minutes is enough to fetch an article
# and convert it to markdown on a loaded system. Note that it
# can take time because of low priority of worker processes.
# So we have to let them a fair amount of time. If it doesn't
# acheive the conversion in this time frame, there is probably
# a more serious problem.
'worker_articles': '180',
# Force Twitter permanent workers to
# release locks & free resources.
'worker_permanent': '300',
# 7 days: 604800
# 4 days seems to be a good start.
# If tasks last more than that, they should
# probably be split into smaller parts.
'worker_longtasks': '345600',
# For general-purpose tasks, 10 minutes can seem very long.
# On a loaded system, this is reasonable for medium-duration
# tasks.
'__all__': '600',
},
}
set_roledefs_and_parallel({
'db': ['1flow.io', ],
'web': ['1flow.io', ],
'beat': ['worker-01.1flow.io', ],
'flower': ['worker-01.1flow.io', ],
'shell': ['worker-01.1flow.io', ],
'worker_sync': [
'worker-01.1flow.io',
],
'worker_default': [
'worker-02.1flow.io',
],
'worker_net': [
'worker-02.1flow.io',
],
'worker_articles': [
# '1flow.io',
'worker-03.1flow.io',
# '10.0.3.111',
],
'worker_longtasks': [
'worker-04.1flow.io',
],
#
# 2015-01-27: I ran:
# fab prod R:worker_mongo stop
# fab prod R:worker_mongo remove
#
# And commented the following lines.
#
# 'worker_mongo': [
# 'worker-04.1flow.io',
# ],
})
env.env_was_set = True