def check_uptodate(client=None):
"""Check wether the distant repo is up to date with the local one. If
not, how many commits behind ?
"""
max_count = 10
if not client:
return statusall()
client = fabutils.select_client_cfg(client, CFG)
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
# that's local so it conuts the unpushed commits. should be remote
git_head = check_output(["git", "rev-parse", "HEAD"]).strip()
with cd(wd):
res = run("git rev-parse HEAD")
# print("commit of {} is {}".format(wd, res))
if res == git_head:
print(termcolor.colored("- {} is up to date".format(client.name), "green"))
else:
git_last_commits = check_output(["git", "rev-list", "HEAD", "--max-count={}".format(max_count)]).split("\n")
if res in git_last_commits:
index = git_last_commits.index(res)
print(termcolor.colored("- {}", 'blue').format(client.name) +
" is " +
termcolor.colored("{}", "yellow").format(index) +
" commits behind")
else:
print(termcolor.colored("- {}", "blue").format(client.name) +
" is more than " +
termcolor.colored("{}", "red").format(max_count) +
" commits behind.")
def create(name=None):
"""Create a new client and call the install task.
- name: name of the client (and of the venv).
"""
if not name:
name = raw_input("Client name ? ")
exists = fabutils.select_client_cfg(name, quiet=True)
if exists:
print("Client {} already exists (venv {} and port {}). Abort.".format(
name, exists['venv'], exists['port']))
exit(1)
venv = name
# Get the first available port
ports = [it.port for it in CFG.clients]
ports = sorted(ports)
possible_ports = range(8000, 8000 + len(CFG.clients) + 1)
free_port = list(set(possible_ports) - set(ports))[0]
port = raw_input("Port ? [{}] ".format(free_port))
port = port or free_port
if int(port) in ports:
print("Error: this port is taken.")
exit(1)
# Write into the config file
with open(CLIENTS, "a") as f:
f.write(CLIENT_TMPL.format(name, venv, port))
install(name)
def check_online(client=None):
"""Check every instance's working.
Parallel task on same host with multiprocessing (fabric does on /different/ hosts).
"""
if not client:
sorted_clients = sorted(CFG.clients, key=lambda it: it.name)
else:
sorted_clients = [fabutils.select_client_cfg(client, CFG)]
urls = ["http://{}:{}/fr/".format(CFG.url, c.port) for c in sorted_clients]
import multiprocessing
pool = multiprocessing.Pool(8)
status = pool.map(_request_call, urls)
res = zip(status, sorted_clients)
for status, client in res:
if status != 200:
print(
termcolor.colored(
u"- {:{}} has a pb".format(client.name, COL_WIDTH), "red")
+ " on {}".format(client['port']))
else:
print(u"- {:{}} ".format(client.name, COL_WIDTH) +
termcolor.colored("ok", "green"))
def create(name=None):
"""Create a new client and call the install task.
- name: name of the client (and of the venv).
"""
if not name:
name = raw_input("Client name ? ")
exists = fabutils.select_client_cfg(name, quiet=True)
if exists:
print("Client {} already exists (venv {} and port {}). Abort.".format(name, exists['venv'], exists['port']))
exit(1)
venv = name
# Get the first available port
ports = [it.port for it in CFG.clients]
ports = sorted(ports)
possible_ports = range(8000, 8000 + len(CFG.clients) + 1)
free_port = list(set(possible_ports) - set(ports))[0]
port = raw_input("Port ? [{}] ".format(free_port))
port = port or free_port
# Write into the config file
with open(CLIENTS, "a") as f:
f.write(CLIENT_TMPL.format(name, venv, port))
install(name)
def stop(name):
"""Stop the gunicorn process found in the pid file.
"""
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
if exists(PID_FILE):
run(CMD_KILL)
run("rm {}".format(PID_FILE))
def stop(name):
"""Stop the gunicorn process found in the pid file.
"""
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
if exists(PID_FILE):
run(CMD_KILL)
run("rm {}".format(PID_FILE))
def cmd(cmd, name=None):
"""Run any command to client "name".
"""
if not name:
print("Please give a client name.")
else:
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
run(cmd)
def start(name):
"""Run gunicorn (daemon).
Read the port in PORT.txt
"""
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
gunicorn = GUNICORN.format(project_name=CFG.project_name, url=CFG.url)
run(gunicorn)
def cmd(cmd, name=None):
"""Run any command to client "name".
"""
if not name:
print("Please give a client name.")
else:
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
run(cmd)
def ssh_to(client):
client = fabutils.select_client_cfg(client, CFG)
cmd = "ssh -Y {}@{}".format(CFG.get('user'),
client.get('url', CFG.get('url')))
if CFG.get('dir') or client.get('dir'):
cmd += " -t 'cd {}; zsh --login;'".format(
os.path.join(CFG.get('home'), CFG.get('dir', client.get('dir')),
client.get('name'), CFG.project_name), )
print("todo: workon venv")
print("connecting to {}".format(cmd))
os.system(cmd)
def openclient(client=None):
"""Open the client page with a web browser.
"""
if not client:
print("Usage: openclient:<part of client name>")
client_info()
exit(0)
client = fabutils.select_client_cfg(client, CFG)
cmd = "firefox {}:{}/fr/ & 2>/dev/null".format(CFG.url, client.port)
os.system(cmd)
def openclient(client=None):
"""Open the client page with a web browser.
"""
if not client:
print("Usage: openclient:<part of client name>")
client_info()
exit(0)
client = fabutils.select_client_cfg(client, CFG)
cmd = "firefox {}:{}/fr/ & 2>/dev/null".format(CFG.url, client.port)
os.system(cmd)
def make(cmd, name=None):
"""Run any make command remotevy
"""
if name:
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
run("make {}".format(cmd))
else:
print("no client name given")
def start(name):
"""Run gunicorn (daemon).
Read the port in PORT.txt
"""
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
gunicorn = GUNICORN.format(project_name=CFG.project_name,
url=CFG.url)
run(gunicorn)
def make(cmd, name=None):
"""Run any make command remotevy
"""
if name:
client = fabutils.select_client_cfg(name)
with cd(fabutils.wd(client, CFG)):
with prefix(VENV_ACTIVATE.format(client.name)):
run("make {}".format(cmd))
else:
print("no client name given")
def client_info(name=None):
"""Show a client info (which port does he use ?). By default, show all.
Prints to stdout.
"""
if name:
client = fabutils.select_client_cfg(name)
print(client)
else:
for it in sorted(CFG.clients):
fabutils.print_client(it)
def client_info(name=None):
"""Show a client info (which port does he use ?). By default, show all.
Prints to stdout.
"""
if name:
client = fabutils.select_client_cfg(name)
print(client)
else:
for it in sorted(CFG.clients):
fabutils.print_client(it)
def file_upload(name, *files):
"""
"""
if not files:
print("Usage: file_upload:name,path-to-file")
exit(1)
client = fabutils.select_client_cfg(name)
tmp_init_data = '/tmp/{}/'.format(client.name)
if not exists(tmp_init_data):
run('mkdir -p {}'.format(tmp_init_data))
put(files[0], tmp_init_data)
def file_upload(name, *files):
"""
Upload the given file(s) to this client's directory root.
"""
if not files:
print("Usage: file_upload:name,path-to-file")
exit(1)
client = fabutils.select_client_cfg(name)
tmp_init_data = '/tmp/{}/'.format(client.name)
if not exists(tmp_init_data):
run('mkdir -p {}'.format(tmp_init_data))
put(files[0], tmp_init_data)
def ssh_to(client):
client = fabutils.select_client_cfg(client, CFG)
cmd = "ssh -Y {}@{}".format(CFG.get('user'),
client.get('url', CFG.get('url')))
if CFG.get('dir') or client.get('dir'):
cmd += " -t 'cd {}; zsh --login;'".format(
os.path.join(CFG.get('home'),
CFG.get('dir', client.get('dir')),
client.get('name'),
CFG.project_name),)
print("todo: workon venv")
print("connecting to {}".format(cmd))
os.system(cmd)
def _save_variables(name):
"""
Another def for multiprocessing. Functions can only be pickled if they are at the toplevel.
"""
if name:
client = fabutils.select_client_cfg(name)
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
ip = client.get('ip', CFG.ip)
sentry_token = CFG.get('sentry_token')
with cd(wd):
run("echo {} > PORT.txt".format(client.port))
run("echo {} > IP.txt".format(ip))
run("echo {} > sentry.txt".format(sentry_token))
else:
print("_save_variables: give a name as argument.")
def _save_variables(name):
"""
Another def for multiprocessing. Functions can only be pickled if they are at the toplevel.
"""
if name:
client = fabutils.select_client_cfg(name)
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
ip = client.get('ip', CFG.ip)
sentry_token = CFG.get('sentry_token')
with cd(wd):
run("echo {} > PORT.txt".format(client.port))
run("echo {} > IP.txt".format(ip))
run("echo {} > sentry.txt".format(sentry_token))
else:
print("_save_variables: give a name as argument.")
def bundles_deploy(name):
"""After bundle_up, we have them in /tmp/.
Now, simply copy them into our client's static/bower_components or node_modules.
"""
client = fabutils.select_client_cfg(name)
fabutils.wd(client, CFG)
if not exists(BOWER_COMPONENTS_REMOTE_DIR):
print("Error: we can't find the bower components directory in remote /tmp.")
return
client_dir = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name, "static")
if not exists(client_dir):
run("mkdir -p {}".format(client_dir))
run("cp -r {} {}".format(BOWER_COMPONENTS_REMOTE_DIR, client_dir))
def update(client):
"""Update a client.
- client: name, or part of name of a client (str). We'll get the
client's full name.
TODO: sometimes, we need to restart the server: to load new
templates, for django to load a new module like templatetags, for
translations,....
"""
client = fabutils.select_client_cfg(client, CFG)
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
with cd(wd):
save_variables(client.name)
with prefix(VENV_ACTIVATE.format(client.venv)):
run("make update")
def update(client):
"""Update a client.
- client: name, or part of name of a client (str). We'll get the
client's full name.
TODO: sometimes, we need to restart the server: to load new
templates, for django to load a new module like templatetags, for
translations,....
"""
client = fabutils.select_client_cfg(client, CFG)
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
with cd(wd):
save_variables(client.name)
with prefix(VENV_ACTIVATE.format(client.venv)):
run("make update")
def bundles_deploy(name):
"""After bundle_up, we have them in /tmp/.
Now, simply copy them into our client's static/bower_components or node_modules.
"""
client = fabutils.select_client_cfg(name)
fabutils.wd(client, CFG)
if not exists(BOWER_COMPONENTS_REMOTE_DIR):
print(
"Error: we can't find the bower components directory in remote /tmp."
)
return
client_dir = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name,
"static")
if not exists(client_dir):
run("mkdir -p {}".format(client_dir))
run("cp -r {} {}".format(BOWER_COMPONENTS_REMOTE_DIR, client_dir))
def check_uptodate(name=None):
"""Check wether the distant repo is up to date with the local one. If
not, how many commits behind ?
"""
max_count = 10
if not name:
return statusall()
client = fabutils.select_client_cfg(name, CFG)
if not client:
print("No client found with '{}'".format(name))
return 1
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
# that's local so it conuts the unpushed commits. should be remote
git_head = check_output(["git", "rev-parse", "HEAD"]).strip()
with cd(wd):
res = run("git rev-parse HEAD")
# print("commit of {} is {}".format(wd, res))
if res == git_head:
print(
termcolor.colored("- {} is up to date".format(client.name),
"green"))
else:
git_last_commits = check_output([
"git", "rev-list", "HEAD", "--max-count={}".format(max_count)
]).split("\n")
if res in git_last_commits:
index = git_last_commits.index(res)
print(
termcolor.colored("- {}", 'blue').format(client.name) +
" is " + termcolor.colored("{}", "yellow").format(index) +
" commits behind")
else:
print(
termcolor.colored("- {}", "blue").format(client.name) +
" is more than " +
termcolor.colored("{}", "red").format(max_count) +
" commits behind.")
def rebase(name=None):
"""Only run git rebase. That may be enough for light updates.
(but be careful nothing breaks !)
Rebase the given client, or all simultaneously.
xxx: check if the pip requirements, the js sources or xxx changed
and update what's needed (try not to run apt, pip and npm every
time).
"""
def do_rebase(client):
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
with cd(wd):
with prefix(VENV_ACTIVATE.format(client.venv)):
run(CMD_REBASE)
if name:
client = fabutils.select_client_cfg(name)
do_rebase(client)
check_online(name)
def rebase(name=None):
"""Only run git rebase. That may be enough for light updates.
(but be careful nothing breaks !)
Rebase the given client, or all simultaneously.
xxx: check if the pip requirements, the js sources or xxx changed
and update what's needed (try not to run apt, pip and npm every
time).
"""
def do_rebase(client):
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
with cd(wd):
with prefix(VENV_ACTIVATE.format(client.venv)):
run(CMD_REBASE)
if name:
client = fabutils.select_client_cfg(name)
do_rebase(client)
check_online(name)
def dbback(name=None):
"""Copy the db file locally (there), appendding a timestamp, and download it.
Only for users marked in production in clients.yaml.
"""
if not name:
clients = sorted(CFG.clients)
else:
clients = [fabutils.select_client_cfg(name)]
# Only for prod:
clients = filter(lambda it: it.status == "prod", clients)
print(
termcolor.colored(
"Backup for {} users in prod: {}.".format(
len(clients), map(lambda it: it.name, clients)), "yellow"))
for client in clients:
with cd(fabutils.wd(client, CFG)):
# Copy locally. Append a timestamp.
run("cp db.db{,.`date +%Y%m%d-%H%M%S`}")
# Download
db_backed = "backup/db-{}-{}.sqlite".format(
client.name,
datetime.datetime.now().strftime(DATE_FORMAT))
cmd = "rsync -av {user}@{url}:/home/{user}/{dir}/{client_name}/{project_name}/{db_name} ./{db_backed}".format(
user=CFG.user,
url=CFG.url,
dir=CFG.dir.strip("/"),
client_name=client.name,
project_name=CFG.project_name,
db_name=CFG.db_name,
db_backed=db_backed)
print("Downloading db of user {}".format(
termcolor.colored("{}".format(client.name), "blue")))
print(cmd)
os.system(cmd)
def dbback(name=None):
"""Copy the db file locally (there), appendding a timestamp, and download it.
Only for users marked in production in clients.yaml.
"""
if not name:
clients = sorted(CFG.clients)
else:
clients = [fabutils.select_client_cfg(name)]
# Only for prod:
clients = filter(lambda it: it.status == "prod", clients)
print(termcolor.colored(
"Backup for {} users in prod: {}.".format(
len(clients),
map(lambda it: it.name, clients)),
"yellow"))
for client in clients:
with cd(fabutils.wd(client, CFG)):
# Copy locally. Append a timestamp.
run("cp db.db{,.`date +%Y%m%d-%H%M%S`}")
# Download
db_backed = "backup/db-{}-{}.sqlite".format(client.name, datetime.datetime.now().strftime(DATE_FORMAT))
cmd = "rsync -av {user}@{url}:/home/{user}/{dir}/{client_name}/{project_name}/{db_name} ./{db_backed}".format(
user=CFG.user,
url=CFG.url,
dir=CFG.dir.strip("/"),
client_name=client.name,
project_name=CFG.project_name,
db_name=CFG.db_name,
db_backed=db_backed
)
print("Downloading db of user {}".format(termcolor.colored("{}".format(client.name), "blue")))
print(cmd)
os.system(cmd)
def bower_package_version(package, names=None):
"""What's the installed packages version ?
- package: bower package name
- names: list of client names (or only beginning of name, as usual).
"""
# usage = "Usage: fab bower_package_version:package,client"
#: With --json, we could parse it and get rid off the sub-deps
BOWER_PACKAGE_VERSION = './node_modules/bower/bin/bower list --offline'
if not names:
clients = CFG.clients
else:
name = names[0]
clients = [fabutils.select_client_cfg(name)]
for client in clients:
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
print(wd)
with cd(wd):
run(BOWER_PACKAGE_VERSION)
def check_online(client=None):
"""Check every instance's working.
Parallel task on same host with multiprocessing (fabric does on /different/ hosts).
"""
if not client:
sorted_clients = sorted(CFG.clients, key=lambda it: it.name)
else:
sorted_clients = [fabutils.select_client_cfg(client, CFG)]
urls = ["http://{}:{}/fr/".format(CFG.url, c.port) for c in sorted_clients]
import multiprocessing
pool = multiprocessing.Pool(8)
status = pool.map(_request_call, urls)
res = zip(status, sorted_clients)
for status, client in res:
if status != 200:
print(termcolor.colored(u"- {:{}} has a pb".format(client.name, COL_WIDTH), "red") + " on {}".format(client['port']))
else:
print(u"- {:{}} ".format(client.name, COL_WIDTH) + termcolor.colored("ok", "green"))
def bower_package_version(package, names=None):
"""What's the installed packages version ?
- package: bower package name
- names: list of client names (or only beginning of name, as usual).
"""
# usage = "Usage: fab bower_package_version:package,client"
#: With --json, we could parse it and get rid off the sub-deps
BOWER_PACKAGE_VERSION = './node_modules/bower/bin/bower list --offline'
if not names:
clients = CFG.clients
else:
name = names[0]
clients = [fabutils.select_client_cfg(name)]
for client in clients:
wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name)
print(wd)
with cd(wd):
run(BOWER_PACKAGE_VERSION)
def install(name):
"""Clone and install Abelujo into the given client directory.
Create a super user,
populate the DB with initial data, if any,
run gunicorn with the right port.
"""
client = fabutils.select_client_cfg(name)
client = addict.Dict(client)
wd = os.path.join(CFG.home, CFG.dir, client.name)
if not exists(wd):
run("mkdir {}".format(wd, wd))
# Copy bower_components (experimental)
bundles_upload()
with cd(wd):
run("test -d {} || git clone --recursive {}".format(
CFG.project_name, CFG.project_git_url))
bundles_deploy(client.name)
# save the port
save_variables(client.name)
with cd(CFG.project_name):
# - create a venv
create_venv(client.venv)
with prefix(VENV_ACTIVATE.format(client.venv)):
# - install,
run('make install')
# TODO:
# - create super user
# - populate DB with initial data if any
# The csv files may be in nested directories.
# run('make odsimport odsfile=$(find /tmp/{}/*csv)'.format(client.name))
# - run gunicorn with the right port,
start(client.name)
def install(name):
"""Clone and install Abelujo into the given client directory.
Create a super user,
populate the DB with initial data, if any,
run gunicorn with the right port.
"""
client = fabutils.select_client_cfg(name)
client = addict.Dict(client)
wd = os.path.join(CFG.home, CFG.dir, client.name)
if not exists(wd):
run("mkdir {}".format(wd, wd))
# Copy bower_components (experimental)
bundles_upload(client.name)
with cd(wd):
run("test -d {} || git clone --recursive {}".format(CFG.project_name, CFG.project_git_url))
bundles_deploy(client.name)
# save the port
save_variables(client.name)
with cd(CFG.project_name):
# - create a venv
create_venv(client.venv)
with prefix(VENV_ACTIVATE.format(client.venv)):
# - install,
run('make install')
# TODO:
# - create super user
# - populate DB with initial data if any
# The csv files may be in nested directories.
# run('make odsimport odsfile=$(find /tmp/{}/*csv)'.format(client.name))
# - run gunicorn with the right port,
start(client.name)
