def check_uptodate(client=None): """Check wether the distant repo is up to date with the local one. If not, how many commits behind ? """ max_count = 10 if not client: return statusall() client = fabutils.select_client_cfg(client, CFG) wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) # that's local so it conuts the unpushed commits. should be remote git_head = check_output(["git", "rev-parse", "HEAD"]).strip() with cd(wd): res = run("git rev-parse HEAD") # print("commit of {} is {}".format(wd, res)) if res == git_head: print(termcolor.colored("- {} is up to date".format(client.name), "green")) else: git_last_commits = check_output(["git", "rev-list", "HEAD", "--max-count={}".format(max_count)]).split("\n") if res in git_last_commits: index = git_last_commits.index(res) print(termcolor.colored("- {}", 'blue').format(client.name) + " is " + termcolor.colored("{}", "yellow").format(index) + " commits behind") else: print(termcolor.colored("- {}", "blue").format(client.name) + " is more than " + termcolor.colored("{}", "red").format(max_count) + " commits behind.")
def create(name=None): """Create a new client and call the install task. - name: name of the client (and of the venv). """ if not name: name = raw_input("Client name ? ") exists = fabutils.select_client_cfg(name, quiet=True) if exists: print("Client {} already exists (venv {} and port {}). Abort.".format( name, exists['venv'], exists['port'])) exit(1) venv = name # Get the first available port ports = [it.port for it in CFG.clients] ports = sorted(ports) possible_ports = range(8000, 8000 + len(CFG.clients) + 1) free_port = list(set(possible_ports) - set(ports))[0] port = raw_input("Port ? [{}] ".format(free_port)) port = port or free_port if int(port) in ports: print("Error: this port is taken.") exit(1) # Write into the config file with open(CLIENTS, "a") as f: f.write(CLIENT_TMPL.format(name, venv, port)) install(name)
def check_online(client=None): """Check every instance's working. Parallel task on same host with multiprocessing (fabric does on /different/ hosts). """ if not client: sorted_clients = sorted(CFG.clients, key=lambda it: it.name) else: sorted_clients = [fabutils.select_client_cfg(client, CFG)] urls = ["http://{}:{}/fr/".format(CFG.url, c.port) for c in sorted_clients] import multiprocessing pool = multiprocessing.Pool(8) status = pool.map(_request_call, urls) res = zip(status, sorted_clients) for status, client in res: if status != 200: print( termcolor.colored( u"- {:{}} has a pb".format(client.name, COL_WIDTH), "red") + " on {}".format(client['port'])) else: print(u"- {:{}} ".format(client.name, COL_WIDTH) + termcolor.colored("ok", "green"))
def create(name=None): """Create a new client and call the install task. - name: name of the client (and of the venv). """ if not name: name = raw_input("Client name ? ") exists = fabutils.select_client_cfg(name, quiet=True) if exists: print("Client {} already exists (venv {} and port {}). Abort.".format(name, exists['venv'], exists['port'])) exit(1) venv = name # Get the first available port ports = [it.port for it in CFG.clients] ports = sorted(ports) possible_ports = range(8000, 8000 + len(CFG.clients) + 1) free_port = list(set(possible_ports) - set(ports))[0] port = raw_input("Port ? [{}] ".format(free_port)) port = port or free_port # Write into the config file with open(CLIENTS, "a") as f: f.write(CLIENT_TMPL.format(name, venv, port)) install(name)
def stop(name): """Stop the gunicorn process found in the pid file. """ client = fabutils.select_client_cfg(name) with cd(fabutils.wd(client, CFG)): if exists(PID_FILE): run(CMD_KILL) run("rm {}".format(PID_FILE))
def cmd(cmd, name=None): """Run any command to client "name". """ if not name: print("Please give a client name.") else: client = fabutils.select_client_cfg(name) with cd(fabutils.wd(client, CFG)): with prefix(VENV_ACTIVATE.format(client.name)): run(cmd)
def start(name): """Run gunicorn (daemon). Read the port in PORT.txt """ client = fabutils.select_client_cfg(name) with cd(fabutils.wd(client, CFG)): with prefix(VENV_ACTIVATE.format(client.name)): gunicorn = GUNICORN.format(project_name=CFG.project_name, url=CFG.url) run(gunicorn)
def ssh_to(client): client = fabutils.select_client_cfg(client, CFG) cmd = "ssh -Y {}@{}".format(CFG.get('user'), client.get('url', CFG.get('url'))) if CFG.get('dir') or client.get('dir'): cmd += " -t 'cd {}; zsh --login;'".format( os.path.join(CFG.get('home'), CFG.get('dir', client.get('dir')), client.get('name'), CFG.project_name), ) print("todo: workon venv") print("connecting to {}".format(cmd)) os.system(cmd)
def openclient(client=None): """Open the client page with a web browser. """ if not client: print("Usage: openclient:<part of client name>") client_info() exit(0) client = fabutils.select_client_cfg(client, CFG) cmd = "firefox {}:{}/fr/ & 2>/dev/null".format(CFG.url, client.port) os.system(cmd)
def make(cmd, name=None): """Run any make command remotevy """ if name: client = fabutils.select_client_cfg(name) with cd(fabutils.wd(client, CFG)): with prefix(VENV_ACTIVATE.format(client.name)): run("make {}".format(cmd)) else: print("no client name given")
def client_info(name=None): """Show a client info (which port does he use ?). By default, show all. Prints to stdout. """ if name: client = fabutils.select_client_cfg(name) print(client) else: for it in sorted(CFG.clients): fabutils.print_client(it)
def file_upload(name, *files): """ """ if not files: print("Usage: file_upload:name,path-to-file") exit(1) client = fabutils.select_client_cfg(name) tmp_init_data = '/tmp/{}/'.format(client.name) if not exists(tmp_init_data): run('mkdir -p {}'.format(tmp_init_data)) put(files[0], tmp_init_data)
def file_upload(name, *files): """ Upload the given file(s) to this client's directory root. """ if not files: print("Usage: file_upload:name,path-to-file") exit(1) client = fabutils.select_client_cfg(name) tmp_init_data = '/tmp/{}/'.format(client.name) if not exists(tmp_init_data): run('mkdir -p {}'.format(tmp_init_data)) put(files[0], tmp_init_data)
def ssh_to(client): client = fabutils.select_client_cfg(client, CFG) cmd = "ssh -Y {}@{}".format(CFG.get('user'), client.get('url', CFG.get('url'))) if CFG.get('dir') or client.get('dir'): cmd += " -t 'cd {}; zsh --login;'".format( os.path.join(CFG.get('home'), CFG.get('dir', client.get('dir')), client.get('name'), CFG.project_name),) print("todo: workon venv") print("connecting to {}".format(cmd)) os.system(cmd)
def _save_variables(name): """ Another def for multiprocessing. Functions can only be pickled if they are at the toplevel. """ if name: client = fabutils.select_client_cfg(name) wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) ip = client.get('ip', CFG.ip) sentry_token = CFG.get('sentry_token') with cd(wd): run("echo {} > PORT.txt".format(client.port)) run("echo {} > IP.txt".format(ip)) run("echo {} > sentry.txt".format(sentry_token)) else: print("_save_variables: give a name as argument.")
def bundles_deploy(name): """After bundle_up, we have them in /tmp/. Now, simply copy them into our client's static/bower_components or node_modules. """ client = fabutils.select_client_cfg(name) fabutils.wd(client, CFG) if not exists(BOWER_COMPONENTS_REMOTE_DIR): print("Error: we can't find the bower components directory in remote /tmp.") return client_dir = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name, "static") if not exists(client_dir): run("mkdir -p {}".format(client_dir)) run("cp -r {} {}".format(BOWER_COMPONENTS_REMOTE_DIR, client_dir))
def update(client): """Update a client. - client: name, or part of name of a client (str). We'll get the client's full name. TODO: sometimes, we need to restart the server: to load new templates, for django to load a new module like templatetags, for translations,.... """ client = fabutils.select_client_cfg(client, CFG) wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) with cd(wd): save_variables(client.name) with prefix(VENV_ACTIVATE.format(client.venv)): run("make update")
def bundles_deploy(name): """After bundle_up, we have them in /tmp/. Now, simply copy them into our client's static/bower_components or node_modules. """ client = fabutils.select_client_cfg(name) fabutils.wd(client, CFG) if not exists(BOWER_COMPONENTS_REMOTE_DIR): print( "Error: we can't find the bower components directory in remote /tmp." ) return client_dir = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name, "static") if not exists(client_dir): run("mkdir -p {}".format(client_dir)) run("cp -r {} {}".format(BOWER_COMPONENTS_REMOTE_DIR, client_dir))
def check_uptodate(name=None): """Check wether the distant repo is up to date with the local one. If not, how many commits behind ? """ max_count = 10 if not name: return statusall() client = fabutils.select_client_cfg(name, CFG) if not client: print("No client found with '{}'".format(name)) return 1 wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) # that's local so it conuts the unpushed commits. should be remote git_head = check_output(["git", "rev-parse", "HEAD"]).strip() with cd(wd): res = run("git rev-parse HEAD") # print("commit of {} is {}".format(wd, res)) if res == git_head: print( termcolor.colored("- {} is up to date".format(client.name), "green")) else: git_last_commits = check_output([ "git", "rev-list", "HEAD", "--max-count={}".format(max_count) ]).split("\n") if res in git_last_commits: index = git_last_commits.index(res) print( termcolor.colored("- {}", 'blue').format(client.name) + " is " + termcolor.colored("{}", "yellow").format(index) + " commits behind") else: print( termcolor.colored("- {}", "blue").format(client.name) + " is more than " + termcolor.colored("{}", "red").format(max_count) + " commits behind.")
def rebase(name=None): """Only run git rebase. That may be enough for light updates. (but be careful nothing breaks !) Rebase the given client, or all simultaneously. xxx: check if the pip requirements, the js sources or xxx changed and update what's needed (try not to run apt, pip and npm every time). """ def do_rebase(client): wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) with cd(wd): with prefix(VENV_ACTIVATE.format(client.venv)): run(CMD_REBASE) if name: client = fabutils.select_client_cfg(name) do_rebase(client) check_online(name)
def dbback(name=None): """Copy the db file locally (there), appendding a timestamp, and download it. Only for users marked in production in clients.yaml. """ if not name: clients = sorted(CFG.clients) else: clients = [fabutils.select_client_cfg(name)] # Only for prod: clients = filter(lambda it: it.status == "prod", clients) print( termcolor.colored( "Backup for {} users in prod: {}.".format( len(clients), map(lambda it: it.name, clients)), "yellow")) for client in clients: with cd(fabutils.wd(client, CFG)): # Copy locally. Append a timestamp. run("cp db.db{,.`date +%Y%m%d-%H%M%S`}") # Download db_backed = "backup/db-{}-{}.sqlite".format( client.name, datetime.datetime.now().strftime(DATE_FORMAT)) cmd = "rsync -av {user}@{url}:/home/{user}/{dir}/{client_name}/{project_name}/{db_name} ./{db_backed}".format( user=CFG.user, url=CFG.url, dir=CFG.dir.strip("/"), client_name=client.name, project_name=CFG.project_name, db_name=CFG.db_name, db_backed=db_backed) print("Downloading db of user {}".format( termcolor.colored("{}".format(client.name), "blue"))) print(cmd) os.system(cmd)
def dbback(name=None): """Copy the db file locally (there), appendding a timestamp, and download it. Only for users marked in production in clients.yaml. """ if not name: clients = sorted(CFG.clients) else: clients = [fabutils.select_client_cfg(name)] # Only for prod: clients = filter(lambda it: it.status == "prod", clients) print(termcolor.colored( "Backup for {} users in prod: {}.".format( len(clients), map(lambda it: it.name, clients)), "yellow")) for client in clients: with cd(fabutils.wd(client, CFG)): # Copy locally. Append a timestamp. run("cp db.db{,.`date +%Y%m%d-%H%M%S`}") # Download db_backed = "backup/db-{}-{}.sqlite".format(client.name, datetime.datetime.now().strftime(DATE_FORMAT)) cmd = "rsync -av {user}@{url}:/home/{user}/{dir}/{client_name}/{project_name}/{db_name} ./{db_backed}".format( user=CFG.user, url=CFG.url, dir=CFG.dir.strip("/"), client_name=client.name, project_name=CFG.project_name, db_name=CFG.db_name, db_backed=db_backed ) print("Downloading db of user {}".format(termcolor.colored("{}".format(client.name), "blue"))) print(cmd) os.system(cmd)
def bower_package_version(package, names=None): """What's the installed packages version ? - package: bower package name - names: list of client names (or only beginning of name, as usual). """ # usage = "Usage: fab bower_package_version:package,client" #: With --json, we could parse it and get rid off the sub-deps BOWER_PACKAGE_VERSION = './node_modules/bower/bin/bower list --offline' if not names: clients = CFG.clients else: name = names[0] clients = [fabutils.select_client_cfg(name)] for client in clients: wd = os.path.join(CFG.home, CFG.dir, client.name, CFG.project_name) print(wd) with cd(wd): run(BOWER_PACKAGE_VERSION)
def check_online(client=None): """Check every instance's working. Parallel task on same host with multiprocessing (fabric does on /different/ hosts). """ if not client: sorted_clients = sorted(CFG.clients, key=lambda it: it.name) else: sorted_clients = [fabutils.select_client_cfg(client, CFG)] urls = ["http://{}:{}/fr/".format(CFG.url, c.port) for c in sorted_clients] import multiprocessing pool = multiprocessing.Pool(8) status = pool.map(_request_call, urls) res = zip(status, sorted_clients) for status, client in res: if status != 200: print(termcolor.colored(u"- {:{}} has a pb".format(client.name, COL_WIDTH), "red") + " on {}".format(client['port'])) else: print(u"- {:{}} ".format(client.name, COL_WIDTH) + termcolor.colored("ok", "green"))
def install(name): """Clone and install Abelujo into the given client directory. Create a super user, populate the DB with initial data, if any, run gunicorn with the right port. """ client = fabutils.select_client_cfg(name) client = addict.Dict(client) wd = os.path.join(CFG.home, CFG.dir, client.name) if not exists(wd): run("mkdir {}".format(wd, wd)) # Copy bower_components (experimental) bundles_upload() with cd(wd): run("test -d {} || git clone --recursive {}".format( CFG.project_name, CFG.project_git_url)) bundles_deploy(client.name) # save the port save_variables(client.name) with cd(CFG.project_name): # - create a venv create_venv(client.venv) with prefix(VENV_ACTIVATE.format(client.venv)): # - install, run('make install') # TODO: # - create super user # - populate DB with initial data if any # The csv files may be in nested directories. # run('make odsimport odsfile=$(find /tmp/{}/*csv)'.format(client.name)) # - run gunicorn with the right port, start(client.name)
def install(name): """Clone and install Abelujo into the given client directory. Create a super user, populate the DB with initial data, if any, run gunicorn with the right port. """ client = fabutils.select_client_cfg(name) client = addict.Dict(client) wd = os.path.join(CFG.home, CFG.dir, client.name) if not exists(wd): run("mkdir {}".format(wd, wd)) # Copy bower_components (experimental) bundles_upload(client.name) with cd(wd): run("test -d {} || git clone --recursive {}".format(CFG.project_name, CFG.project_git_url)) bundles_deploy(client.name) # save the port save_variables(client.name) with cd(CFG.project_name): # - create a venv create_venv(client.venv) with prefix(VENV_ACTIVATE.format(client.venv)): # - install, run('make install') # TODO: # - create super user # - populate DB with initial data if any # The csv files may be in nested directories. # run('make odsimport odsfile=$(find /tmp/{}/*csv)'.format(client.name)) # - run gunicorn with the right port, start(client.name)