Пример #1
0
def patch_text(patch, reverse=False):
    '''Apply text patch to remote server.'''

    def cmd(reverse, dry, path):
        cmd = ['patch']
        if dry: cmd.append('--dry-run')
        if reverse: cmd.append('-R')
        cmd.append('-p1')
        cmd.append('<')
        cmd.append(path)
        return ' '.join(cmd)

    require('base')
    remotep = _remote('%s.patch'%patch)
    put(join(path(patch), 'text.patch'), remotep)
    if confirm('Dry run patch?'):
        with settings(show('stdout'), warn_only = True):
            with cd(env.base):
                run(cmd(reverse, True, remotep))
    if confirm('Execute patch?'):
        with settings(show('stdout'), warn_only = True):
            with cd(env.base):
                run(cmd(reverse, False, remotep))
                log('Applied text patch: %s'%patch)
                run('mv %s patches'%remotep)
Пример #2
0
def dependencies():
    """Installe les modules pythons nécessaires au projet"""
    with settings(show("user"), hide("warnings", "running", "stdout", "stderr")):
        with cd("projects/%(projet)s" % env):
            if exists("requirements.txt"):
                print(yellow("Installation des dépendances du projet..."))
                with prefix("workon %(projet)s" % env):
                    with settings(show("running", "stdout", "stderr")):
                        run("pip install -r requirements.txt")
            else:
                print(red('Aucun fichier "requirements.txt" trouvé.'))
Пример #3
0
def dependencies():
    '''Installe les modules pythons nécessaires au projet'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):
        with cd('projects/%(projet)s' % env):
            if exists('requirements.txt'):
                print(yellow('Installation des dépendances du projet...'))
                with prefix('workon %(projet)s' % env):
                    with settings(show('running', 'stdout', 'stderr')):
                        run('pip install -r requirements.txt')
            else:
                print(red('Aucun fichier "requirements.txt" trouvé.'))
Пример #4
0
def dependencies():
    '''Vérification des modules nécessaires au projet'''
    with settings(show('user'), hide('warnings', 'running', 'stdout', 'stderr')):
        with cd('%(base_install)s' % env):
            if exists('requirements.txt'):
                print(yellow('Installation des dépendances du projet...'))
                with prefix('workon %(projet)s' % env):
                    with settings(show('running', 'stdout', 'stderr')):
                        run('pip install -r requirements.txt')
            else:
                print(red('Aucun fichier "requirements.txt" trouvé.'))
        with prefix('workon %(projet)s' % env):
            with cd('%(base_install)s' % env):
                run('./manage.py collectstatic --noinput')
Пример #5
0
def pull_seaflowpy():
    gitdir = "/home/ubuntu/git"
    repodir = os.path.join(gitdir, "seaflowpy")
    with quiet():
        if run("test -d {}".format(gitdir)).failed:
            run("mkdir {}".format(gitdir))
        if run("test -d {}".format(repodir)).failed:
            with show("running", "warnings", "stderr"):
                run("git clone https://github.com/armbrustlab/seaflowpy {}".format(repodir))
    with cd(repodir), hide("stdout"):
        run("git pull")
        run("python setup.py install")
        run("python setup.py test")
        with show("stdout"):
            run("seaflowpy_filter --version")
Пример #6
0
def django_project():
    """Créer un projet django dans son virtualenv"""
    with settings(show("user"), hide("warnings", "running", "stdout", "stderr")):
        project()
        domain()
        locale()
        if not exists("/home/%(user)s/.virtualenvs/%(projet)s" % env):
            # if confirm('Pas de virtualenv "%(projet)s", faut-il le créer ?' % env, default=False):
            run("mkvirtualenv %(projet)s" % env)
            run("source .bash_profile")
        with prefix("workon %(projet)s" % env):
            run("pip install django")
            print(green("Django installé."))
        if not exists("projects/%s/" % (env.projet)):
            print(yellow("le projet %(projet)s n’existe pas encore" % env))
            if confirm('Créer un projet django nommé "%(projet)s" ?' % env, default=False):
                with cd("projects/"):
                    with prefix("workon %(projet)s" % env):
                        run("django-admin.py startproject %s" % (env.projet))
                        print(green('Projet Django "%(projet)s" : OK.' % env))
        else:
            print(green('Projet Django "%(projet)s" : OK.' % env))
        # créer la db avec le nom du projet (idempotent)
        icanhaz.postgres.database(env.projet, env.user, template="template_postgis", locale=env.locale)
        print(green("Base de données %(projet)s : OK." % env))
        django_wsgi()
        apache_vhost()
        dependencies()
        sudo("apachectl restart")
Пример #7
0
def fabcommand(args, actions):
    levels = [
        "status", "aborts", "warnings", "running", "stdout", "stderr", "user",
        "debug"
    ]

    level_aliases = {
        "output": ["stdout", "stderr"],
        "everything": ["warnings", "running", "user", "output"]
    }

    lhide = level_aliases["everything"]
    lshow = []

    if args.verbose == 1:
        lshow = levels[:3]
        lhide = levels[3:]
    elif args.verbose == 2:
        lshow = levels[:4]
        lhide = levels[4:]
    elif args.verbose >= 3 or args.debug:
        lshow = levels
        lhide = []


#   fabcmd += " --fabfile {4}/fabfile.py \
# setup_env:confdir={0},packages={1},templates={2},cluster_name={3},\
# autoconf={5},disable_colors={6},key_inject={7} \
# ".format(args.confdir, env.packages, env.templates, args.cluster_name,
#          env.lib, args.autoconf, args.disable_colors, args.key_inject)

    with settings(hide(*lhide), show(*lshow)):
        for a in actions:
            fn = getattr(fabfile, a)
            execute(fn)
Пример #8
0
def dsetool_cmd(nodes, options):
    """Run a dsetool command simultaneously on each node specified"""
    cmd = 'JAVA_HOME={java_home} {dsetool_cmd} {options}'.format(java_home=JAVA_HOME,
                                                                 dsetool_cmd=os.path.join(dse.get_bin_path(),
                                                                                          'dsetool'), options=options)
    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=nodes, warn_only=True):
        return execute(fab.run, cmd)
Пример #9
0
def server_setup():
    '''Installation serveur pour Ubuntu >= 10.10'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):

        sudo('apt-get -y install aptitude')  # demande un input
        print(yellow('Mise à jour de l’index APT...'))
        fabtools.deb.update_index()  # apt-get quiet update
        print(yellow('Mise à jour des paquets debian installés...'))
        fabtools.deb.upgrade()
        # paquets communs à tous les serveurs Django+geodjango
        print(yellow('Installation des paquets de base...'))
        pretty_apt([
            'git-core', 'mercurial', 'gcc', 'curl', 'build-essential',
            'libfreetype6', 'libfreetype6-dev', 'liblcms1-dev', 'libpng12-dev',
            'libjpeg8-dev', 'python-imaging', 'supervisor',
            'python-setuptools', 'nano', 'python-dev', 'swig', 'memcached',
            'python-memcache', 'libgeoip1'
        ])

        # pip special case
        if not fabtools.python.is_pip_installed():
            fabtools.python.install_pip()
            print(green('pip : installé.'))

        virtualenv_setup()
        #config apache
        apache_setup()

        postgresql()
Пример #10
0
def fabcommand(args, actions):
    levels = ["status", "aborts", "warnings", "running",
              "stdout", "stderr", "user", "debug"]

    level_aliases = {
        "output": ["stdout", "stderr"],
        "everything": ["warnings", "running", "user", "output"]
    }

    lhide = level_aliases["everything"]
    lshow = []

    if args.verbose == 1:
        lshow = levels[:3]
        lhide = levels[3:]
    elif args.verbose == 2:
        lshow = levels[:4]
        lhide = levels[4:]
    elif args.verbose >= 3 or args.debug:
        lshow = levels
        lhide = []

#   fabcmd += " --fabfile {4}/fabfile.py \
# setup_env:confdir={0},packages={1},templates={2},cluster_name={3},\
# autoconf={5},disable_colors={6},key_inject={7} \
# ".format(args.confdir, env.packages, env.templates, args.cluster_name,
#          env.lib, args.autoconf, args.disable_colors, args.key_inject)

    with settings(hide(*lhide), show(*lshow)):
        for a in actions:
            fn = getattr(fabfile, a)
            execute(fn)
Пример #11
0
def postgresql():
    '''PostgreSQL 9.1 + PostGIS 1.5'''
    with settings(show('user'), hide('warnings', 'running', 'stdout', 'stderr')):
        #if 'pgpass' not in env.keys():
        #    prompt('Passe PostgreSQL :', default=pgpass, key='pgpass')
        print(yellow('Configuration PostgreSQL+PostGIS...'))
        pretty_apt(['postgresql', 'binutils', 'gdal-bin', 'libproj-dev', 'postgresql-9.1-postgis',
                    'postgresql-server-dev-9.1', 'python-psycopg2', 'libgeoip1'])
        # print(yellow('Upgrading all packages...'))
        # fabtools.deb.upgrade()
        # création d'un utilisateur postgresql avec le meme nom d'utilisateur
        if not fabtools.postgres.user_exists(env.user):
            fabtools.postgres.create_user(env.user, env.pg_pass)
            sudo('''psql -c "ALTER ROLE %(user)s CREATEDB;"''' % env, user='******')
            sudo('''psql -c "ALTER USER %(user)s with SUPERUSER;"''' % env, user='******')
            print(green('Création d’un superuser "%(user)s" PostgreSQL.' % env))
        if not exists('.pgpass'):
            run('echo "*:*:*:%(user)s:%(pg_pass)s" >> .pgpass' % env)
            sudo('chmod 0600 .pgpass')
            print(green('Création du fichier .pgpass.'))
        run('curl https://docs.djangoproject.com/en/dev/_downloads/create_template_postgis-debian.sh -o postgis.sh')
        run('chmod +x postgis.sh')
        run('./postgis.sh')
        #postgresql_net_access()
        require.postgres.server()  # start server
Пример #12
0
def test_perf(params=''):
    """
    Runs the performance tests against the configured service and produce the report
    in dist/

    :params str params: Parameters to pass to Funkload bench

    Examples::

        fab dist.test_perf
        fab dist.test_perf:"-c 1:15 -D 1"

    """
    try:
        import funkload
    except ImportError:
        abort('Funkload module missing, please install it first')

    # Create report folder if needed
    report_dir = dist_join('report/html')
    if not os.path.exists(report_dir):
        os.makedirs(report_dir)

    # Run the Funkload tests in perf test folder
    with lcd(rel_join('tests/perf')):
        with settings(show('running', 'stdout')):
            local('fl-run-bench %s tests.py MultiprojectTestCase.test_smoke' %
                  params)
            local('fl-build-report -o %s --html smoke-bench.xml' % report_dir)

    logger.info('Testing completed. Test report can be found in: %s' %
                report_dir)
Пример #13
0
    def run(self, *args, **kwargs):
        """Called by fabric. Will set up the task and launch it. 
        Do not override this in your children tasks, use `operation` instead."""

        # check if the task is called via Python or via `fab` in terminal
        self.called_via_fab = self._called_via_fab
        self._called_via_fab = True
        try:
            # display help if the users ask for it, then exit
            assert args[0] == "help"
            self.logging = True
            self.log(self.get_usage())
            return
        except: 
            pass

        self.setup(*args, **kwargs)        
        self.pre_run()

        with hide(*self.hide), show(*self.show):
            # run the actual task logic
            r = self.operation(*self.args, **self.kwargs)            

        self.post_run()
        return r        
Пример #14
0
def pvcluster(remote_dir, paraview_cmd, paraview_args,
              paraview_port, paraview_remote_port, job_dict,
              shell_cmd):

    with show('debug'), \
        remote_tunnel(int(paraview_remote_port),
                      local_port=int(paraview_port)):
        with shell_env(PARAVIEW_CMD=paraview_cmd,
                       PARAVIEW_ARGS=paraview_args):
            run('echo $PARAVIEW_HOME')
            run('echo $PARAVIEW_ARGS')
            run('mkdir -p ' + remote_dir)
            with cd(remote_dir):
                cmd_line = shell_cmd
                cmd_line += 'mycluster --create pvserver.job --jobname=pvserver'
                cmd_line += ' --jobqueue ' + job_dict['job_queue']
                cmd_line += ' --ntasks ' + job_dict['job_ntasks']
                cmd_line += ' --taskpernode ' + job_dict['job_ntaskpernode']
                if 'vizstack' in paraview_args:
                    cmd_line += ' --script mycluster-viz-paraview.bsh'
                else:
                    cmd_line += ' --script mycluster-paraview.bsh'
                cmd_line += ' --project ' + job_dict['job_project']
                run(cmd_line)
                run('chmod u+rx pvserver.job')
                run(shell_cmd+'mycluster --immediate --submit pvserver.job')
Пример #15
0
def django_project():
    '''Créer un projet django dans son virtualenv'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):
        project()
        domain()
        locale()
        if not exists('/home/%(user)s/.virtualenvs/%(projet)s' % env):
            #if confirm('Pas de virtualenv "%(projet)s", faut-il le créer ?' % env, default=False):
            run('mkvirtualenv %(projet)s' % env)
            run('source .bash_profile')
        with prefix('workon %(projet)s' % env):
            run('pip install django')
            print(green('Django installé.'))
        if not exists('projects/%s/' % (env.projet)):
            print(yellow('le projet %(projet)s n’existe pas encore' % env))
            if confirm('Créer un projet django nommé "%(projet)s" ?' % env,
                       default=False):
                with cd('projects/'):
                    with prefix('workon %(projet)s' % env):
                        run('django-admin.py startproject %s' % (env.projet))
                        print(green('Projet Django "%(projet)s" : OK.' % env))
        else:
            print(green('Projet Django "%(projet)s" : OK.' % env))
        #créer la db avec le nom du projet (idempotent)
        icanhaz.postgres.database(env.projet,
                                  env.user,
                                  template='template_postgis',
                                  locale=env.locale)
        print(green('Base de données %(projet)s : OK.' % env))
        django_wsgi()
        apache_vhost()
        dependencies()
        sudo('apachectl restart')
Пример #16
0
def free_cache():
    if dconf.HOST_CONN != 'docker':
        with show('everything'), settings(warn_only=True):  # pylint: disable=not-context-manager
            res = sudo("sh -c \"echo 3 > /proc/sys/vm/drop_caches\"")
            if res.failed:
                LOG.error('%s (return code %s)', res.stderr.strip(),
                          res.return_code)
Пример #17
0
def servers_deploy():
    #Skipping virtualenv because of complication from different server OS versios etc.
    with show('debug'):
        # Create experiment directory if not exists
        run('apt-get update', shell=False)
        # Because of error that curl was not installed when cloning
        # Because of confusion between git and git-fm for olde debian versions
        run('apt-get install -y curl git-core screen rsync', shell=False)
        with settings(warn_only=True):
            if (run("test -d %s" % '/root/ttfb',
                    shell=False).return_code) == 1:
                print 'Creating working directory'
                run("mkdir %s" % '/root/ttfb', shell=False)
        with cd('/root/ttfb'):
            with settings(warn_only=True):
                if (run("test -d %s" % 'TTFB_ML',
                        shell=False).return_code) == 1:
                    print 'Cloning repository'
                    #Using git in the url since HTTP does not work for older versions of git
                    run("git clone git://github.com/emmdim/TTFB_ML.git",
                        shell=False)
        #Not using env.code_dir_clients cause olde git versions do not pull when in subfolder
        with cd('/root/ttfb/TTFB_ML'):
            run("git pull", shell=False)
            run("ls", shell=False)
Пример #18
0
    def run(self, *args, **kwargs):
        """Called by fabric. Will set up the task and launch it. 
        Do not override this in your children tasks, use `operation` instead."""

        # check if the task is called via Python or via `fab` in terminal
        self.called_via_fab = self._called_via_fab
        self._called_via_fab = True
        try:
            # display help if the users ask for it, then exit
            assert args[0] == "help"
            self.logging = True
            self.log(self.get_usage())
            return
        except:
            pass

        self.setup(*args, **kwargs)
        self.pre_run()

        with hide(*self.hide), show(*self.show):
            # run the actual task logic
            r = self.operation(*self.args, **self.kwargs)

        self.post_run()
        return r
Пример #19
0
def postgresql():
    '''PostgreSQL 8.4 + PostGIS 1.5'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):
        project()
        if 'pgpass' not in env.keys():
            prompt('Passe PostgreSQL :', default=pgpass, key='pgpass')
        print(yellow('Configuration PostgreSQL+PostGIS...'))
        pretty_apt([
            'libpq-dev', 'binutils', 'gdal-bin', 'libproj-dev',
            'postgresql-8.4-postgis', 'postgresql-server-dev-8.4',
            'python-psycopg2'
        ])
        fabtools.deb.upgrade()
        # création d'un utilisateur postgresql avec le meme nom d'utilisateur
        if not fabtools.postgres.user_exists(env.user):
            fabtools.postgres.create_user(env.user, env.pgpass)
            sudo('''psql -c "ALTER ROLE %(user)s CREATEDB;"''' % env,
                 user='******')
            sudo('''psql -c "ALTER USER %(user)s with SUPERUSER;"''' % env,
                 user='******')
            print(green('Création d’un superuser "%(user)s" PostgreSQL.' %
                        env))
        if not exists('.pgpass'):
            run('echo "*:*:*:%(user)s:%(pgpass)s" >> .pgpass' % env)
            sudo('chmod 0600 .pgpass')
            print(green('Création du fichier .pgpass.'))
        postgis_template()
        postgresql_net_access()
        icanhaz.postgres.server()  #start server
Пример #20
0
def dsetool_cmd(nodes, options):
    """Run a dsetool command simultaneously on each node specified"""
    cmd = 'JAVA_HOME={java_home} {dsetool_cmd} {options}'.format(java_home=JAVA_HOME,
                                                                 dsetool_cmd=os.path.join(dse.get_bin_path(),
                                                                                          'dsetool'), options=options)
    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=nodes, warn_only=True):
        return execute(fab.run, cmd)
Пример #21
0
def setup():
    '''Installation de base pour Ubuntu >= 10.10'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):
        project()
        domain()
        #if gandi
        locale()
        sudo('apt-get install aptitude')
        print(yellow('Mise à jour de l’index APT...'))
        fabtools.deb.update_index()  # apt-get quiet update
        print(yellow('Mise à jour des paquets debian installés...'))
        fabtools.deb.upgrade()
        # paquets communs à tous les serveurs Django+geodjango
        print(yellow('Installation des paquets de base...'))
        pretty_apt([
            'git-core', 'mercurial', 'gcc', 'curl', 'build-essential',
            'python-imaging', 'python-setuptools', 'nano', 'memcached',
            'python-memcache'
        ])

        # pip special case
        if not fabtools.python.is_pip_installed():
            fabtools.python.install_pip()
            print(green('pip : installé.'))

        environnement()
        #config apache
        if (env.websrv == 1):
            apache()
        elif (env.websrv == 2):
            apache_nginx()
Пример #22
0
def postgresql():
    '''PostgreSQL 9.1 + PostGIS 1.5'''
    with settings(show('user'), hide('warnings', 'running', 'stdout',
                                     'stderr')):
        #if 'pgpass' not in env.keys():
        #    prompt('Passe PostgreSQL :', default=pgpass, key='pgpass')
        print(yellow('Configuration PostgreSQL+PostGIS...'))
        pretty_apt([
            'postgresql', 'binutils', 'gdal-bin', 'libproj-dev',
            'postgresql-9.1-postgis', 'postgresql-server-dev-9.1',
            'python-psycopg2', 'libgeoip1'
        ])
        # print(yellow('Upgrading all packages...'))
        # fabtools.deb.upgrade()
        # création d'un utilisateur postgresql avec le meme nom d'utilisateur
        if not fabtools.postgres.user_exists(env.user):
            fabtools.postgres.create_user(env.user, env.pg_pass)
            sudo('''psql -c "ALTER ROLE %(user)s CREATEDB;"''' % env,
                 user='******')
            sudo('''psql -c "ALTER USER %(user)s with SUPERUSER;"''' % env,
                 user='******')
            print(green('Création d’un superuser "%(user)s" PostgreSQL.' %
                        env))
        if not exists('.pgpass'):
            run('echo "*:*:*:%(user)s:%(pg_pass)s" >> .pgpass' % env)
            sudo('chmod 0600 .pgpass')
            print(green('Création du fichier .pgpass.'))
        run('curl https://docs.djangoproject.com/en/dev/_downloads/create_template_postgis-debian.sh -o postgis.sh'
            )
        run('chmod +x postgis.sh')
        run('./postgis.sh')
        #postgresql_net_access()
        require.postgres.server()  # start server
Пример #23
0
def postgresql():
    """PostgreSQL 8.4 + PostGIS 1.5"""
    with settings(show("user"), hide("warnings", "running", "stdout", "stderr")):
        project()
        if "pgpass" not in env.keys():
            prompt("Passe PostgreSQL :", default=pgpass, key="pgpass")
        print(yellow("Configuration PostgreSQL+PostGIS..."))
        pretty_apt(
            [
                "libpq-dev",
                "binutils",
                "gdal-bin",
                "libproj-dev",
                "postgresql-8.4-postgis",
                "postgresql-server-dev-8.4",
                "python-psycopg2",
            ]
        )
        fabtools.deb.upgrade()
        # création d'un utilisateur postgresql avec le meme nom d'utilisateur
        if not fabtools.postgres.user_exists(env.user):
            fabtools.postgres.create_user(env.user, env.pgpass)
            sudo('''psql -c "ALTER ROLE %(user)s CREATEDB;"''' % env, user="******")
            sudo('''psql -c "ALTER USER %(user)s with SUPERUSER;"''' % env, user="******")
            print(green('Création d’un superuser "%(user)s" PostgreSQL.' % env))
        if not exists(".pgpass"):
            run('echo "*:*:*:%(user)s:%(pgpass)s" >> .pgpass' % env)
            sudo("chmod 0600 .pgpass")
            print(green("Création du fichier .pgpass."))
        postgis_template()
        postgresql_net_access()
        icanhaz.postgres.server()  # start server
Пример #24
0
def _create_or_update_virtualenv(virtualenv_root,
                                 virtualenv_name,
                                 requirements_paths,
                                 virtualenv_flags=None):
    with show('output'):
        virtualenv_path = '/'.join((virtualenv_root, virtualenv_name))
        if not exists(virtualenv_path):
            if virtualenv_flags is None:
                virtualenv_flags = ''
            puts("virtualenv not found in {}, creating one.".format(virtualenv_root))
            run("virtualenv {} {}".format(virtualenv_path, virtualenv_flags))

        if isinstance(requirements_paths, string_types):
            requirements_paths = [requirements_paths]
        for requirements_path in requirements_paths:
            puts("Uploading {} to temporary file.".format(requirements_path))
            temp_req = run("mktemp /tmp/streamparse_requirements-XXXXXXXXX.txt")
            put(requirements_path, temp_req)

            puts("Updating virtualenv: {}".format(virtualenv_name))
            cmd = "source {}".format(os.path.join(virtualenv_path, 'bin/activate'))
            with prefix(cmd):
                # Make sure we're using latest pip so options work as expected
                run("pip install --upgrade 'pip~=9.0'", pty=False)
                run("pip install -r {} --exists-action w --upgrade "
                    "--upgrade-strategy only-if-needed".format(temp_req),
                    pty=False)

            run("rm {}".format(temp_req))
Пример #25
0
def remote_build(hoststring, password):
    env.host_string = hoststring
    env.password = password
    env.abort_on_prompts = True
    env.reject_unknown_hosts = False
    with show('exceptions'):
        try:
            run('clang-format-3.7 --version')
        except:
            install_clang_format()
        if not fabric.contrib.files.exists('~/desktop-crashup'):
            run('mkdir ~/desktop-crashup')
        things_to_put = [
            'demoapp', 'crashup', 'tests', 'cmake',
            'build_linux.py', 'CMakeLists.txt'
        ]

        for f in things_to_put:
            put_tar(f, '~/desktop-crashup/')
        with cd('~/desktop-crashup/'):
            if not fabric.contrib.files.exists('~/desktop-crashup/venv'):
                run('virtualenv venv')
            with prefix('source venv/bin/activate'):
                # verbose version in case of problems:
                # run('xvfb-run -e /dev/stdout -a python build_linux.py')
                # '-a' to try different display number if 99 is already taken
                run('xvfb-run -a python build_linux.py')
Пример #26
0
def server_setup():
    '''Installation serveur pour Ubuntu >= 10.10'''
    with settings(show('user'), hide('warnings', 'running', 'stdout', 'stderr')):

        sudo('apt-get -y install aptitude')  # demande un input
        print(yellow('Mise à jour de l’index APT...'))
        fabtools.deb.update_index()  # apt-get quiet update
        print(yellow('Mise à jour des paquets debian installés...'))
        fabtools.deb.upgrade()
        # paquets communs à tous les serveurs Django+geodjango
        print(yellow('Installation des paquets de base...'))
        pretty_apt(['git-core', 'mercurial', 'gcc', 'curl', 'build-essential',
                    'libfreetype6', 'libfreetype6-dev', 'liblcms1-dev', 'libpng12-dev',
                    'libjpeg8-dev', 'python-imaging', 'supervisor',
                    'python-setuptools', 'nano', 'python-dev', 'swig',
                    'memcached', 'python-memcache', 'libgeoip1'])

        # pip special case
        if not fabtools.python.is_pip_installed():
            fabtools.python.install_pip()
            print(green('pip : installé.'))

        virtualenv_setup()
        #config apache
        apache_setup()

        postgresql()
Пример #27
0
def test_perf(params=''):
    """
    Runs the performance tests against the configured service and produce the report
    in dist/

    :params str params: Parameters to pass to Funkload bench

    Examples::

        fab dist.test_perf
        fab dist.test_perf:"-c 1:15 -D 1"

    """
    try:
        import funkload
    except ImportError:
        abort('Funkload module missing, please install it first')

    # Create report folder if needed
    report_dir = dist_join('report/html')
    if not os.path.exists(report_dir):
        os.makedirs(report_dir)

    # Run the Funkload tests in perf test folder
    with lcd(rel_join('tests/perf')):
        with settings(show('running','stdout')):
            local('fl-run-bench %s tests.py MultiprojectTestCase.test_smoke' % params)
            local('fl-build-report -o %s --html smoke-bench.xml' % report_dir)

    logger.info('Testing completed. Test report can be found in: %s' % report_dir)
Пример #28
0
def remote_tmpdir():
    """Create and return the name of a temporary directory at the remote
    location.
    """
    with settings(show('stdout')):
        return remote_py_cmd(['import tempfile',
                              'print tempfile.mkdtemp()'])
Пример #29
0
def remote_listdir(path):
    """Return a list of files found in the given remote directory."""
    with settings(show('stdout')):
        s = remote_py_cmd(["import os",
                           "print os.listdir('%s')" % path.replace('\\', '/')])
    s = s.strip()[1:-1]
    return [part.strip("'") for part in s.split(', ')]
Пример #30
0
def coop_project_setup():
    '''Créer un projet django dans son virtualenv'''
    set_project()
    set_domain()
    with settings(show('user')):#, hide('warnings', 'running', 'stdout', 'stderr')):
        if not exists('/home/%(user)s/.virtualenvs/%(projet)s' % env):
            # if confirm('Pas de virtualenv "%(projet)s", faut-il le créer ?' % env, default=False):
            run('mkvirtualenv --system-site-packages %(projet)s' % env)
            run('source .bash_profile')
        with prefix('workon %(projet)s' % env):
            run('pip install git+git://github.com/credis/django-coop.git')
            print(green('Django et django-coop installé.'))
        if not exists('projects/%s/' % (env.projet)):
            print(yellow('le projet %(projet)s n’existe pas encore' % env))
            if confirm('Créer un projet django nommé "%(projet)s" ?' % env,
                        default=False):
                with cd('projects/'):
                    with prefix('workon %(projet)s' % env):
                        run('coop-admin.py startproject %(projet)s --domain %(domain)s' % env)
                        print(green('Projet Django-coop "%(projet)s" : Installé.' % env))
                        # coop-admin scripts creates the WSGI script so we won't call django_wsgi()
                with cd('projects/%(projet)s' % env):
                    with prefix('workon %(projet)s' % env):
                        run('chmod +x manage.py')
                        run('chmod -R g+rw media')
                vrfy_supervisor_conf()

        else:
            print(yellow('Projet Django-coop nommé "%(projet)s" : déjà installé.' % env))
            # TODO proposer de réinstaller
            vrfy_supervisor_conf()
Пример #31
0
def coop_set_project():
    '''Créer un projet django dans son virtualenv'''
    with settings(show('user'), hide('warnings', 'running', 'stdout', 'stderr')):
        if not exists('/home/%(user)s/.virtualenvs/%(projet)s' % env):
            # if confirm('Pas de virtualenv "%(projet)s", faut-il le créer ?' % env, default=False):
            run('mkvirtualenv --no-site-packages %(projet)s' % env)
            run('source .bash_profile')

            with cd('%(base_install)s' % env):
                with prefix('workon %(projet)s' % env):
                    run('chmod +x manage.py')
                    run('mkdir media')
                    run('chmod -R g+rw media')

        else:
            #with prefix('workon %(projet)s' % env):
            #run('pip install --timeout=240 -r %(base_install)s/requirements.txt' % env)
            print(yellow('Projet Django-coop nommé "%(projet)s" : déjà installé.' % env))
            # TODO proposer de réinstaller

        with prefix('workon %(projet)s' % env):
            print(yellow('Récupération des dépendances python du projet coop-mes'))
            run('pip install --timeout=1024 --use-mirrors -r %(base_install)s/requirements.txt' % env)
            print(green('Récupération des dépendances python du projet coop-mes'))
             
        # Création du répertoire de logs   
        if not exists('%(base_install)s/logs' % env):
            run('mkdir %(base_install)s/logs' % env)
Пример #32
0
def download_master_builder_key(stackname):
    pdata = project_data_for_stackname(stackname)
    region = pdata['aws']['region']
    master_stack = core.find_master(region)
    private_key = "/root/.ssh/id_rsa"
    with stack_conn(master_stack):
        with show('exceptions'): # I actually get better exceptions with this disabled
            return fab_get(private_key, use_sudo=True, return_stream=True, label="master builder key %s:%s" % (master_stack, private_key))
Пример #33
0
def cqlsh(script, node):
    """Run a cqlsh script on a node"""
    global cqlsh_path
    script = script.replace("\n", " ")
    cmd = '{cqlsh_path} --no-color {host} -e "{script}"'.format(cqlsh_path=cqlsh_path, host=node, script=script)

    with common.fab.settings(fab.show("warnings", "running", "stdout", "stderr"), hosts=node):
        return execute(fab.run, cmd)[node]
Пример #34
0
def cqlsh(script, node):
    """Run a cqlsh script on a node"""
    global cqlsh_path
    script = script.replace('\n', ' ')
    cmd = '{cqlsh_path} --no-color {host} -e "{script}"'.format(cqlsh_path=cqlsh_path, host=node, script=script)

    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=node):
        return execute(fab.run, cmd)[node]
Пример #35
0
def run_cmd(cmd, user, **kwargs):
    with show("everything"):
        with settings(warn_only=True):
            command_result = (run(cmd, **kwargs) if user == env.user else sudo(
                cmd, user=user, **kwargs))
    if command_result.return_code != 0:
        raise RuntimeError('Command failed to run: %s' % cmd)
    return command_result
Пример #36
0
def relocate_env():
    with cd(env.release_path):
        with show('output'):
            if env.virtualenvtools_strategy == 'update-path':
                run('%(virtualenvtools_executable)s --update-path %(virtualenvtools_venv_path)s' % env)
            elif env.virtualenvtools_strategy == 'reinitialize':
                run('rm %(virtualenvtools_venv_path)s/bin/python*' % env)
                run('%(virtualenvtools_executable)s --reinitialize %(virtualenvtools_venv_path)s' % env)
Пример #37
0
def cqlsh(script, node):
    """Run a cqlsh script on a node"""
    global cqlsh_path
    script = script.replace('\n', ' ')
    cmd = '{cqlsh_path} --no-color {host} -e "{script}"'.format(cqlsh_path=cqlsh_path, host=node, script=script)

    with common.fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=node):
        return execute(fab.run, cmd)[node]
Пример #38
0
 def test_local_false_global_true(self):
     """
     combine_stderr: False kwarg => overrides global True value
     """
     with show('everything'):
         env.combine_stderr = True
         r = run("both_streams", combine_stderr=False)
     eq_("stdout", r.stdout)
     eq_("stderr", r.stderr)
Пример #39
0
 def test_local_true_global_false(self):
     """
     combine_stderr: True kwarg => overrides global False value
     """
     with show('everything'):
         env.combine_stderr = False
         r = run("both_streams", combine_stderr=True)
     eq_("ssttddoeurtr", r.stdout)
     eq_(r.stderr, "")
Пример #40
0
 def test_local_none_global_false(self):
     """
     combine_stderr: no kwarg => uses global value (False)
     """
     with show('everything'):
         env.combine_stderr = False
         r = run("both_streams")
     eq_("stdout", r.stdout)
     eq_("stderr", r.stderr)
Пример #41
0
    def post_run(self):
        """Called after the task has been run"""

        with show('everything'):
            self.trigger_hooks()

        # reset some attributes
        self.parent = None
        self.start_message = None
Пример #42
0
def jexec(name, command, *args, **kwargs):
    """ execute the given command inside the given jail by creating a new ssh session
        to the host.
    """
    with settings(show("output"), warn_only=True):
        hosts = env.hosts
        env.hosts = [name]
        execute(command, *args, **kwargs)
        env.hosts = hosts
Пример #43
0
def delta(upstream='upstream', bsd=True):
    with cd(env.current_path):
        version = run("git rev-parse --short HEAD", quiet=True)

    local('git fetch -q %s' % upstream)
    with show('output'):
        local('git log --pretty="%%h %%s: %%b" --merges %s..%s/master | '
              'sed -%s "s/Merge pull request #([0-9]+) from ([^/]+)\\/[^:]+/#\\1\\/\\2/"' % (
                  version, upstream, 'E' if bsd else 'r'))
Пример #44
0
def run_local_topology(
    name=None, env_name=None, time=0, options=None, config_file=None
):
    """Run a topology locally using Flux and `storm jar`."""
    name, topology_file = get_topology_definition(name, config_file=config_file)
    config = get_config(config_file=config_file)
    env_name, env_config = get_env_config(env_name, config_file=config_file)
    topology_class = get_topology_from_file(topology_file)

    set_topology_serializer(env_config, config, topology_class)

    storm_options = resolve_options(
        options, env_config, topology_class, name, local_only=True
    )
    if storm_options["topology.acker.executors"] != 0:
        storm_options["topology.acker.executors"] = 1
    storm_options["topology.workers"] = 1

    # Set parallelism based on env_name if necessary
    for spec in topology_class.specs:
        if isinstance(spec.par, dict):
            spec.par = spec.par.get(env_name)

    # Check Storm version is the same
    local_version = local_storm_version()
    project_version = storm_lib_version()
    if local_version != project_version:
        raise ValueError(
            "Local Storm version, {}, is not the same as the "
            "version in your project.clj, {}. The versions must "
            "match.".format(local_version, project_version)
        )

    # Prepare a JAR that has Storm dependencies packaged
    topology_jar = jar_for_deploy(simple_jar=False)

    if time <= 0:
        time = 9223372036854775807  # Max long value in Java

    # Write YAML file
    with show("output"):
        with NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as yaml_file:
            topology_flux_dict = topology_class.to_flux_dict(name)
            topology_flux_dict["config"] = storm_options
            if yaml.version_info < (0, 15):
                yaml.safe_dump(topology_flux_dict, yaml_file, default_flow_style=False)
            else:
                yml = yaml.YAML(typ="safe", pure=True)
                yml.default_flow_style = False
                yml.dump(topology_flux_dict, yaml_file)
            cmd = (
                "storm jar {jar} org.apache.storm.flux.Flux --local --no-splash "
                "--sleep {time} {yaml}".format(
                    jar=topology_jar, time=time, yaml=yaml_file.name
                )
            )
            local(cmd)
Пример #45
0
    def post_run(self):
        """Called after the task has been run"""

        with show('everything'):
            self.trigger_hooks()

        # reset some attributes
        self.parent = None
        self.start_message = None
Пример #46
0
def pvserver(remote_dir, paraview_cmd, paraview_port, paraview_remote_port):

    with show('debug'), remote_tunnel(int(paraview_remote_port), local_port=int(paraview_port)), cd(remote_dir):
        # with cd(remote_dir):
        if not use_multiprocess:
            run('sleep 2;' + paraview_cmd + '</dev/null &>/dev/null&', pty=False)
        else:
            #    # run('sleep 2;'+paraview_cmd+'&>/dev/null',pty=False)
            run('sleep 2;' + paraview_cmd)  # , pty=False)
Пример #47
0
def pvserver(remote_dir, paraview_cmd, paraview_port, paraview_remote_port):

    with show('debug'), remote_tunnel(int(paraview_remote_port),local_port=int(paraview_port)), cd(remote_dir):
        # with cd(remote_dir):
        if not use_multiprocess:
            run('sleep 2;'+paraview_cmd+'</dev/null &>/dev/null&', pty=False)
        else:
            #    # run('sleep 2;'+paraview_cmd+'&>/dev/null',pty=False)
            run('sleep 2;'+paraview_cmd)  # , pty=False)
Пример #48
0
 def test_local_none_global_false(self):
     """
     combine_stderr: no kwarg => uses global value (False)
     """
     with show('everything'):
         env.combine_stderr = False
         r = run("both_streams")
     eq_("stdout", r.stdout)
     eq_("stderr", r.stderr)
Пример #49
0
 def test_local_false_global_true(self):
     """
     combine_stderr: False kwarg => overrides global True value
     """
     with show('everything'):
         env.combine_stderr = True
         r = run("both_streams", combine_stderr=False)
     eq_("stdout", r.stdout)
     eq_("stderr", r.stderr)
Пример #50
0
 def test_local_true_global_false(self):
     """
     combine_stderr: True kwarg => overrides global False value
     """
     with show('everything'):
         env.combine_stderr = False
         r = run("both_streams", combine_stderr=True)
     eq_("ssttddoeurtr", r.stdout)
     eq_(r.stderr, "")
Пример #51
0
 def extractfile(commit, path):
     dirs, file = os.path.split(path)
     dirs = mkdirs('%s/%s'%(commit, dirs))
     with settings(show('stdout'), warn_only = True):
         if commit == 'target':
             commit_id = master.commit
         else:
             commit_id = selected_branch.commit
         get_file(commit_id, path, join(binary_base, commit, path))
Пример #52
0
def relocate_env():
    with cd(env.release_path):
        with show('output'):
            if env.virtualenvtools_strategy == 'update-path':
                run('%(virtualenvtools_executable)s --update-path %(virtualenvtools_venv_path)s'
                    % env)
            elif env.virtualenvtools_strategy == 'reinitialize':
                run('rm %(virtualenvtools_venv_path)s/bin/python*' % env)
                run('%(virtualenvtools_executable)s --reinitialize %(virtualenvtools_venv_path)s'
                    % env)
Пример #53
0
def remote_machine(user, address, key_filename, debug):

    display = show('everything') if debug else hide('everything')

    return settings(
        display,
        host_string='{0}@{1}'.format(user, address),
        key_filename=key_filename,
        disable_known_hosts=True,
    )
Пример #54
0
def _setup_postgres(db_name):
    create_user = False
    with hide('output'):
        sudo('apt-get install -y postgresql postgresql-contrib libpq-dev')
        if not sudo('psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname=\'{}\'"'.format(db_name), user='******').strip():
            create_user = True

    with show('output', 'user'):
        sudo('createuser --no-createdb --no-superuser --no-createrole --pwprompt {}'.format(db_name), user='******')
        sudo('createdb --owner={db_name} {db_name}'.format(db_name=db_name), user='******')
Пример #55
0
 def test_local_none_global_true(self):
     """
     combine_stderr: no kwarg => uses global value (True)
     """
     with show('everything'):
         r = run("both_streams")
     # Note: the exact way the streams are jumbled here is an implementation
     # detail of our fake SSH server and may change in the future.
     eq_("ssttddoeurtr", r.stdout)
     eq_(r.stderr, "")
Пример #56
0
def set_permissions_on_spark_data_dir(nodes,
                                      spark_data_dir=os.path.join(
                                          '/', 'var', 'lib', 'spark'),
                                      user='******'):
    with fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'),
                      hosts=nodes):
        execute(fab.sudo,
                'chmod -R 777 {spark_data}'.format(spark_data=spark_data_dir))
        execute(
            fab.sudo, 'chown {user}:{user} {spark_data}'.format(
                user=user, spark_data=spark_data_dir))
Пример #57
0
def make_remote_spark_data_dir(nodes,
                               spark_data_dir=os.path.join(
                                   '/', 'var', 'lib', 'spark'),
                               remove_existing_spark_data=True):
    with fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'),
                      hosts=nodes):
        if remove_existing_spark_data:
            execute(fab.sudo,
                    'rm -rf {spark_data}'.format(spark_data=spark_data_dir))
        execute(fab.sudo,
                'mkdir -p {spark_data}'.format(spark_data=spark_data_dir))