Exemple #1
0
def postgresql_firstboot(force=False):
    exists = os.path.exists(POSTGRESQL_FIRSTBOOT_PATH)
    if exists and not force:
        # skip if we have already run this before
        return
    utils.make_dirs(POSTGRESQL_PGDATA_PATH)

    cmd = 'pg_ctl -D %s -l %s init' % (POSTGRESQL_PGDATA_PATH,
                                       POSTGRESQL_LOGFILE)
    utils.sys_call(cmd)

    started = False
    try:
        started = postgresql_start()
        time.sleep(1)
        cmd = 'createdb -h 127.0.0.1'
        utils.sys_call(cmd)
        cmd = 'psql -h 127.0.0.1 -c "%s"'
        P = PASSWORD
        tz = "set timezone TO 'GMT';"
        encoding = "set client_encoding TO 'utf8';"
        admin_user = "******" % P
        admin_db = "CREATE DATABASE admin WITH OWNER admin;"
        test_user = "******" % P
        test_db = "CREATE DATABASE test WITH OWNER test;"
        [utils.sys_call(cmd % sql) for sql in (tz, encoding, admin_user,
                                               admin_db, test_user, test_db)]
    finally:
        if started:
            postgresql_stop()

    utils.write_file(POSTGRESQL_FIRSTBOOT_PATH, '')
    return True
Exemple #2
0
def cython(args=None, cmd=''):
    cmd = getattr(args, 'command', cmd)
    if cmd == 'compile':
        utils.sys_call('./setup.py build_ext --inplace')
    elif cmd == 'clean':
        # FIXME: add *.c too?
        utils.remove_file('metrique/*.so')
Exemple #3
0
def celerybeat_run(args):
    fork = not args.nofork
    x = 'beat'
    logfile = '--logfile=%s' % CELERYBEAT_LOGFILE
    loglvl = '-l INFO'
    pidfile = '--pidfile=%s' % CELERYBEAT_PIDFILE
    app = '-A %s' % args.tasks_mod
    cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
    utils.sys_call(cmd, fork=fork, pid_file=CELERYBEAT_PIDFILE)
Exemple #4
0
def celeryd_loop(args):
    fork = not args.nofork
    x = 'worker'
    logfile = '--logfile=%s' % CELERYD_LOGFILE
    loglvl = '-l INFO'
    pidfile = '--pidfile=%s' % CELERYD_PIDFILE
    app = '-A %s' % args.tasks_mod
    cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
    utils.sys_call(cmd, fork=fork, pid_file=CELERYD_PIDFILE)
Exemple #5
0
def postgresql_stop(quiet=True):
    try:
        cmd = 'pg_ctl -D %s stop' % (POSTGRESQL_PGDATA_PATH)
        utils.sys_call(cmd)
    except RuntimeError as e:
        if not quiet:
            raise
        else:
            logger.warn('Failed to stop PostgreSQL: %s' % e)
            return False
Exemple #6
0
def setup(args, cmd, pip=False):
    pre = not getattr(args, 'no_pre', False)
    if pip and not pre:
        cmd += ' --pre'
    if isinstance(cmd, basestring):
        cmd = cmd.strip()
    else:
        cmd = ' '.join([s.strip() for s in cmd])
    if pip:
        out = utils.sys_call('pip %s -e .' % cmd)
    else:
        out = utils.sys_call('python setup.py %s' % cmd)
    logger.info(utils.to_encoding(out))
Exemple #7
0
def ssl(args=None):
    logger.info("Generating self-signed SSL certificate + key + combined pem")
    utils.sys_call(
        'openssl req -new -x509 -days 365 -nodes '
        '-out %s -keyout %s -batch' % (SSL_CERT, SSL_KEY))
    with open(SSL_PEM, 'w') as pem:
        with open(SSL_CERT) as cert:
            pem.write(''.join(cert.readlines()))
        with open(SSL_KEY) as key:
            pem.write(''.join(key.readlines()))
    os.chmod(SSL_CERT, 0600)
    os.chmod(SSL_KEY, 0600)
    os.chmod(SSL_PEM, 0600)
Exemple #8
0
def postgresql_start():
    if os.path.exists(POSTGRESQL_PIDFILE):
        logger.info('PostgreSQL pid found not starting...')
        return False

    try:
        cmd = 'pg_ctl -D %s -l %s -o "-k %s" start' % (
            POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE, PIDS_DIR)
        utils.sys_call(cmd, sig=signal.SIGTERM, sig_func=postgresql_terminate)
    except Exception as e:
        logger.warn(e)
        return False
    else:
        return True
def test_osinfo_rpm():
    from metrique import pyclient
    from metrique.utils import sys_call

    if sys_call("which rpm", ignore_errors=True) is None:
        # skip these tests, since we don't have rpm installed
        return

    name = "osinfo_rpm"
    db_file = os.path.join(cache_dir, "%s.sqlite" % name)
    remove_file(db_file)
    m = pyclient(cube=name)
    m.objects.drop()

    print "Getting RPM objects; might take a few seconds."
    m.get_objects()
    print " ... done."
    k = len(m.objects)
    assert k > 0

    name = "bash"
    _filtered = m.objects.filter(where={"name": name})
    assert len(_filtered) == 1
    print "Object: %s" % _filtered

    _ids = m.objects.flush()
    assert len(_ids) == k

    remove_file(db_file)
Exemple #10
0
def test_sys_call():
    from metrique.utils import sys_call

    try:
        sys_call('ls FILE_THAT_DOES_NOT_EXIST')
    except Exception:
        pass
    else:
        assert False

    assert sys_call('ls FILE_THAT_DOES_NOT_EXIST', ignore_errors=True) is None

    csv_path = os.path.join(fixtures, 'test.csv')
    out = sys_call('ls %s' % csv_path).strip()
    assert out == csv_path

    # should work if passed in as a list of args too
    out = sys_call(['ls', csv_path]).strip()
    assert out == csv_path
Exemple #11
0
def test_terminate():
    from metrique.utils import terminate, sys_call, get_pid, clear_stale_pids
    import signal

    pid_file = os.path.join(cache_dir, 'test.pid')
    sys_call('sleep 30', fork=True, pid_file=pid_file, shell=False)
    sleep(1)
    pid = get_pid(pid_file)
    running = clear_stale_pids(pid)
    assert running
    terminate(pid, sig=signal.SIGTERM)
    sleep(1)
    running = clear_stale_pids(pid)
    assert not running
    # since we didn't tell it where to find the pid_file
    # it won't be cleaned up
    assert exists(pid_file)
    # this time we point to a pid_file and it gets cleaned up
    terminate(pid_file, sig=signal.SIGTERM)
    assert not exists(pid_file)
    # invalid pids are ignored
    assert terminate(-1) is None
Exemple #12
0
def deploy(args):
    virtenv = _deploy_virtenv_init(args)

    # make sure we have some basic defaults configured in the environment
    firstboot()

    # install all dependencies first, before installing metrique
    _deploy_extras(args)

    cmd = 'install'
    setup(args, cmd, pip=False)

    if args.develop:
        path = pjoin(virtenv, 'lib/python2.7/site-packages/metrique*')
        utils.remove_file(path, force=True)
        develop(args)

    if (args.all or args.cython) and args.develop:
        cython(cmd='compile')

    # run py.test after install
    if args.test:
        utils.sys_call('coverage run --source=metrique -m py.test tests')
Exemple #13
0
def nginx(args):
    fork = not args.nofork
    cmd = 'nginx -c %s' % NGINX_CONF
    if args.command == 'test':
        utils.sys_call('%s -t' % cmd)
    elif args.command == 'start':
        utils.sys_call(cmd, fork=fork)
    elif args.command == 'stop':
        utils.sys_call('%s -s stop' % cmd)
    elif args.command == 'restart':
        for cmd in ('stop', 'start'):
            args.command = cmd
            nginx(args)
    elif args.command == 'reload':
        utils.sys_call('%s -s reload' % cmd)
    else:
        raise ValueError("unknown command %s" % args.command)
Exemple #14
0
def test_rsync():
    from metrique.utils import rsync, sys_call, rand_chars, remove_file
    from metrique.utils import read_file
    #remove_file(f_1)
    #remove_file(dest, force=True)
    if not sys_call('which rsync'):
        return   # skip this test if rsync isn't available
    fname = rand_chars(prefix='rsync')
    path = os.path.join(cache_dir, fname)
    with open(path, 'w') as f:
        f.write('test')
    dest = os.path.join(tmp_dir, 'rsync')
    rsync(targets=path, dest=dest)
    assert read_file(os.path.join(dest, fname)) == 'test'
    with open(path, 'w') as f:
        f.write('test 2')
    rsync(targets=path, dest=dest)
    assert read_file(os.path.join(dest, fname)) == 'test 2'
    remove_file(path, force=True)
    remove_file(dest, force=True)
Exemple #15
0
def _deploy_extras(args):
    # make sure we have the installer basics and their up2date
    # pip-accel caches compiled binaries
    utils.sys_call('pip install -U pip setuptools virtualenv')

    _all = args.all
    _ = _all or args.ipython
    utils.sys_call('pip install -U ipython pyzmq jinja2') if _ else None
    _ = _all or args.test or args.pytest
    utils.sys_call('pip install -U pytest coveralls') if _ else None
    _ = args.all or args.docs
    utils.sys_call('pip install -U sphinx') if _ else None
    # pip-accel fails to install this package...
    utils.sys_call('pip install -U sphinx_bootstrap_theme') if _ else None
    _ = _all or args.supervisord
    utils.sys_call('pip install -U supervisor') if _ else None
    _ = _all or args.joblib
    utils.sys_call('pip install -U joblib') if _ else None
    _ = _all or args.postgres
    utils.sys_call('pip install -U psycopg2') if _ else None
    _ = _all or args.celery
    utils.sys_call('pip install -U celery') if _ else None
    _ = _all or args.sqlalchemy
    utils.sys_call('pip install -U sqlalchemy') if _ else None
    _ = _all or args.pymongo
    utils.sys_call('pip install -U pymongo pql') if _ else None
    _ = _all or args.pandas
    utils.sys_call('pip install -U pandas') if _ else None
    _ = _all or args.matplotlib
    utils.sys_call('pip install -U matplotlib') if _ else None
    _ = _all or args.dulwich
    utils.sys_call('pip install -U dulwich') if _ else None
    _ = _all or args.paramiko
    utils.sys_call('pip install -U paramiko') if _ else None
    _ = _all or args.cython
    utils.sys_call('pip install -U cython') if _ else None
Exemple #16
0
    def get_objects(self, uri, pull=True, **kwargs):
        """
        Walk through repo commits to generate a list of repo commit
        objects.

        Each object has the following properties:
            * repo uri
            * general commit info
            * files added, removed fnames
            * lines added, removed
            * acked_by
            * signed_off_by
            * resolves
            * related
        """
        self.repo = repo = git_clone(uri, pull=pull, reflect=True)
        # get a full list of all commit SHAs in the repo (all branches)
        cmd = "git rev-list --all"
        output = sys_call(cmd, cwd=repo.path)
        repo_shas = set(x.strip() for x in output.split("\n") if x)
        logger.debug("Total Commits: %s" % len(repo_shas))

        cmd = "git --no-pager log --all --format=sha:%H --numstat"
        output = sys_call(cmd)
        all_logs = re.sub("\n+", "\n", output)
        c_logs = [x for x in [s.strip() for s in all_logs.split("sha:")] if x]

        _end = None  # once was true, always is true...
        objs = []
        for c_log in c_logs:
            sha, s, all_changes = c_log.partition("\n")
            # try:
            c = repo.get_object(sha)
            # FIXME: not normalizing to UTC
            _start = ts2dt(c.commit_time)
            # except Exception as e:
            #    _start = now
            #    obj = dict(_oid=sha, _start=_start, _end=_end,
            #               repo_uri=uri, _e={sha: to_encoding(e)})
            #    self.objects.add(obj)
            #    continue

            # and some basic stuff...
            obj = dict(
                _oid=sha,
                _start=_start,
                _end=_end,
                repo_uri=uri,
                tree=c.tree,
                parents=c.parents,
                author=c.author,
                committer=c.committer,
                author_time=c.author_time,
                message=c.message,
                mergetag=c.mergetag,
                extra=c.extra,
            )

            for _file in all_changes.split("\n"):
                _file = _file.strip()
                obj.setdefault("files", {})
                if not _file:
                    added, removed, fname = 0, 0, None
                else:
                    added, removed, fname = _file.split("\t")
                    added = 0 if added == "-" else int(added)
                    removed = 0 if removed == "-" else int(removed)
                    # FIXME: sql doesn't nest well..
                    changes = {"added": added, "removed": removed}
                    obj["files"][fname] = changes

            # file +/- totals
            obj["added"] = sum([v.get("added", 0) for v in obj["files"].itervalues()])
            obj["removed"] = sum([v.get("removed", 0) for v in obj["files"].itervalues()])

            # extract interesting bits from the message
            obj["acked_by"] = acked_by_re.findall(c.message)
            obj["signed_off_by"] = signed_off_by_re.findall(c.message)
            obj["resolves"] = resolves_re.findall(c.message)
            obj["related"] = related_re.findall(c.message)
            objs.append(obj)
        self.objects.extend(objs)

        return super(Commit, self).get_objects(**kwargs)
Exemple #17
0
def supervisord_run(args):
    cmd = 'supervisord -c %s' % SUPERVISORD_CONF
    utils.sys_call(cmd, fork=True)