def postgresql_firstboot(force=False):
exists = os.path.exists(POSTGRESQL_FIRSTBOOT_PATH)
if exists and not force:
# skip if we have already run this before
return
utils.make_dirs(POSTGRESQL_PGDATA_PATH)
cmd = 'pg_ctl -D %s -l %s init' % (POSTGRESQL_PGDATA_PATH,
POSTGRESQL_LOGFILE)
utils.sys_call(cmd)
started = False
try:
started = postgresql_start()
time.sleep(1)
cmd = 'createdb -h 127.0.0.1'
utils.sys_call(cmd)
cmd = 'psql -h 127.0.0.1 -c "%s"'
P = PASSWORD
tz = "set timezone TO 'GMT';"
encoding = "set client_encoding TO 'utf8';"
admin_user = "******" % P
admin_db = "CREATE DATABASE admin WITH OWNER admin;"
test_user = "******" % P
test_db = "CREATE DATABASE test WITH OWNER test;"
[utils.sys_call(cmd % sql) for sql in (tz, encoding, admin_user,
admin_db, test_user, test_db)]
finally:
if started:
postgresql_stop()
utils.write_file(POSTGRESQL_FIRSTBOOT_PATH, '')
return True
def cython(args=None, cmd=''):
cmd = getattr(args, 'command', cmd)
if cmd == 'compile':
utils.sys_call('./setup.py build_ext --inplace')
elif cmd == 'clean':
# FIXME: add *.c too?
utils.remove_file('metrique/*.so')
def postgresql_firstboot(force=False):
exists = os.path.exists(POSTGRESQL_FIRSTBOOT_PATH)
if exists and not force:
# skip if we have already run this before
return
utils.make_dirs(POSTGRESQL_PGDATA_PATH)
cmd = 'pg_ctl -D %s -l %s init' % (POSTGRESQL_PGDATA_PATH,
POSTGRESQL_LOGFILE)
utils.sys_call(cmd)
started = False
try:
started = postgresql_start()
time.sleep(1)
cmd = 'createdb -h 127.0.0.1'
utils.sys_call(cmd)
cmd = 'psql -h 127.0.0.1 -c "%s"'
P = PASSWORD
tz = "set timezone TO 'GMT';"
encoding = "set client_encoding TO 'utf8';"
admin_user = "******" % P
admin_db = "CREATE DATABASE admin WITH OWNER admin;"
test_user = "******" % P
test_db = "CREATE DATABASE test WITH OWNER test;"
[
utils.sys_call(cmd % sql)
for sql in (tz, encoding, admin_user, admin_db, test_user, test_db)
]
finally:
if started:
postgresql_stop()
utils.write_file(POSTGRESQL_FIRSTBOOT_PATH, '')
return True
def cython(args=None, cmd=''):
cmd = getattr(args, 'command', cmd)
if cmd == 'compile':
utils.sys_call('./setup.py build_ext --inplace')
elif cmd == 'clean':
# FIXME: add *.c too?
utils.remove_file('metrique/*.so')
def celerybeat_run(args):
fork = not args.nofork
x = 'beat'
logfile = '--logfile=%s' % CELERYBEAT_LOGFILE
loglvl = '-l INFO'
pidfile = '--pidfile=%s' % CELERYBEAT_PIDFILE
app = '-A %s' % args.tasks_mod
cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
utils.sys_call(cmd, fork=fork, pid_file=CELERYBEAT_PIDFILE)
def celeryd_loop(args):
fork = not args.nofork
x = 'worker'
logfile = '--logfile=%s' % CELERYD_LOGFILE
loglvl = '-l INFO'
pidfile = '--pidfile=%s' % CELERYD_PIDFILE
app = '-A %s' % args.tasks_mod
cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
utils.sys_call(cmd, fork=fork, pid_file=CELERYD_PIDFILE)
def celerybeat_run(args):
fork = not args.nofork
x = 'beat'
logfile = '--logfile=%s' % CELERYBEAT_LOGFILE
loglvl = '-l INFO'
pidfile = '--pidfile=%s' % CELERYBEAT_PIDFILE
app = '-A %s' % args.tasks_mod
cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
utils.sys_call(cmd, fork=fork, pid_file=CELERYBEAT_PIDFILE)
def celeryd_loop(args):
fork = not args.nofork
x = 'worker'
logfile = '--logfile=%s' % CELERYD_LOGFILE
loglvl = '-l INFO'
pidfile = '--pidfile=%s' % CELERYD_PIDFILE
app = '-A %s' % args.tasks_mod
cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app)
utils.sys_call(cmd, fork=fork, pid_file=CELERYD_PIDFILE)
def postgresql_stop(quiet=True):
try:
cmd = 'pg_ctl -D %s stop' % (POSTGRESQL_PGDATA_PATH)
utils.sys_call(cmd)
except RuntimeError as e:
if not quiet:
raise
else:
logger.warn('Failed to stop PostgreSQL: %s' % e)
return False
def postgresql_stop(quiet=True):
try:
cmd = 'pg_ctl -D %s stop' % (POSTGRESQL_PGDATA_PATH)
utils.sys_call(cmd)
except RuntimeError as e:
if not quiet:
raise
else:
logger.warn('Failed to stop PostgreSQL: %s' % e)
return False
def ssl(args=None):
logger.info("Generating self-signed SSL certificate + key + combined pem")
utils.sys_call('openssl req -new -x509 -days 365 -nodes '
'-out %s -keyout %s -batch' % (SSL_CERT, SSL_KEY))
with open(SSL_PEM, 'w') as pem:
with open(SSL_CERT) as cert:
pem.write(''.join(cert.readlines()))
with open(SSL_KEY) as key:
pem.write(''.join(key.readlines()))
os.chmod(SSL_CERT, 0600)
os.chmod(SSL_KEY, 0600)
os.chmod(SSL_PEM, 0600)
def ssl(args=None):
logger.info("Generating self-signed SSL certificate + key + combined pem")
utils.sys_call(
'openssl req -new -x509 -days 365 -nodes '
'-out %s -keyout %s -batch' % (SSL_CERT, SSL_KEY))
with open(SSL_PEM, 'w') as pem:
with open(SSL_CERT) as cert:
pem.write(''.join(cert.readlines()))
with open(SSL_KEY) as key:
pem.write(''.join(key.readlines()))
os.chmod(SSL_CERT, 0600)
os.chmod(SSL_KEY, 0600)
os.chmod(SSL_PEM, 0600)
def setup(args, cmd, pip=False):
pre = not getattr(args, 'no_pre', False)
if pip and not pre:
cmd += ' --pre'
if isinstance(cmd, basestring):
cmd = cmd.strip()
else:
cmd = ' '.join([s.strip() for s in cmd])
if pip:
out = utils.sys_call('pip %s -e .' % cmd)
else:
out = utils.sys_call('python setup.py %s' % cmd)
logger.info(utils.to_encoding(out))
def setup(args, cmd, pip=False):
pre = not getattr(args, 'no_pre', False)
if pip and not pre:
cmd += ' --pre'
if isinstance(cmd, basestring):
cmd = cmd.strip()
else:
cmd = ' '.join([s.strip() for s in cmd])
if pip:
out = utils.sys_call('pip %s -e .' % cmd)
else:
out = utils.sys_call('python setup.py %s' % cmd)
logger.info(utils.to_encoding(out))
def postgresql_start():
if os.path.exists(POSTGRESQL_PIDFILE):
logger.info('PostgreSQL pid found not starting...')
return False
try:
cmd = 'pg_ctl -D %s -l %s -o "-k %s" start' % (
POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE, PIDS_DIR)
utils.sys_call(cmd, sig=signal.SIGTERM, sig_func=postgresql_terminate)
except Exception as e:
logger.warn(e)
return False
else:
return True
def postgresql_start():
if os.path.exists(POSTGRESQL_PIDFILE):
logger.info('PostgreSQL pid found not starting...')
return False
try:
cmd = 'pg_ctl -D %s -l %s -o "-k %s" start' % (
POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE, PIDS_DIR)
utils.sys_call(cmd, sig=signal.SIGTERM, sig_func=postgresql_terminate)
except Exception as e:
logger.warn(e)
return False
else:
return True
def test_osinfo_rpm():
from metrique import pyclient
from metrique.utils import sys_call
if sys_call("which rpm", ignore_errors=True) is None:
# skip these tests, since we don't have rpm installed
return
name = "osinfo_rpm"
db_file = os.path.join(cache_dir, "%s.sqlite" % name)
remove_file(db_file)
m = pyclient(cube=name)
m.objects.drop()
print "Getting RPM objects; might take a few seconds."
m.get_objects()
print " ... done."
k = len(m.objects)
assert k > 0
name = "bash"
_filtered = m.objects.filter(where={"name": name})
assert len(_filtered) == 1
print "Object: %s" % _filtered
_ids = m.objects.flush()
assert len(_ids) == k
remove_file(db_file)
def test_osinfo_rpm():
from metrique import pyclient
from metrique.utils import sys_call
if sys_call('which rpm', ignore_errors=True) is None:
# skip these tests, since we don't have rpm installed
return
name = 'osinfo_rpm'
db_file = os.path.join(cache_dir, '%s.sqlite' % name)
remove_file(db_file)
m = pyclient(cube=name)
m.objects.drop()
print 'Getting RPM objects; might take a few seconds.'
m.get_objects()
print ' ... done.'
k = len(m.objects)
assert k > 0
name = 'bash'
_filtered = m.objects.filter(where={'name': name})
assert len(_filtered) == 1
print 'Object: %s' % _filtered
_ids = m.objects.flush()
assert len(_ids) == k
remove_file(db_file)
def test_sys_call():
from metrique.utils import sys_call
try:
sys_call('ls FILE_THAT_DOES_NOT_EXIST')
except Exception:
pass
else:
assert False
assert sys_call('ls FILE_THAT_DOES_NOT_EXIST', ignore_errors=True) is None
csv_path = os.path.join(fixtures, 'test.csv')
out = sys_call('ls %s' % csv_path).strip()
assert out == csv_path
# should work if passed in as a list of args too
out = sys_call(['ls', csv_path]).strip()
assert out == csv_path
def test_sys_call():
from metrique.utils import sys_call
try:
sys_call('ls FILE_THAT_DOES_NOT_EXIST')
except Exception:
pass
else:
assert False
assert sys_call('ls FILE_THAT_DOES_NOT_EXIST', ignore_errors=True) is None
csv_path = os.path.join(fixtures, 'test.csv')
out = sys_call('ls %s' % csv_path).strip()
assert out == csv_path
# should work if passed in as a list of args too
out = sys_call(['ls', csv_path]).strip()
assert out == csv_path
def test_terminate():
from metrique.utils import terminate, sys_call, get_pid, clear_stale_pids
import signal
pid_file = os.path.join(cache_dir, 'test.pid')
sys_call('sleep 30', fork=True, pid_file=pid_file, shell=False)
sleep(1)
pid = get_pid(pid_file)
running = clear_stale_pids(pid)
assert running
terminate(pid, sig=signal.SIGTERM)
sleep(1)
running = clear_stale_pids(pid)
assert not running
# since we didn't tell it where to find the pid_file
# it won't be cleaned up
assert exists(pid_file)
# this time we point to a pid_file and it gets cleaned up
terminate(pid_file, sig=signal.SIGTERM)
assert not exists(pid_file)
# invalid pids are ignored
assert terminate(-1) is None
def test_terminate():
from metrique.utils import terminate, sys_call, get_pid, clear_stale_pids
import signal
pid_file = os.path.join(cache_dir, 'test.pid')
sys_call('sleep 30', fork=True, pid_file=pid_file, shell=False)
sleep(1)
pid = get_pid(pid_file)
running = clear_stale_pids(pid)
assert running
terminate(pid, sig=signal.SIGTERM)
sleep(1)
running = clear_stale_pids(pid)
assert not running
# since we didn't tell it where to find the pid_file
# it won't be cleaned up
assert exists(pid_file)
# this time we point to a pid_file and it gets cleaned up
terminate(pid_file, sig=signal.SIGTERM)
assert not exists(pid_file)
# invalid pids are ignored
assert terminate(-1) is None
def deploy(args):
virtenv = _deploy_virtenv_init(args)
# make sure we have some basic defaults configured in the environment
firstboot()
# install all dependencies first, before installing metrique
_deploy_extras(args)
cmd = 'install'
setup(args, cmd, pip=False)
if args.develop:
path = pjoin(virtenv, 'lib/python2.7/site-packages/metrique*')
utils.remove_file(path, force=True)
develop(args)
if (args.all or args.cython) and args.develop:
cython(cmd='compile')
# run py.test after install
if args.test:
utils.sys_call('coverage run --source=metrique -m py.test tests')
def deploy(args):
virtenv = _deploy_virtenv_init(args)
# make sure we have some basic defaults configured in the environment
firstboot()
# install all dependencies first, before installing metrique
_deploy_extras(args)
cmd = 'install'
setup(args, cmd, pip=False)
if args.develop:
path = pjoin(virtenv, 'lib/python2.7/site-packages/metrique*')
utils.remove_file(path, force=True)
develop(args)
if (args.all or args.cython) and args.develop:
cython(cmd='compile')
# run py.test after install
if args.test:
utils.sys_call('coverage run --source=metrique -m py.test tests')
def nginx(args):
fork = not args.nofork
cmd = 'nginx -c %s' % NGINX_CONF
if args.command == 'test':
utils.sys_call('%s -t' % cmd)
elif args.command == 'start':
utils.sys_call(cmd, fork=fork)
elif args.command == 'stop':
utils.sys_call('%s -s stop' % cmd)
elif args.command == 'restart':
for cmd in ('stop', 'start'):
args.command = cmd
nginx(args)
elif args.command == 'reload':
utils.sys_call('%s -s reload' % cmd)
else:
raise ValueError("unknown command %s" % args.command)
def nginx(args):
fork = not args.nofork
cmd = 'nginx -c %s' % NGINX_CONF
if args.command == 'test':
utils.sys_call('%s -t' % cmd)
elif args.command == 'start':
utils.sys_call(cmd, fork=fork)
elif args.command == 'stop':
utils.sys_call('%s -s stop' % cmd)
elif args.command == 'restart':
for cmd in ('stop', 'start'):
args.command = cmd
nginx(args)
elif args.command == 'reload':
utils.sys_call('%s -s reload' % cmd)
else:
raise ValueError("unknown command %s" % args.command)
def test_rsync():
from metrique.utils import rsync, sys_call, rand_chars, remove_file
from metrique.utils import read_file
#remove_file(f_1)
#remove_file(dest, force=True)
if not sys_call('which rsync'):
return # skip this test if rsync isn't available
fname = rand_chars(prefix='rsync')
path = os.path.join(cache_dir, fname)
with open(path, 'w') as f:
f.write('test')
dest = os.path.join(tmp_dir, 'rsync')
rsync(targets=path, dest=dest)
assert read_file(os.path.join(dest, fname)) == 'test'
with open(path, 'w') as f:
f.write('test 2')
rsync(targets=path, dest=dest)
assert read_file(os.path.join(dest, fname)) == 'test 2'
remove_file(path, force=True)
remove_file(dest, force=True)
def test_rsync():
from metrique.utils import rsync, sys_call, rand_chars, remove_file
from metrique.utils import read_file
#remove_file(f_1)
#remove_file(dest, force=True)
if not sys_call('which rsync'):
return # skip this test if rsync isn't available
fname = rand_chars(prefix='rsync')
path = os.path.join(cache_dir, fname)
with open(path, 'w') as f:
f.write('test')
dest = os.path.join(tmp_dir, 'rsync')
rsync(targets=path, dest=dest)
assert read_file(os.path.join(dest, fname)) == 'test'
with open(path, 'w') as f:
f.write('test 2')
rsync(targets=path, dest=dest)
assert read_file(os.path.join(dest, fname)) == 'test 2'
remove_file(path, force=True)
remove_file(dest, force=True)
def get_objects(self, uri, pull=True, **kwargs):
'''
Walk through repo commits to generate a list of repo commit
objects.
Each object has the following properties:
* repo uri
* general commit info
* files added, removed fnames
* lines added, removed
* acked_by
* signed_off_by
* resolves
* related
'''
self.repo = repo = git_clone(uri, pull=pull, reflect=True)
# get a full list of all commit SHAs in the repo (all branches)
cmd = 'git rev-list --all'
output = sys_call(cmd, cwd=repo.path)
repo_shas = set(x.strip() for x in output.split('\n') if x)
logger.debug("Total Commits: %s" % len(repo_shas))
cmd = 'git --no-pager log --all --format=sha:%H --numstat'
output = sys_call(cmd)
all_logs = re.sub('\n+', '\n', output)
c_logs = [x for x in [s.strip() for s in all_logs.split('sha:')] if x]
_end = None # once was true, always is true...
objs = []
for c_log in c_logs:
sha, s, all_changes = c_log.partition('\n')
#try:
c = repo.get_object(sha)
# FIXME: not normalizing to UTC
_start = ts2dt(c.commit_time)
#except Exception as e:
# _start = now
# obj = dict(_oid=sha, _start=_start, _end=_end,
# repo_uri=uri, _e={sha: to_encoding(e)})
# self.objects.add(obj)
# continue
# and some basic stuff...
obj = dict(_oid=sha,
_start=_start,
_end=_end,
repo_uri=uri,
tree=c.tree,
parents=c.parents,
author=c.author,
committer=c.committer,
author_time=c.author_time,
message=c.message,
mergetag=c.mergetag,
extra=c.extra)
for _file in all_changes.split('\n'):
_file = _file.strip()
obj.setdefault('files', {})
if not _file:
added, removed, fname = 0, 0, None
else:
added, removed, fname = _file.split('\t')
added = 0 if added == '-' else int(added)
removed = 0 if removed == '-' else int(removed)
# FIXME: sql doesn't nest well..
changes = {'added': added, 'removed': removed}
obj['files'][fname] = changes
# file +/- totals
obj['added'] = sum(
[v.get('added', 0) for v in obj['files'].itervalues()])
obj['removed'] = sum(
[v.get('removed', 0) for v in obj['files'].itervalues()])
# extract interesting bits from the message
obj['acked_by'] = acked_by_re.findall(c.message)
obj['signed_off_by'] = signed_off_by_re.findall(c.message)
obj['resolves'] = resolves_re.findall(c.message)
obj['related'] = related_re.findall(c.message)
objs.append(obj)
self.objects.extend(objs)
return super(Commit, self).get_objects(**kwargs)
def _deploy_extras(args):
# make sure we have the installer basics and their up2date
# pip-accel caches compiled binaries
utils.sys_call('pip install -U pip setuptools virtualenv')
_all = args.all
_ = _all or args.ipython
utils.sys_call('pip install -U ipython pyzmq jinja2') if _ else None
_ = _all or args.test or args.pytest
utils.sys_call('pip install -U pytest coveralls') if _ else None
_ = args.all or args.docs
utils.sys_call('pip install -U sphinx') if _ else None
# pip-accel fails to install this package...
utils.sys_call('pip install -U sphinx_bootstrap_theme') if _ else None
_ = _all or args.supervisord
utils.sys_call('pip install -U supervisor') if _ else None
_ = _all or args.joblib
utils.sys_call('pip install -U joblib') if _ else None
_ = _all or args.postgres
utils.sys_call('pip install -U psycopg2') if _ else None
_ = _all or args.celery
utils.sys_call('pip install -U celery') if _ else None
_ = _all or args.sqlalchemy
utils.sys_call('pip install -U sqlalchemy') if _ else None
_ = _all or args.pymongo
utils.sys_call('pip install -U pymongo pql') if _ else None
_ = _all or args.pandas
utils.sys_call('pip install -U pandas') if _ else None
_ = _all or args.matplotlib
utils.sys_call('pip install -U matplotlib') if _ else None
_ = _all or args.dulwich
utils.sys_call('pip install -U dulwich') if _ else None
_ = _all or args.paramiko
utils.sys_call('pip install -U paramiko') if _ else None
_ = _all or args.cython
utils.sys_call('pip install -U cython') if _ else None
def _deploy_extras(args):
# make sure we have the installer basics and their up2date
# pip-accel caches compiled binaries
utils.sys_call('pip install -U pip setuptools virtualenv')
_all = args.all
_ = _all or args.ipython
utils.sys_call('pip install -U ipython pyzmq jinja2') if _ else None
_ = _all or args.test or args.pytest
utils.sys_call('pip install -U pytest coveralls') if _ else None
_ = args.all or args.docs
utils.sys_call('pip install -U sphinx') if _ else None
# pip-accel fails to install this package...
utils.sys_call('pip install -U sphinx_bootstrap_theme') if _ else None
_ = _all or args.supervisord
utils.sys_call('pip install -U supervisor') if _ else None
_ = _all or args.joblib
utils.sys_call('pip install -U joblib') if _ else None
_ = _all or args.postgres
utils.sys_call('pip install -U psycopg2') if _ else None
_ = _all or args.celery
utils.sys_call('pip install -U celery') if _ else None
_ = _all or args.sqlalchemy
utils.sys_call('pip install -U sqlalchemy') if _ else None
_ = _all or args.pymongo
utils.sys_call('pip install -U pymongo pql') if _ else None
_ = _all or args.pandas
utils.sys_call('pip install -U pandas') if _ else None
_ = _all or args.matplotlib
utils.sys_call('pip install -U matplotlib') if _ else None
_ = _all or args.dulwich
utils.sys_call('pip install -U dulwich') if _ else None
_ = _all or args.paramiko
utils.sys_call('pip install -U paramiko') if _ else None
_ = _all or args.cython
utils.sys_call('pip install -U cython') if _ else None
def get_objects(self, uri, pull=True, **kwargs):
"""
Walk through repo commits to generate a list of repo commit
objects.
Each object has the following properties:
* repo uri
* general commit info
* files added, removed fnames
* lines added, removed
* acked_by
* signed_off_by
* resolves
* related
"""
self.repo = repo = git_clone(uri, pull=pull, reflect=True)
# get a full list of all commit SHAs in the repo (all branches)
cmd = "git rev-list --all"
output = sys_call(cmd, cwd=repo.path)
repo_shas = set(x.strip() for x in output.split("\n") if x)
logger.debug("Total Commits: %s" % len(repo_shas))
cmd = "git --no-pager log --all --format=sha:%H --numstat"
output = sys_call(cmd)
all_logs = re.sub("\n+", "\n", output)
c_logs = [x for x in [s.strip() for s in all_logs.split("sha:")] if x]
_end = None # once was true, always is true...
objs = []
for c_log in c_logs:
sha, s, all_changes = c_log.partition("\n")
# try:
c = repo.get_object(sha)
# FIXME: not normalizing to UTC
_start = ts2dt(c.commit_time)
# except Exception as e:
# _start = now
# obj = dict(_oid=sha, _start=_start, _end=_end,
# repo_uri=uri, _e={sha: to_encoding(e)})
# self.objects.add(obj)
# continue
# and some basic stuff...
obj = dict(
_oid=sha,
_start=_start,
_end=_end,
repo_uri=uri,
tree=c.tree,
parents=c.parents,
author=c.author,
committer=c.committer,
author_time=c.author_time,
message=c.message,
mergetag=c.mergetag,
extra=c.extra,
)
for _file in all_changes.split("\n"):
_file = _file.strip()
obj.setdefault("files", {})
if not _file:
added, removed, fname = 0, 0, None
else:
added, removed, fname = _file.split("\t")
added = 0 if added == "-" else int(added)
removed = 0 if removed == "-" else int(removed)
# FIXME: sql doesn't nest well..
changes = {"added": added, "removed": removed}
obj["files"][fname] = changes
# file +/- totals
obj["added"] = sum([v.get("added", 0) for v in obj["files"].itervalues()])
obj["removed"] = sum([v.get("removed", 0) for v in obj["files"].itervalues()])
# extract interesting bits from the message
obj["acked_by"] = acked_by_re.findall(c.message)
obj["signed_off_by"] = signed_off_by_re.findall(c.message)
obj["resolves"] = resolves_re.findall(c.message)
obj["related"] = related_re.findall(c.message)
objs.append(obj)
self.objects.extend(objs)
return super(Commit, self).get_objects(**kwargs)
def supervisord_run(args):
cmd = 'supervisord -c %s' % SUPERVISORD_CONF
utils.sys_call(cmd, fork=True)
def supervisord_run(args):
cmd = 'supervisord -c %s' % SUPERVISORD_CONF
utils.sys_call(cmd, fork=True)
