def postgresql_firstboot(force=False): exists = os.path.exists(POSTGRESQL_FIRSTBOOT_PATH) if exists and not force: # skip if we have already run this before return utils.make_dirs(POSTGRESQL_PGDATA_PATH) cmd = 'pg_ctl -D %s -l %s init' % (POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE) utils.sys_call(cmd) started = False try: started = postgresql_start() time.sleep(1) cmd = 'createdb -h 127.0.0.1' utils.sys_call(cmd) cmd = 'psql -h 127.0.0.1 -c "%s"' P = PASSWORD tz = "set timezone TO 'GMT';" encoding = "set client_encoding TO 'utf8';" admin_user = "******" % P admin_db = "CREATE DATABASE admin WITH OWNER admin;" test_user = "******" % P test_db = "CREATE DATABASE test WITH OWNER test;" [utils.sys_call(cmd % sql) for sql in (tz, encoding, admin_user, admin_db, test_user, test_db)] finally: if started: postgresql_stop() utils.write_file(POSTGRESQL_FIRSTBOOT_PATH, '') return True
def cython(args=None, cmd=''): cmd = getattr(args, 'command', cmd) if cmd == 'compile': utils.sys_call('./setup.py build_ext --inplace') elif cmd == 'clean': # FIXME: add *.c too? utils.remove_file('metrique/*.so')
def postgresql_firstboot(force=False): exists = os.path.exists(POSTGRESQL_FIRSTBOOT_PATH) if exists and not force: # skip if we have already run this before return utils.make_dirs(POSTGRESQL_PGDATA_PATH) cmd = 'pg_ctl -D %s -l %s init' % (POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE) utils.sys_call(cmd) started = False try: started = postgresql_start() time.sleep(1) cmd = 'createdb -h 127.0.0.1' utils.sys_call(cmd) cmd = 'psql -h 127.0.0.1 -c "%s"' P = PASSWORD tz = "set timezone TO 'GMT';" encoding = "set client_encoding TO 'utf8';" admin_user = "******" % P admin_db = "CREATE DATABASE admin WITH OWNER admin;" test_user = "******" % P test_db = "CREATE DATABASE test WITH OWNER test;" [ utils.sys_call(cmd % sql) for sql in (tz, encoding, admin_user, admin_db, test_user, test_db) ] finally: if started: postgresql_stop() utils.write_file(POSTGRESQL_FIRSTBOOT_PATH, '') return True
def celerybeat_run(args): fork = not args.nofork x = 'beat' logfile = '--logfile=%s' % CELERYBEAT_LOGFILE loglvl = '-l INFO' pidfile = '--pidfile=%s' % CELERYBEAT_PIDFILE app = '-A %s' % args.tasks_mod cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app) utils.sys_call(cmd, fork=fork, pid_file=CELERYBEAT_PIDFILE)
def celeryd_loop(args): fork = not args.nofork x = 'worker' logfile = '--logfile=%s' % CELERYD_LOGFILE loglvl = '-l INFO' pidfile = '--pidfile=%s' % CELERYD_PIDFILE app = '-A %s' % args.tasks_mod cmd = 'celery %s %s %s %s %s' % (x, logfile, loglvl, pidfile, app) utils.sys_call(cmd, fork=fork, pid_file=CELERYD_PIDFILE)
def postgresql_stop(quiet=True): try: cmd = 'pg_ctl -D %s stop' % (POSTGRESQL_PGDATA_PATH) utils.sys_call(cmd) except RuntimeError as e: if not quiet: raise else: logger.warn('Failed to stop PostgreSQL: %s' % e) return False
def ssl(args=None): logger.info("Generating self-signed SSL certificate + key + combined pem") utils.sys_call('openssl req -new -x509 -days 365 -nodes ' '-out %s -keyout %s -batch' % (SSL_CERT, SSL_KEY)) with open(SSL_PEM, 'w') as pem: with open(SSL_CERT) as cert: pem.write(''.join(cert.readlines())) with open(SSL_KEY) as key: pem.write(''.join(key.readlines())) os.chmod(SSL_CERT, 0600) os.chmod(SSL_KEY, 0600) os.chmod(SSL_PEM, 0600)
def ssl(args=None): logger.info("Generating self-signed SSL certificate + key + combined pem") utils.sys_call( 'openssl req -new -x509 -days 365 -nodes ' '-out %s -keyout %s -batch' % (SSL_CERT, SSL_KEY)) with open(SSL_PEM, 'w') as pem: with open(SSL_CERT) as cert: pem.write(''.join(cert.readlines())) with open(SSL_KEY) as key: pem.write(''.join(key.readlines())) os.chmod(SSL_CERT, 0600) os.chmod(SSL_KEY, 0600) os.chmod(SSL_PEM, 0600)
def setup(args, cmd, pip=False): pre = not getattr(args, 'no_pre', False) if pip and not pre: cmd += ' --pre' if isinstance(cmd, basestring): cmd = cmd.strip() else: cmd = ' '.join([s.strip() for s in cmd]) if pip: out = utils.sys_call('pip %s -e .' % cmd) else: out = utils.sys_call('python setup.py %s' % cmd) logger.info(utils.to_encoding(out))
def postgresql_start(): if os.path.exists(POSTGRESQL_PIDFILE): logger.info('PostgreSQL pid found not starting...') return False try: cmd = 'pg_ctl -D %s -l %s -o "-k %s" start' % ( POSTGRESQL_PGDATA_PATH, POSTGRESQL_LOGFILE, PIDS_DIR) utils.sys_call(cmd, sig=signal.SIGTERM, sig_func=postgresql_terminate) except Exception as e: logger.warn(e) return False else: return True
def test_osinfo_rpm(): from metrique import pyclient from metrique.utils import sys_call if sys_call("which rpm", ignore_errors=True) is None: # skip these tests, since we don't have rpm installed return name = "osinfo_rpm" db_file = os.path.join(cache_dir, "%s.sqlite" % name) remove_file(db_file) m = pyclient(cube=name) m.objects.drop() print "Getting RPM objects; might take a few seconds." m.get_objects() print " ... done." k = len(m.objects) assert k > 0 name = "bash" _filtered = m.objects.filter(where={"name": name}) assert len(_filtered) == 1 print "Object: %s" % _filtered _ids = m.objects.flush() assert len(_ids) == k remove_file(db_file)
def test_osinfo_rpm(): from metrique import pyclient from metrique.utils import sys_call if sys_call('which rpm', ignore_errors=True) is None: # skip these tests, since we don't have rpm installed return name = 'osinfo_rpm' db_file = os.path.join(cache_dir, '%s.sqlite' % name) remove_file(db_file) m = pyclient(cube=name) m.objects.drop() print 'Getting RPM objects; might take a few seconds.' m.get_objects() print ' ... done.' k = len(m.objects) assert k > 0 name = 'bash' _filtered = m.objects.filter(where={'name': name}) assert len(_filtered) == 1 print 'Object: %s' % _filtered _ids = m.objects.flush() assert len(_ids) == k remove_file(db_file)
def test_sys_call(): from metrique.utils import sys_call try: sys_call('ls FILE_THAT_DOES_NOT_EXIST') except Exception: pass else: assert False assert sys_call('ls FILE_THAT_DOES_NOT_EXIST', ignore_errors=True) is None csv_path = os.path.join(fixtures, 'test.csv') out = sys_call('ls %s' % csv_path).strip() assert out == csv_path # should work if passed in as a list of args too out = sys_call(['ls', csv_path]).strip() assert out == csv_path
def test_terminate(): from metrique.utils import terminate, sys_call, get_pid, clear_stale_pids import signal pid_file = os.path.join(cache_dir, 'test.pid') sys_call('sleep 30', fork=True, pid_file=pid_file, shell=False) sleep(1) pid = get_pid(pid_file) running = clear_stale_pids(pid) assert running terminate(pid, sig=signal.SIGTERM) sleep(1) running = clear_stale_pids(pid) assert not running # since we didn't tell it where to find the pid_file # it won't be cleaned up assert exists(pid_file) # this time we point to a pid_file and it gets cleaned up terminate(pid_file, sig=signal.SIGTERM) assert not exists(pid_file) # invalid pids are ignored assert terminate(-1) is None
def deploy(args): virtenv = _deploy_virtenv_init(args) # make sure we have some basic defaults configured in the environment firstboot() # install all dependencies first, before installing metrique _deploy_extras(args) cmd = 'install' setup(args, cmd, pip=False) if args.develop: path = pjoin(virtenv, 'lib/python2.7/site-packages/metrique*') utils.remove_file(path, force=True) develop(args) if (args.all or args.cython) and args.develop: cython(cmd='compile') # run py.test after install if args.test: utils.sys_call('coverage run --source=metrique -m py.test tests')
def nginx(args): fork = not args.nofork cmd = 'nginx -c %s' % NGINX_CONF if args.command == 'test': utils.sys_call('%s -t' % cmd) elif args.command == 'start': utils.sys_call(cmd, fork=fork) elif args.command == 'stop': utils.sys_call('%s -s stop' % cmd) elif args.command == 'restart': for cmd in ('stop', 'start'): args.command = cmd nginx(args) elif args.command == 'reload': utils.sys_call('%s -s reload' % cmd) else: raise ValueError("unknown command %s" % args.command)
def test_rsync(): from metrique.utils import rsync, sys_call, rand_chars, remove_file from metrique.utils import read_file #remove_file(f_1) #remove_file(dest, force=True) if not sys_call('which rsync'): return # skip this test if rsync isn't available fname = rand_chars(prefix='rsync') path = os.path.join(cache_dir, fname) with open(path, 'w') as f: f.write('test') dest = os.path.join(tmp_dir, 'rsync') rsync(targets=path, dest=dest) assert read_file(os.path.join(dest, fname)) == 'test' with open(path, 'w') as f: f.write('test 2') rsync(targets=path, dest=dest) assert read_file(os.path.join(dest, fname)) == 'test 2' remove_file(path, force=True) remove_file(dest, force=True)
def get_objects(self, uri, pull=True, **kwargs): ''' Walk through repo commits to generate a list of repo commit objects. Each object has the following properties: * repo uri * general commit info * files added, removed fnames * lines added, removed * acked_by * signed_off_by * resolves * related ''' self.repo = repo = git_clone(uri, pull=pull, reflect=True) # get a full list of all commit SHAs in the repo (all branches) cmd = 'git rev-list --all' output = sys_call(cmd, cwd=repo.path) repo_shas = set(x.strip() for x in output.split('\n') if x) logger.debug("Total Commits: %s" % len(repo_shas)) cmd = 'git --no-pager log --all --format=sha:%H --numstat' output = sys_call(cmd) all_logs = re.sub('\n+', '\n', output) c_logs = [x for x in [s.strip() for s in all_logs.split('sha:')] if x] _end = None # once was true, always is true... objs = [] for c_log in c_logs: sha, s, all_changes = c_log.partition('\n') #try: c = repo.get_object(sha) # FIXME: not normalizing to UTC _start = ts2dt(c.commit_time) #except Exception as e: # _start = now # obj = dict(_oid=sha, _start=_start, _end=_end, # repo_uri=uri, _e={sha: to_encoding(e)}) # self.objects.add(obj) # continue # and some basic stuff... obj = dict(_oid=sha, _start=_start, _end=_end, repo_uri=uri, tree=c.tree, parents=c.parents, author=c.author, committer=c.committer, author_time=c.author_time, message=c.message, mergetag=c.mergetag, extra=c.extra) for _file in all_changes.split('\n'): _file = _file.strip() obj.setdefault('files', {}) if not _file: added, removed, fname = 0, 0, None else: added, removed, fname = _file.split('\t') added = 0 if added == '-' else int(added) removed = 0 if removed == '-' else int(removed) # FIXME: sql doesn't nest well.. changes = {'added': added, 'removed': removed} obj['files'][fname] = changes # file +/- totals obj['added'] = sum( [v.get('added', 0) for v in obj['files'].itervalues()]) obj['removed'] = sum( [v.get('removed', 0) for v in obj['files'].itervalues()]) # extract interesting bits from the message obj['acked_by'] = acked_by_re.findall(c.message) obj['signed_off_by'] = signed_off_by_re.findall(c.message) obj['resolves'] = resolves_re.findall(c.message) obj['related'] = related_re.findall(c.message) objs.append(obj) self.objects.extend(objs) return super(Commit, self).get_objects(**kwargs)
def _deploy_extras(args): # make sure we have the installer basics and their up2date # pip-accel caches compiled binaries utils.sys_call('pip install -U pip setuptools virtualenv') _all = args.all _ = _all or args.ipython utils.sys_call('pip install -U ipython pyzmq jinja2') if _ else None _ = _all or args.test or args.pytest utils.sys_call('pip install -U pytest coveralls') if _ else None _ = args.all or args.docs utils.sys_call('pip install -U sphinx') if _ else None # pip-accel fails to install this package... utils.sys_call('pip install -U sphinx_bootstrap_theme') if _ else None _ = _all or args.supervisord utils.sys_call('pip install -U supervisor') if _ else None _ = _all or args.joblib utils.sys_call('pip install -U joblib') if _ else None _ = _all or args.postgres utils.sys_call('pip install -U psycopg2') if _ else None _ = _all or args.celery utils.sys_call('pip install -U celery') if _ else None _ = _all or args.sqlalchemy utils.sys_call('pip install -U sqlalchemy') if _ else None _ = _all or args.pymongo utils.sys_call('pip install -U pymongo pql') if _ else None _ = _all or args.pandas utils.sys_call('pip install -U pandas') if _ else None _ = _all or args.matplotlib utils.sys_call('pip install -U matplotlib') if _ else None _ = _all or args.dulwich utils.sys_call('pip install -U dulwich') if _ else None _ = _all or args.paramiko utils.sys_call('pip install -U paramiko') if _ else None _ = _all or args.cython utils.sys_call('pip install -U cython') if _ else None
def get_objects(self, uri, pull=True, **kwargs): """ Walk through repo commits to generate a list of repo commit objects. Each object has the following properties: * repo uri * general commit info * files added, removed fnames * lines added, removed * acked_by * signed_off_by * resolves * related """ self.repo = repo = git_clone(uri, pull=pull, reflect=True) # get a full list of all commit SHAs in the repo (all branches) cmd = "git rev-list --all" output = sys_call(cmd, cwd=repo.path) repo_shas = set(x.strip() for x in output.split("\n") if x) logger.debug("Total Commits: %s" % len(repo_shas)) cmd = "git --no-pager log --all --format=sha:%H --numstat" output = sys_call(cmd) all_logs = re.sub("\n+", "\n", output) c_logs = [x for x in [s.strip() for s in all_logs.split("sha:")] if x] _end = None # once was true, always is true... objs = [] for c_log in c_logs: sha, s, all_changes = c_log.partition("\n") # try: c = repo.get_object(sha) # FIXME: not normalizing to UTC _start = ts2dt(c.commit_time) # except Exception as e: # _start = now # obj = dict(_oid=sha, _start=_start, _end=_end, # repo_uri=uri, _e={sha: to_encoding(e)}) # self.objects.add(obj) # continue # and some basic stuff... obj = dict( _oid=sha, _start=_start, _end=_end, repo_uri=uri, tree=c.tree, parents=c.parents, author=c.author, committer=c.committer, author_time=c.author_time, message=c.message, mergetag=c.mergetag, extra=c.extra, ) for _file in all_changes.split("\n"): _file = _file.strip() obj.setdefault("files", {}) if not _file: added, removed, fname = 0, 0, None else: added, removed, fname = _file.split("\t") added = 0 if added == "-" else int(added) removed = 0 if removed == "-" else int(removed) # FIXME: sql doesn't nest well.. changes = {"added": added, "removed": removed} obj["files"][fname] = changes # file +/- totals obj["added"] = sum([v.get("added", 0) for v in obj["files"].itervalues()]) obj["removed"] = sum([v.get("removed", 0) for v in obj["files"].itervalues()]) # extract interesting bits from the message obj["acked_by"] = acked_by_re.findall(c.message) obj["signed_off_by"] = signed_off_by_re.findall(c.message) obj["resolves"] = resolves_re.findall(c.message) obj["related"] = related_re.findall(c.message) objs.append(obj) self.objects.extend(objs) return super(Commit, self).get_objects(**kwargs)
def supervisord_run(args): cmd = 'supervisord -c %s' % SUPERVISORD_CONF utils.sys_call(cmd, fork=True)