def __init__(self, context): from vcttesting.docker import Docker, params_from_env from vcttesting.hgmo import HgCluster if 'HGMO_STATE_FILE' not in os.environ: print('Do not know where to store state.') print( 'Set the HGMO_STATE_FILE environment variable and try again.') sys.exit(1) if 'DOCKER_STATE_FILE' not in os.environ: print('Do not where to store Docker state.') print( 'Set the DOCKER_STATE_FILE environment variable and try again.' ) sys.exit(1) docker_url, tls = params_from_env(os.environ) docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls) if not docker.is_alive(): print('Docker not available') sys.exit(1) self.c = HgCluster( docker, os.environ['HGMO_STATE_FILE'], ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'), master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'), web_image=os.environ.get('DOCKER_HGWEB_IMAGE'), pulse_image=os.environ.get('DOCKER_PULSE_IMAGE'))
def __init__(self, context): from vcttesting.docker import Docker, params_from_env from vcttesting.hgmo import HgCluster if 'DOCKER_STATE_FILE' not in os.environ: print('Do not where to store Docker state.') print('Set the DOCKER_STATE_FILE environment variable and try again.') sys.exit(1) docker_url, tls = params_from_env(os.environ) docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls) if not docker.is_alive(): print('Docker not available') sys.exit(1) self.c = HgCluster(docker)
def __init__(self, context): from vcttesting.docker import Docker, params_from_env from vcttesting.hgmo import HgCluster if 'HGMO_STATE_FILE' not in os.environ: print('Do not know where to store state.') print('Set the HGMO_STATE_FILE environment variable and try again.') sys.exit(1) if 'DOCKER_STATE_FILE' not in os.environ: print('Do not where to store Docker state.') print('Set the DOCKER_STATE_FILE environment variable and try again.') sys.exit(1) docker_url, tls = params_from_env(os.environ) docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls) self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'], ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'), master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'), web_image=os.environ.get('DOCKER_HGWEB_IMAGE'))
class HgmoCommands(object): def __init__(self, context): from vcttesting.docker import Docker, params_from_env from vcttesting.hgmo import HgCluster if 'HGMO_STATE_FILE' not in os.environ: print('Do not know where to store state.') print('Set the HGMO_STATE_FILE environment variable and try again.') sys.exit(1) if 'DOCKER_STATE_FILE' not in os.environ: print('Do not where to store Docker state.') print('Set the DOCKER_STATE_FILE environment variable and try again.') sys.exit(1) docker_url, tls = params_from_env(os.environ) docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls) if not docker.is_alive(): print('Docker not available') sys.exit(1) self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'], ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'), master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'), web_image=os.environ.get('DOCKER_HGWEB_IMAGE'), pulse_image=os.environ.get('DOCKER_PULSE_IMAGE')) @Command('start', category='hgmo', description='Start a hg.mozilla.org cluster') @CommandArgument('--master-ssh-port', type=int, help='Port number on which SSH server should listen') def start(self, master_ssh_port=None): s = self.c.start(master_ssh_port=master_ssh_port, coverage='CODE_COVERAGE' in os.environ) print('SSH Hostname: %s' % s['master_ssh_hostname']) print('SSH Port: %s' % s['master_ssh_port']) print('LDAP URI: %s' % s['ldap_uri']) for url in s['web_urls']: print('Web URL: %s' % url) print('Pulse: %s:%d' % (s['pulse_hostname'], s['pulse_hostport'])) @Command('shellinit', category='hgmo', description='Print shell commands to export variables') def shellinit(self): print('export SSH_CID=%s' % self.c.master_id) print('export PULSE_HOST=%s' % self.c.pulse_hostname) print('export PULSE_PORT=%s' % self.c.pulse_hostport) print('export SSH_SERVER=%s' % self.c.master_ssh_hostname) print('export SSH_PORT=%d' % self.c.master_ssh_port) # Don't export the full value because spaces. print('export SSH_HOST_RSA_KEY=%s' % self.c.master_host_rsa_key.split()[1]) print('export SSH_HOST_ED25519_KEY=%s' % self.c.master_host_ed25519_key.split()[1]) for i, url in enumerate(self.c.web_urls): print('export HGWEB_%d_URL=%s' % (i, url)) for i, cid in enumerate(self.c.web_ids): print('export HGWEB_%d_CID=%s' % (i, cid)) for i, hostport in enumerate(self.c.kafka_hostports): print('export KAFKA_%d_HOSTPORT=%s' % (i, hostport)) print('export ZOOKEEPER_CONNECT=%s' % self.c.zookeeper_connect) @Command('stop', category='hgmo', description='Stop a hg.mozilla.org cluster') def stop(self): self.c.stop() @Command('clean', category='hgmo', description='Clean up all references to this cluster') def clean(self): self.c.clean() @Command('create-ldap-user', category='hgmo', description='Create a new user in LDAP') @CommandArgument('email', help='Email address associated with user') @CommandArgument('username', help='System account name') @CommandArgument('uid', type=int, help='Numeric user ID to associate with user') @CommandArgument('fullname', help='Full name of the user') @CommandArgument('--key-file', help='Use or create an SSH key') @CommandArgument('--scm-level', type=int, choices=(1, 2, 3), help='Add the user to the specified SCM level groups') @CommandArgument('--no-hg-access', action='store_true', help='Do not grant Mercurial access to user') @CommandArgument('--hg-disabled', action='store_true', help='Set hgAccess to FALSE') def create_ldap_user(self, email, username, uid, fullname, key_file=None, scm_level=None, no_hg_access=False, hg_disabled=False): self.c.ldap.create_user(email, username, uid, fullname, key_filename=key_file, scm_level=scm_level, hg_access=not no_hg_access, hg_enabled=not hg_disabled) @Command('add-ssh-key', category='hgmo', description='Add an SSH public key to a user') @CommandArgument('email', help='Email address of user to modify') @CommandArgument('key', help='SSH public key string') def add_ssh_key(self, email, key): if key == '-': key = sys.stdin.read().strip().encode('utf-8') self.c.ldap.add_ssh_key(email, key) @Command('add-user-to-group', category='hgmo', description='Add a user to an LDAP group') @CommandArgument('email', help='Email address of user to modify') @CommandArgument('group', help='Name of LDAP group to add user to') def add_user_to_group(self, email, group): self.c.ldap.add_user_to_group(email, group) @Command('create-repo', category='hgmo', description='Create a repository in the cluster') @CommandArgument('name', help='Name of repository to create') @CommandArgument('group', default='scm_level_1', help='LDAP group that owns repo') @CommandArgument('--generaldelta', action='store_true', help='Create repository with generaldelta storage') def create_repo(self, name, group, generaldelta=True): out = self.c.create_repo(name, group=group, generaldelta=generaldelta) if out: sys.stdout.write(out) @Command('aggregate-code-coverage', category='hgmo', description='Aggregate code coverage results to a directory') @CommandArgument('destdir', help='Directory where to save code coverage files') def aggregate_code_coverage(self, destdir): self.c.aggregate_code_coverage(destdir) @Command('exec', category='hgmo', description='Execute a command in a Docker container') @CommandArgument('--detach', action='store_true', help='Do not wait for process to finish') @CommandArgument('name', help='Name of container to execute inside') @CommandArgument('command', help='Command to execute', nargs=argparse.REMAINDER) def execute(self, name, command, detach=False): if name == 'hgssh': cid = self.c.master_id elif name == 'pulse': cid = self.c.pulse_id elif name.startswith('hgweb'): i = int(name[5:]) cid = self.c.web_ids[i] else: print('invalid name. must be "hgssh" or "hgwebN"') return 1 cmd = ['docker', 'exec'] if 'TESTTMP' not in os.environ: cmd.append('-it') if detach: cmd.append('-d') cmd.append(cid) cmd.extend(command) return subprocess.call(cmd) @Command('download-mirror-ssh-keys', category='hgmo', description='Downloads SSH keys used by mirrors') @CommandArgument('out_dir', help='Directory in which to write the keys') def download_mirror_ssh_keys(self, out_dir): priv, pub = self.c.get_mirror_ssh_keys()[0:2] with open(os.path.join(out_dir, 'mirror'), 'wb') as fh: fh.write(priv) os.chmod(os.path.join(out_dir, 'mirror'), stat.S_IRUSR | stat.S_IWUSR) with open(os.path.join(out_dir, 'mirror.pub'), 'wb') as fh: fh.write(pub) print('SSH keys written to %s' % out_dir)
class HgmoCommands(object): def __init__(self, context): from vcttesting.docker import Docker, params_from_env from vcttesting.hgmo import HgCluster if 'HGMO_STATE_FILE' not in os.environ: print('Do not know where to store state.') print('Set the HGMO_STATE_FILE environment variable and try again.') sys.exit(1) if 'DOCKER_STATE_FILE' not in os.environ: print('Do not where to store Docker state.') print('Set the DOCKER_STATE_FILE environment variable and try again.') sys.exit(1) docker_url, tls = params_from_env(os.environ) docker = Docker(os.environ['DOCKER_STATE_FILE'], docker_url, tls=tls) self.c = HgCluster(docker, os.environ['HGMO_STATE_FILE'], ldap_image=os.environ.get('DOCKER_LDAP_IMAGE'), master_image=os.environ.get('DOCKER_HGMASTER_IMAGE'), web_image=os.environ.get('DOCKER_HGWEB_IMAGE')) @Command('start', category='hgmo', description='Start a hg.mozilla.org cluster') @CommandArgument('--master-ssh-port', type=int, help='Port number on which SSH server should listen') def start(self, master_ssh_port=None): s = self.c.start(master_ssh_port=master_ssh_port, coverage='CODE_COVERAGE' in os.environ) print('SSH Hostname: %s' % s['master_ssh_hostname']) print('SSH Port: %s' % s['master_ssh_port']) print('LDAP URI: %s' % s['ldap_uri']) for url in s['web_urls']: print('Web URL: %s' % url) @Command('shellinit', category='hgmo', description='Print shell commands to export variables') def shellinit(self): print('export SSH_CID=%s' % self.c.master_id) print('export SSH_SERVER=%s' % self.c.master_ssh_hostname) print('export SSH_PORT=%d' % self.c.master_ssh_port) # Don't export the full value because spaces. print('export SSH_HOST_KEY=%s' % self.c.master_host_key.split()[1]) for i, url in enumerate(self.c.web_urls): print('export HGWEB_%d_URL=%s' % (i, url)) for i, cid in enumerate(self.c.web_ids): print('export HGWEB_%d_CID=%s' % (i, cid)) for i, hostport in enumerate(self.c.kafka_hostports): print('export KAFKA_%d_HOSTPORT=%s' % (i, hostport)) print('export ZOOKEEPER_CONNECT=%s' % self.c.zookeeper_connect) @Command('stop', category='hgmo', description='Stop a hg.mozilla.org cluster') def stop(self): self.c.stop() @Command('clean', category='hgmo', description='Clean up all references to this cluster') def clean(self): self.c.clean() @Command('create-ldap-user', category='hgmo', description='Create a new user in LDAP') @CommandArgument('email', help='Email address associated with user') @CommandArgument('username', help='System account name') @CommandArgument('uid', type=int, help='Numeric user ID to associate with user') @CommandArgument('fullname', help='Full name of the user') @CommandArgument('--key-file', help='Use or create an SSH key') @CommandArgument('--scm-level', type=int, choices=(1, 2, 3), help='Add the user to the specified SCM level groups') @CommandArgument('--no-hg-access', action='store_true', help='Do not grant Mercurial access to user') def create_ldap_user(self, email, username, uid, fullname, key_file=None, scm_level=None, no_hg_access=False): self.c.ldap.create_user(email, username, uid, fullname, key_filename=key_file, scm_level=scm_level, hg_access=not no_hg_access) @Command('add-ssh-key', category='hgmo', description='Add an SSH public key to a user') @CommandArgument('email', help='Email address of user to modify') @CommandArgument('key', help='SSH public key string') def add_ssh_key(self, email, key): if key == '-': key = sys.stdin.read().strip().encode('utf-8') self.c.ldap.add_ssh_key(email, key) @Command('create-repo', category='hgmo', description='Create a repository in the cluster') @CommandArgument('name', help='Name of repository to create') @CommandArgument('level', type=int, choices=[1, 2, 3], default=1, help='SCM level access for this repository') def create_repo(self, name, level): out = self.c.create_repo(name, level=level) if out: sys.stdout.write(out) @Command('aggregate-code-coverage', category='hgmo', description='Aggregate code coverage results to a directory') @CommandArgument('destdir', help='Directory where to save code coverage files') def aggregate_code_coverage(self, destdir): self.c.aggregate_code_coverage(destdir) @Command('exec', category='hgmo', description='Execute a command in a Docker container') @CommandArgument('--detach', action='store_true', help='Do not wait for process to finish') @CommandArgument('name', help='Name of container to execute inside') @CommandArgument('command', help='Command to execute', nargs=argparse.REMAINDER) def execute(self, name, command, detach=False): if name == 'hgssh': cid = self.c.master_id elif name == 'pulse': cid = self.c.pulse_id elif name.startswith('hgweb'): i = int(name[5:]) cid = self.c.web_ids[i] else: print('invalid name. must be "hgssh" or "hgwebN"') return 1 args = '' if 'TESTTMP' in os.environ else '-it' if detach: args += ' -d ' args = args.strip() return subprocess.call('docker exec %s %s %s' % ( args, cid, ' '.join(command)), shell=True)