def configure_jupyter_notebook(): conf = hookenv.config() jupyter_dir = '/opt/jupyter' port = conf['open-port'] # Get or create and get password kv_store = unitdata.kv() password = kv_store.get('password') if not password: password = generate_password() kv_store.set('password', password) # Convert to string because some functions can't handle kv object type. password = str(password) password_hash = generate_hash(password) context = { 'port': port, 'password_hash': password_hash, } if data_changed('jupyter-conf', context): # Create config directory and render config file host.mkdir(jupyter_dir) templating.render( source='jupyter_notebook_config.py.jinja2', target=jupyter_dir + '/jupyter_notebook_config.py', context=context ) # Generate upstart template / service file context = {} if lsb_release.get_lsb_information()['RELEASE'] == "14.04": render_api_upstart_template(context) else: render_api_systemd_template(context) restart_notebook() chownr(jupyter_dir, 'ubuntu', 'ubuntu', chowntopdir=True)
def configure_events_dir(self, mode): """ Create directory for spark event data. This directory is used by workers to store event data. It is also read by the history server when displaying event information. :param string mode: Spark execution mode to determine the dir location. """ dc = self.dist_config # Directory needs to be 777 so non-spark users can write job history # there. It needs to be g+s (HDFS is g+s by default) so all entries # are readable by spark (in the spark group). It needs to be +t so # users cannot remove files they don't own. if mode.startswith('yarn'): events_dir = 'hdfs://{}'.format(dc.path('spark_events')) utils.run_as('hdfs', 'hdfs', 'dfs', '-mkdir', '-p', events_dir) utils.run_as('hdfs', 'hdfs', 'dfs', '-chmod', '1777', events_dir) utils.run_as('hdfs', 'hdfs', 'dfs', '-chown', '-R', 'ubuntu:spark', events_dir) else: events_dir = dc.path('spark_events') events_dir.makedirs_p() events_dir.chmod(0o3777) host.chownr(events_dir, 'ubuntu', 'spark', chowntopdir=True)
def install_packages(): # Add TrilioVault repository to install required package # and add queens repo to install nova libraries if not add_user(): log("Adding dmapi user failed!") return os.system('sudo echo "{}" > ' '/etc/apt/sources.list.d/trilio-gemfury-sources.list'.format( config('triliovault-pkg-source'))) new_src = config('openstack-origin') configure_installation_source(new_src) if config('python-version') == 2: dmapi_pkg = 'dmapi' else: dmapi_pkg = 'python3-dmapi' apt_update() dmapi.install() # Placing the service file os.system('sudo cp files/trilio/tvault-datamover-api.service ' '/etc/systemd/system/') chownr('/var/log/dmapi', DMAPI_USR, DMAPI_GRP) mkdir('/var/cache/dmapi', DMAPI_USR, DMAPI_GRP, perms=493) os.system('sudo systemctl enable tvault-datamover-api') service_restart('tvault-datamover-api') application_version_set(get_new_version(dmapi_pkg)) reactive.set_state('charm.installed')
def install(sojobo): api_dir = list(sojobo.connection())[0]['api-dir'] copyfile('{}/files/controller_aws.py'.format(charm_dir()), '{}/controllers/controller_aws.py'.format(api_dir)) chownr(api_dir, 'sojobo', 'www-data', chowntopdir=True) service_restart('nginx') status_set('active', 'data copied') set_state('controller-aws.installed')
def setup_images_folder(): status_set("maintenance", "setting up VM images folder") mkdir('/opt/VNF', owner='ubuntu', group='ubuntu', perms=0o775, force=False) symlink('/var/lib/libvirt/images', '/opt/VNF/images') chownr('/opt/VNF', owner='ubuntu', group='ubuntu', follow_links=False, chowntopdir=True) chownr('/var/lib/libvirt/images', owner='root', group='ubuntu', follow_links=False, chowntopdir=True) chmod('/var/lib/libvirt/images', 0o775)
def configured_devpi(): status.maintenance('Configuring devpi') DEVPI_PATH.mkdir(mode=0o755, parents=True, exist_ok=True) devpi_server_bin = DEVPI_ENV_BIN / 'devpi-server' # initialize devpi adduser('devpi') chownr(str(DEVPI_PATH), 'devpi', 'devpi', chowntopdir=True) check_call([ 'sudo', '-u', 'devpi', str(devpi_server_bin), '--init', '--serverdir', str(DEVPI_PATH) ]) # render service render('devpi.service', '/etc/systemd/system/devpi.service', context={ 'devpi_server_bin': devpi_server_bin, 'devpi_path': str(DEVPI_PATH) }) open_port(3141) # enable service check_call(['systemctl', 'enable', 'devpi.service']) # start service check_call(['systemctl', 'start', 'devpi.service']) status.active('devpi running') set_flag('devpi.configured')
def install(): conf = hookenv.config() context = get_install_context() gogs_bdist = hookenv.resource_get('bdist') check_call(["tar", "xzf", gogs_bdist], cwd="/opt") # Create gogs user & group add_group(context['group']) adduser(context['user'], system_user=True) for dir in ('.ssh', 'repositories', 'data', 'logs'): os.makedirs( os.path.join(context['home'], dir), mode=0o700, exist_ok=True) os.makedirs(os.path.join(context['home'], 'custom', 'conf'), mode=0o755, exist_ok=True) chownr(context['home'], context['user'], context['group'], True, True) render(source='upstart', target="/etc/init/gogs.conf", perms=0o644, context=context) render(source='gogs.service', target="/lib/systemd/system/gogs.service", perms=0o644, context=context) hookenv.status_set('maintenance', 'installation complete')
def install_review_queue(): status_set('maintenance', 'Installing Review Queue') with tempfile.TemporaryDirectory() as tmp_dir: install_dir = install_remote(config['repo'], dest=tmp_dir) contents = os.listdir(install_dir) if install_dir == tmp_dir and len(contents) == 1: # unlike the git handler, the archive handler just returns tmp_dir # even if the archive contents are nested in a folder as they # should be, so we have to normalize for that here install_dir = os.path.join(install_dir, contents[0]) shutil.rmtree(APP_DIR, ignore_errors=True) log('Moving app source from {} to {}'.format( install_dir, APP_DIR)) shutil.move(install_dir, APP_DIR) subprocess.check_call('make .venv'.split(), cwd=APP_DIR) if init_is_systemd(): shutil.copyfile(SYSTEMD_SRC, SYSTEMD_DEST) shutil.copyfile(SYSTEMD_TASK_SRC, SYSTEMD_TASK_DEST) subprocess.check_call(['systemctl', 'daemon-reload']) else: shutil.copyfile(UPSTART_SRC, UPSTART_DEST) shutil.copyfile(UPSTART_TASK_SRC, UPSTART_TASK_DEST) subprocess.check_call(['initctl', 'reload-configuration']) shutil.copyfile(LP_CREDS_SRC, LP_CREDS_DEST) shutil.copyfile(APP_INI_SRC, APP_INI_DEST) chownr(APP_DIR, APP_USER, APP_GROUP) set_state('reviewqueue.installed') change_config() update_db() update_amqp() update_secret() set_state('reviewqueue.restart')
def prepare_env(): saved_path = os.getcwd() os.chdir('{}'.format(CHARM_LIB_DIR + "oracle_neutron")) cmd = "useradd --home-dir /var/lib/neutron --create-home \ --system --shell /bin/false neutron" os.system(cmd) mkdir("/var/log/neutron") mkdir("/etc/neutron") mkdir("/etc/neutron/rootwrap.d") mkdir("/etc/neutron/plugins/ml2") chownr("/var/log/neutron", 'neutron', 'neutron') chownr("/var/lib/neutron", 'neutron', 'neutron') chownr("/etc/neutron", 'neutron', 'neutron') chownr("/etc/neutron", 'neutron', 'neutron') chownr("/etc/neutron/plugins", 'neutron', 'neutron') os.chmod("/var/log/neutron", 0o766) os.system(cmd) cmd = "./tools/generate_config_file_samples.sh" os.system(cmd) cmd = "cp etc/api-paste.ini /etc/neutron/api-paste.ini" os.system(cmd) cmd = "cp etc/policy.json /etc/neutron/policy.json" os.system(cmd) cmd = "cp etc/rootwrap.conf /etc/rootwrap.conf" os.system(cmd) cmd = "cp -R etc/neutron/rootwrap.d/* /etc/neutron/rootwrap.d/" os.system(cmd) cmd = "cp etc/rootwrap.conf /etc/neutron/rootwrap.conf" os.system(cmd) os.chdir(saved_path)
def _install_mattermost(): # Backup existing installation if it exists backup_path = None if os.path.isdir('/opt/mattermost'): backup_path = "/opt/mattermost.back{}".format(str(datetime.datetime.now())) shutil.move('/opt/mattermost', backup_path) # Create mattermost user & group if not exists if not group_exists('mattermost'): add_group("mattermost") if not user_exists('mattermost'): adduser("mattermost", system_user=True) # Get and uppack resource mattermost_bdist = resource_get('bdist') extract_tarfile(mattermost_bdist, destpath="/opt") # Render systemd template render(source="mattermost.service.tmpl", target="/etc/systemd/system/mattermost.service", perms=0o644, owner="root", context={}) check_call(['systemctl', 'daemon-reload']) if backup_path: shutil.move( '{}/config/config.json'.format(backup_path), '/opt/mattermost/config/config.json') shutil.move( '{}/data'.format(backup_path), '/opt/mattermost/') # Create dirs that don't exist yet for folder in ("data", "logs", "config"): os.makedirs("/opt/mattermost/{}".format(folder), mode=0o700, exist_ok=True) chownr("/opt/mattermost", "mattermost", "mattermost", chowntopdir=True)
def install_packages(): # Add TrilioVault repository to install required package # and add queens repo to install nova libraries if not validate_ip(config('triliovault-ip')): log("Invalid IP address !") status_set('blocked', 'Invalid IP address, please provide correct IP address') return if not add_user(): log("Adding dmapi user failed!") return add_source('deb http://{}:8085 deb-repo/'.format(config('triliovault-ip'))) os.system('sudo add-apt-repository cloud-archive:queens') apt_update() dmapi.install() apt_install(['dmapi'], options=['--allow-unauthenticated'], fatal=True) # Placing the service file os.system('sudo cp files/trilio/tvault-datamover-api.service ' '/etc/systemd/system/') chownr('/var/log/dmapi', DMAPI_USR, DMAPI_GRP) os.system('sudo systemctl enable tvault-datamover-api') service_restart('tvault-datamover-api') reactive.set_state('charm.installed')
def install(): """ Install Hook """ log('ftb-infinity: install') status_set('maintenance', 'installing FTB modpack') # Add user adduser(FTB_USER) mkdir(FTB_HOME, owner=FTB_USER, group=FTB_USER, perms=0o750) check_call(['usermod', '-s', '/bin/bash', '-d', FTB_HOME, FTB_USER]) # Download ftb ArchiveUrlFetchHandler().install(FTB_DL_URL, FTB_HOME) # Sanitize permissions, zip! chownr(FTB_HOME, FTB_USER, FTB_USER) path = os.path.join(FTB_HOME, 'FTBInstall.sh') s = os.stat(path) os.chmod(path, s.st_mode | stat.S_IXUSR | stat.S_IXGRP) # Accept EULA sed(os.path.join(FTB_HOME, 'eula.txt'), 'eula=false', 'eula=true') # Download minecraft jars with chdir(FTB_HOME): check_call(['sudo', '-u', FTB_USER, '-H', os.path.join(FTB_HOME, 'FTBInstall.sh')]) # Render server.properties ftb_config_server() # Deploy systemd service ftb_systemd_install() set_state(CHARM_STATE_AVAILABLE) status_set('waiting', 'ftb downloaded')
def install_crowd(): host.adduser('crowd') fetch.install_remote( source=CROWD_URL.format(hookenv.config('crowd-version')), # version dest=CROWD_INSTALL, # checksum=None, # hash_type='sha1' ) host.lchownr( CROWD_INSTALL, owner='crowd', group='crowd', ) for dir in [ '{}/atlassian-crowd-{}'.format(CROWD_INSTALL, hookenv.config('crowd-version')), '/var/crowd-home', '/var/crowd-home/shared/', ]: try: mkdir(dir) except: pass host.chownr( dir, owner='crowd', group='crowd', chowntopdir=True, ) set_flag('crowd.installed')
def setup_images_folder(): status_set("maintenance", "Setting up VM images folder") mkdir('/opt/VNF', owner='openvim', group='openvim', perms=0o775, force=False) symlink('/var/lib/libvirt/images', '/opt/VNF/images') chownr('/opt/VNF', owner='openvim', group='openvim', follow_links=False, chowntopdir=True) chownr('/var/lib/libvirt/images', owner='root', group='openvim', follow_links=False, chowntopdir=True) chmod('/var/lib/libvirt/images', 0o775)
def install_layer_openmano(): status_set('maintenance', 'Installing') cfg = config() # TODO change user home # XXX security issue! host.adduser(USER, password=USER) # TODO check out a branch dest_dir = install_remote( cfg['source'], dest=INSTALL_PATH, depth='1', branch='master', ) os.mkdir(os.path.join(dest_dir, 'logs')) host.chownr(dest_dir, USER, USER) kvdb.set('repo', dest_dir) os.mkdir('/home/{}/bin'.format(USER)) os.symlink( "{}/openmano".format(dest_dir), "/home/{}/bin/openmano".format(USER)) os.symlink( "{}/scripts/openmano-report.sh".format(dest_dir), "/home/{}/bin/openmano-report.sh".format(USER)) os.symlink( "{}/scripts/service-openmano.sh".format(dest_dir), "/home/{}/bin/service-openmano".format(USER)) open_port(9090) set_state('openmano.installed')
def install(): hookenv.status_set('maintenance', 'Installing CABS') fetch.apt_update() fetch.apt_install(fetch.filter_installed_packages([ 'graphite-carbon', 'graphite-web', 'apache2', 'apache2-mpm-worker', 'libapache2-mod-wsgi', 'postgresql', 'python-virtualenv', 'python-dev', 'python-requests', ])) touch('/etc/apache2/sites-available/cabs-graphite.conf') shutil.copyfile('files/graphite.conf', '/etc/apache2/sites-available/cabs-graphite.conf') shutil.copyfile('files/graphite-carbon', '/etc/default/graphite-carbon') apache2.enable_site('cabs-graphite') host.chownr('/var/lib/graphite', '_graphite', '_graphite') subprocess.check_call('sudo -u _graphite graphite-manage syncdb --noinput', shell=True) extract_tar('payload/collector-web.tar.gz', '/opt/collector-web') config = hookenv.config() try: env = None if config.get('proxy'): env = dict(os.environ) env.update({'http_proxy': config.get('proxy'), 'https_proxy': config.get('proxy')}) subprocess.check_call(['make', '.venv'], cwd='/opt/collector-web', env=env) except subprocess.CalledProcessError as e: logging.exception(e) hookenv.status_set( 'blocked', 'Failed to create venv - do you require a proxy?') return # setup postgres for collector-web subprocess.check_call(['scripts/ensure_db_user.sh']) subprocess.check_call(['scripts/ensure_db.sh']) # Install upstart config for collector-web shutil.copyfile('/opt/collector-web/conf/upstart/collectorweb.conf', '/etc/init/collectorweb.conf') host.chownr('/opt/collector-web', 'ubuntu', 'ubuntu') host.service_restart('apache2') host.service_restart('carbon-cache') host.service_restart('collectorweb') # Install cron, vhost for gui, etc hookenv.open_port(9000) hookenv.open_port(9001) hookenv.open_port(2003)
def setup_repository(): config = hookenv.config() if config['git-repo'] is not "": pull_repository() else: create_repository() chownr('/srv', owner='ubuntu', group='ubuntu') status_set('active', '')
def init_zkrest(self): # Zookeeper user needs to compile the rest contrib server. # So zookeeper needs to: # 1. Have a home dir for ant cache to exist # 2. Write to the /usr/lib/zookeeper chownr(self.dist_config.path('zookeeper'), 'zookeeper', 'zookeeper', chowntopdir=True) with chdir(self.dist_config.path('zookeeper')): utils.run_as('zookeeper', 'ant') unitdata.kv().set('rest.initialised', True)
def create_certs_dir(): if not ES_CERTS_DIR.exists(): ES_CERTS_DIR.mkdir() chownr(path=str(ES_CERTS_DIR), owner='elasticsearch', group='elasticsearch', follow_links=True, chowntopdir=True) set_flag('cert.dir.available')
def provision_certs_all_nodes(): certs = charms.leadership.leader_get('elasticsearch_certs') ES_CERTS.write_bytes(b64decode(certs)) chownr(path=str(ES_CERTS), owner='elasticsearch', group='elasticsearch', follow_links=True, chowntopdir=True) set_flag('elasticsearch.certs.provisioned')
def check_dir(self, path): """Check if dir has been provided and is configured correctly for usage.""" if path: if not os.path.isdir(path): os.mkdir(path) chownr(path=path, owner=self.opts.get("owner"), group=self.opts.get("group")) return False
def install_tengu(): if not os.path.isdir('/home/{}'.format(USER)): os.mkdir('/home/{}'.format(USER)) adduser(USER) chownr('/home/{}'.format(USER), USER, USER, chowntopdir=True) if not os.path.isdir(API_DIR): os.mkdir(API_DIR) mergecopytree('files/tengu_ui', API_DIR) chownr(API_DIR, USER, GROUP, chowntopdir=True)
def create_conf_dir(): """Ensure config dir exists """ status_set('maintenance', "Creating application directories") for directory in [SU_CONF_DIR, LOG_DIR]: if not os.path.isdir(directory): os.makedirs(directory, mode=0o755, exist_ok=True) chownr(directory, owner='www-data', group='www-data') status_set('active', "Application directories created") set_flag('conf.dirs.available')
def install(self): jujuresources.install(self.resources['hue'], destination=self.dist_config.path('hue'), skip_top_level=True) self.dist_config.add_users() self.dist_config.add_dirs() self.dist_config.add_packages() chownr(self.dist_config.path('hue'), 'hue', 'hadoop') unitdata.kv().set('hue.installed', True)
def download_openvim(): status_set("maintenance", "Downloading OpenVIM") if os.path.isdir("/opt/openmano"): rmtree("/opt/openmano") gitrepo.clone_from('https://github.com/tvansteenburgh/openmano.git', '/opt/openmano') chownr('/opt/openmano', owner='openvim', group='openvim', follow_links=False, chowntopdir=True)
def install(self, force=False): if not force and self.is_installed(): return jujuresources.install(self.resources['hue'], destination=self.dist_config.path('hue'), skip_top_level=True) self.dist_config.add_users() self.dist_config.add_dirs() self.dist_config.add_packages() chownr(self.dist_config.path('hue'), 'hue', 'hadoop') unitdata.kv().set('hue.installed', True)
def install(sojobo): api_dir = list(sojobo.connection())[0]['api-dir'] copyfile('{}/files/controller_azure.py'.format(charm_dir()), '{}/controllers/controller_azure.py'.format(api_dir)) copyfile('{}/files/bootstrap_azure_controller.py'.format(charm_dir()), '{}/scripts/bootstrap_azure_controller.py'.format(api_dir)) copyfile('{}/files/add_azure_credential.py'.format(charm_dir()), '{}/scripts/add_azure_credential.py'.format(api_dir)) chownr(api_dir, 'sojobo', 'www-data', chowntopdir=True) service_restart('nginx') status_set('active', 'data copied') set_flag('controller-azure.installed')
def configure_installdir(self): ''' Create and fix permissions on install dir''' if not os.path.exists(self.installdir): os.makedirs(self.installdir) hookenv.log("Fixing data dir permissions: {}".format( self.installdir), 'DEBUG') if host.user_exists(self.user) and host.group_exists(self.user): host.chownr(self.installdir, self.user, self.user, chowntopdir=True) else: hookenv.log("Skipping chown because user/group {} is missing".format( self.installdir), 'DEBUG')
def run_smoke_tests(self, smoke_components=None, smoke_env=None): """ Run the Bigtop smoke tests for given components using the gradle wrapper script. :param list smoke_components: Bigtop components to smoke test :param list smoke_env: Dict of required environment variables (merged with /etc/environment) """ if not is_state('bigtop.available'): hookenv.log('Bigtop is not ready to run smoke tests') return None if not smoke_components: hookenv.log('Missing Bigtop smoke test component list') return None # We always need TERM and JAVA_HOME; merge with any user provided dict subprocess_env = {'TERM': 'dumb', 'JAVA_HOME': java_home()} if isinstance(smoke_env, dict): subprocess_env.update(smoke_env) # Ensure the base dir is owned by ubuntu so we can create a .gradle dir. chownr(self.bigtop_base, 'ubuntu', 'ubuntu', chowntopdir=True) # Gradle doesn't honor env proxies; check for either http* or HTTP* and # set cli args as needed. http_url = os.environ.get('http_proxy', os.environ.get('HTTP_PROXY')) https_url = os.environ.get('https_proxy', os.environ.get('HTTPS_PROXY')) proxy_args = [] if http_url: parsed_url = urlparse(http_url) proxy_args += ['-Dhttp.proxyHost={}'.format(parsed_url.hostname), '-Dhttp.proxyPort={}'.format(parsed_url.port)] if https_url: parsed_url = urlparse(https_url) proxy_args += ['-Dhttps.proxyHost={}'.format(parsed_url.hostname), '-Dhttps.proxyPort={}'.format(parsed_url.port)] # Bigtop can run multiple smoke tests at once; construct the right args. comp_args = ['bigtop-tests:smoke-tests:%s:test' % c for c in smoke_components] gradlew_args = ['-Psmoke.tests', '--info'] + proxy_args + comp_args hookenv.log('Bigtop smoke test environment: {}'.format(subprocess_env)) hookenv.log('Bigtop smoke test args: {}'.format(gradlew_args)) with chdir(self.bigtop_base): try: utils.run_as('ubuntu', './gradlew', *gradlew_args, env=subprocess_env) smoke_out = 'success' except subprocess.CalledProcessError as e: smoke_out = e.output return smoke_out
def _copy_resources(self): self._ensure_folders() for f in self.FILE_MAP: if os.path.isfile(f) is False: raise ValueError( "Missing required file %s. Package not installed?" % f) shutil.copy(f, os.path.join(self.TFTP_ROOT, self.FILE_MAP[f]), follow_symlinks=True) ch_host.chownr(self.TFTP_ROOT, _IRONIC_USER, _IRONIC_GROUP, chowntopdir=True)
def generate_ssh_key(user): if ssh_priv_key(user).exists(): return sshdir = ssh_key_dir(user) if not sshdir.exists(): host.mkdir(sshdir, owner=user, group='hadoop', perms=0o755) keyfile = ssh_priv_key(user) (sshdir / 'config').write_lines([ 'Host *', ' StrictHostKeyChecking no' ], append=True) check_call(['ssh-keygen', '-t', 'rsa', '-P', '', '-f', keyfile]) host.chownr(sshdir, user, 'hadoop')
def install_solr(): version = config('solr_version') adduser('solr') au.download( "http://archive.apache.org/dist/lucene/solr/{0}/solr-{0}.tgz".format( version), "/tmp/solr.tgz") mkdir('/opt/solr') check_output([ 'tar', 'xvfz', "/tmp/solr.tgz", '-C', "/opt/solr", '--strip-components=1' ]) chownr('/opt/solr', 'solr', 'solr', chowntopdir=True) set_state('solr.installed')
def fix_perms(data_dir): # The path to data_dir must be world readable, so the postgres user # can traverse to it. p = data_dir while p != '/': p = os.path.dirname(p) subprocess.check_call(['chmod', 'a+rX', p], universal_newlines=True) # data_dir and all of its contents should be owned by the postgres # user and group. host.chownr(data_dir, 'postgres', 'postgres', follow_links=False) # data_dir should not be world readable. os.chmod(data_dir, 0o700)
def render_ini(pgsql): db_uri = "postgresql://{}:{}@{}:{}/{}".format( pgsql.user(), pgsql.password(), pgsql.host(), pgsql.port(), pgsql.database() ) ini = "" with open(APP_INI_SRC, "r") as f: ini = f.read() ini = re.sub(r"(sqlalchemy.url\s*=)(.*)", r"\1 " + db_uri, ini) ini = re.sub(r"(port\s*=)(.*)", r"\1 " + str(config["port"]), ini) with open(APP_INI_DEST, "w") as f: f.write(ini) chownr(APP_INI_DEST, APP_USER, APP_GROUP)
def install_whelp(): ''' Reactive hook to install whelp ''' hookenv.status_set('maintenance', 'Installing whelp') whelp = Whelp() whelp_tar = hookenv.resource_get('webapp') hookenv.status_set('maintenance', 'installing webapp') # Extract tar resource tar = tarfile.open(whelp_tar) tar.extractall(WHELP_HOME) # Install pip3 reqs with chdir('/srv/whelp/whelp'): with open('requirements.txt', 'r') as f: for i in list(map(lambda b: b.strip('\n'), f.readlines())): pip_install(i) # Set permissions chownr(WHELP_HOME, 'www-data', 'www-data') # Get state files for whelp to run whelp.get_whelp_bucket_files() # Configure NGINX configure_site('whelp', 'whelp.vhost', port=config['port'], whelp_port=config['whelp-port']) # Start Supervisor subprocess.call('supervisord -c /etc/supervisor/supervisord.conf'.split(), shell=False) # Render whelp supervisor.conf whelp.render_whelp_supervisor_conf() # Open port 80 hookenv.open_port(config['port']) # Set status to active hookenv.status_set('active', 'Whelp is active on port %s' % config['port']) # Set whelp.available state set_state('whelp.installed')
def perms(): for p in paths: makedirs(p[0], exist_ok=True) try: getgrnam(p[2]) except KeyError: add_group(p[2], system_group=True) if not user_exists(p[1]): adduser(p[1], shell='/bin/false', system_user=True, primary_group=p[2]) # Ensure path is owned appropriately chownr(path=p[0], owner=p[1], group=p[2], chowntopdir=True)
def get_ssh_key(user): sshdir = Path("/home/%s/.ssh" % user) if not sshdir.exists(): host.mkdir(sshdir, owner=user, group="hadoop", perms=0o755) keyfile = sshdir / "id_rsa" pubfile = sshdir / "id_rsa.pub" authfile = sshdir / "authorized_keys" if not pubfile.exists(): (sshdir / "config").write_lines(["Host *", " StrictHostKeyChecking no"], append=True) check_call(["ssh-keygen", "-t", "rsa", "-P", "", "-f", keyfile]) host.chownr(sshdir, user, "hadoop") # allow ssh'ing to localhost; useful for things like start_dfs.sh if not authfile.exists(): Path.copy(pubfile, authfile) return pubfile.text()
def prepare_es_data_dir(): ''' Create (if not exists) and set perms on elasticsearch data dir. ''' if not ES_DATA_DIR.exists(): ES_DATA_DIR.mkdir(parents=True, exist_ok=True) chownr(path=str(ES_DATA_DIR), owner='elasticsearch', group='elasticsearch', follow_links=True, chowntopdir=True) set_flag('elasticsearch.storage.dir.prepared')
def set_indexers(self, status): '''Enable or disable all indexer searching based on provided status status: True will turn on indexers status: False will turn off indexers''' conn = sqlite3.connect(self.database_file) c = conn.cursor() if status: c.execute( '''UPDATE Indexers SET EnableRss = 1, EnableSearch = 1''') else: c.execute( '''UPDATE Indexers SET EnableRss = 0, EnableSearch = 0''') conn.commit() host.chownr(self.home_dir, owner=self.charm_config['radarr-user'], group=self.charm_config['radarr-user'])
def check_for_and_mount_direct_attached_storage(): direct_attached_device = Path('/dev/nvme0n1') if direct_attached_device.exists(): sp.call(['mkfs.ext4', str(direct_attached_device)]) sp.call(['mount', str(direct_attached_device), str(ES_DATA_DIR)]) chownr(path=str(ES_DATA_DIR), owner='elasticsearch', group='elasticsearch', follow_links=True, chowntopdir=True) with open('/etc/fstab', 'a') as f: f.write((f'/dev/nvme0n1 {str(ES_DATA_DIR)} ' 'ext4 defaults,nofail 0 2')) set_flag('direct.attached.storage.check.complete')
def osdize_dir(path): if os.path.exists(os.path.join(path, 'upstart')): log('Path {} is already configured as an OSD - bailing'.format(path)) return if cmp_pkgrevno('ceph', "0.56.6") < 0: log('Unable to use directories for OSDs with ceph < 0.56.6', level=ERROR) raise mkdir(path, owner=ceph_user(), group=ceph_user(), perms=0o755) chownr('/var/lib/ceph', ceph_user(), ceph_user()) cmd = [ 'sudo', '-u', ceph_user(), 'ceph-disk', 'prepare', '--data-dir', path ] subprocess.check_call(cmd)
def install(): if db.get("repo") != config["repo"]: status_set("maintenance", "Installing app") apt_install(APT_PKGS) tmp_dir = install_remote(config["repo"], dest="/tmp", depth=1) shutil.rmtree(APP_DIR, ignore_errors=True) log("Moving app source from {} to {}".format(tmp_dir, APP_DIR)) shutil.move(tmp_dir, APP_DIR) subprocess.check_call("make .venv".split(), cwd=APP_DIR) shutil.copyfile(UPSTART_SRC, UPSTART_DEST) chownr(APP_DIR, APP_USER, APP_GROUP) db.set("repo", config["repo"]) if config.changed("port"): open_port(config["port"]) if config.previous("port"): close_port(config.previous("port"))
def get_ssh_key(user): sshdir = Path('/home/%s/.ssh' % user) if not sshdir.exists(): host.mkdir(sshdir, owner=user, group='hadoop', perms=0o755) keyfile = sshdir / 'id_rsa' pubfile = sshdir / 'id_rsa.pub' authfile = sshdir / 'authorized_keys' if not pubfile.exists(): (sshdir / 'config').write_lines([ 'Host *', ' StrictHostKeyChecking no' ], append=True) check_call(['ssh-keygen', '-t', 'rsa', '-P', '', '-f', keyfile]) host.chownr(sshdir, user, 'hadoop') # allow ssh'ing to localhost; useful for things like start_dfs.sh if not authfile.exists(): Path.copy(pubfile, authfile) return pubfile.text()
def bootstrap_monitor_cluster(secret): hostname = get_unit_hostname() path = '/var/lib/ceph/mon/ceph-{}'.format(hostname) done = '{}/done'.format(path) if systemd(): init_marker = '{}/systemd'.format(path) else: init_marker = '{}/upstart'.format(path) keyring = '/var/lib/ceph/tmp/{}.mon.keyring'.format(hostname) if os.path.exists(done): log('bootstrap_monitor_cluster: mon already initialized.') else: # Ceph >= 0.61.3 needs this for ceph-mon fs creation mkdir('/var/run/ceph', owner=ceph_user(), group=ceph_user(), perms=0o755) mkdir(path, owner=ceph_user(), group=ceph_user()) mkdir("/var/lib/ceph/tmp", owner=ceph_user(), group=ceph_user()) # end changes for Ceph >= 0.61.3 try: subprocess.check_call(['ceph-authtool', keyring, '--create-keyring', '--name=mon.', '--add-key={}'.format(secret), '--cap', 'mon', 'allow *']) subprocess.check_call(['ceph-mon', '--mkfs', '-i', hostname, '--keyring', keyring]) chownr(path, ceph_user(), ceph_user()) with open(done, 'w'): pass with open(init_marker, 'w'): pass if systemd(): subprocess.check_call(['systemctl', 'enable', 'ceph-mon']) service_restart('ceph-mon') else: service_restart('ceph-mon-all') except: raise finally: os.unlink(keyring)
def install_resource(): remove_state('charm-svg.ready') hookenv.status_set('maintenance', 'extracting resources') svg_bin = hookenv.resource_get('python-jujusvg') web_tar = hookenv.resource_get('webapp') hookenv.status_set('maintenance', 'installing python-jujusvg') shutil.copy(svg_bin, charmsvg.JUJUSVG_PATH) os.chmod(charmsvg.JUJUSVG_PATH, 0o755) hookenv.status_set('maintenance', 'installing webapp') tar = tarfile.open(web_tar) tar.extractall(charmsvg.INSTALL_PATH) chownr(charmsvg.INSTALL_PATH, 'www-data', 'www-data') set_state('charm-svg.installed')
def install(): status_set('maintenance', 'Installing PDI') adduser('etl') mkdir('/home/etl') chownr('/home/etl', 'etl', 'etl', chowntopdir=True) os.chmod('/home/etl', 0o755) #au = ArchiveUrlFetchHandler() #au.install(hookenv.config()['pdi_url'], '/opt/') pdiarchive = hookenv.resource_get('pdi-archive') tar = tarfile.open(pdiarchive) tar.extractall("/opt/") chownr('/opt/data-integration', 'etl', 'etl', chowntopdir=True) st = os.stat('/opt/data-integration/spoon.sh') os.chmod('/opt/data-integration/spoon.sh', st.st_mode | stat.S_IEXEC) os.chmod('/opt/data-integration/carte.sh', st.st_mode | stat.S_IEXEC) os.chmod('/opt/data-integration/encr.sh', st.st_mode | stat.S_IEXEC) os.chmod('/opt/data-integration/kitchen.sh', st.st_mode | stat.S_IEXEC) os.chmod('/opt/data-integration/pan.sh', st.st_mode | stat.S_IEXEC) status_set('maintenance', 'PDI Installed') set_state('pdi.installed')
def prepare_env(): saved_path = os.getcwd() os.chdir('{}'.format(CHARM_LIB_DIR + "oracle_keystone")) cmd = "useradd --home-dir /var/lib/keystone --create-home \ --system --shell /bin/false keystone" os.system(cmd) mkdir("/var/log/keystone") mkdir("/etc/keystone") chownr("/var/log/keystone", 'keystone', 'keystone') chownr("/var/lib/keystone", 'keystone', 'keystone') chownr("/etc/keystone", 'keystone', 'keystone') cmd = "cp ./etc/keystone.conf.sample /etc/keystone/keystone.conf" os.system(cmd) cmd = "cp ./etc/keystone-paste.ini /etc/keystone/keystone-paste.ini" os.system(cmd) cmd = "cp ./etc/default_catalog.templates /etc/keystone/\ default_catalog.templates" os.system(cmd) cmd = "cp ./etc/logging.conf.sample /etc/keystone/logging.conf" os.system(cmd) cmd = "cp ./etc/policy.v3cloudsample.json /etc/keystone/policy.json" os.system(cmd) cmd = "cp ./etc/sso_callback_template.html /etc/keystone/\ sso_callback_template.html" os.system(cmd) cmd = "cp ./httpd/wsgi-keystone.conf /etc/apache2/sites-available/\ keystone.conf" os.system(cmd) cmd = "cp ./httpd/wsgi-keystone.conf /etc/apache2/sites-enabled/\ keystone.conf" os.system(cmd) os.chdir(saved_path)
def upgrade_monitor(): current_version = get_version() status_set("maintenance", "Upgrading monitor") log("Current ceph version is {}".format(current_version)) new_version = config('release-version') log("Upgrading to: {}".format(new_version)) try: add_source(config('source'), config('key')) apt_update(fatal=True) except subprocess.CalledProcessError as err: log("Adding the ceph source failed with message: {}".format( err.message)) status_set("blocked", "Upgrade to {} failed".format(new_version)) sys.exit(1) try: if systemd(): for mon_id in get_local_mon_ids(): service_stop('ceph-mon@{}'.format(mon_id)) else: service_stop('ceph-mon-all') apt_install(packages=PACKAGES, fatal=True) # Ensure the ownership of Ceph's directories is correct chownr(path=os.path.join(os.sep, "var", "lib", "ceph"), owner=ceph_user(), group=ceph_user()) if systemd(): for mon_id in get_local_mon_ids(): service_start('ceph-mon@{}'.format(mon_id)) else: service_start('ceph-mon-all') status_set("active", "") except subprocess.CalledProcessError as err: log("Stopping ceph and upgrading packages failed " "with message: {}".format(err.message)) status_set("blocked", "Upgrade to {} failed".format(new_version)) sys.exit(1)
def configure_queue_processors(): if os.path.isdir(CONSUMER_CONFIG_ROOT) is False: host.mkdir(CONSUMER_CONFIG_ROOT, REDDIT_USER, REDDIT_GROUP, 0755) def set_consumer_count(queue, count): queueFile = '%s/%s' % (CONSUMER_CONFIG_ROOT, queue) if os.path.isfile(queueFile) is False: f = open(queueFile, 'w') f.write('%s' % count) f.close() set_consumer_count('log_q', 0) set_consumer_count('cloudsearch_q', 0) set_consumer_count('scraper_q', 1) set_consumer_count('scraper_q', 1) set_consumer_count('commentstree_q', 1) set_consumer_count('newcomments_q', 1) set_consumer_count('vote_link_q', 1) set_consumer_count('vote_comment_q', 1) host.chownr(CONSUMER_CONFIG_ROOT, REDDIT_USER, REDDIT_GROUP) return
def download_openvim(): status_set("maintenance", "downloading openvim") if os.path.isdir("/opt/openmano"): rmtree("/opt/openmano") gitrepo.clone_from('https://github.com/wwwtyro/openmano.git', '/opt/openmano') chownr('/opt/openmano', owner='openvim', group='openvim', follow_links=False, chowntopdir=True)
def configure_sparkbench(self): """ Install/configure/remove Spark-Bench based on user config. If config[spark_bench_enabled], fetch, install, and configure Spark-Bench on initial invocation. Subsequent invocations will skip the fetch/install, but will reconfigure Spark-Bench since we may need to adjust the data dir (eg: benchmark data is stored in hdfs when spark is in yarn mode; locally in all other execution modes). """ install_sb = hookenv.config()['spark_bench_enabled'] sb_dir = '/home/ubuntu/SparkBench' if install_sb: # Fetch/install on our first go-round, then set unit data so we # don't reinstall every time this function is called. if not unitdata.kv().get('spark_bench.installed', False): sb_url = hookenv.config()['spark_bench_url'] Path(sb_dir).rmtree_p() au = ArchiveUrlFetchHandler() au.install(sb_url, '/home/ubuntu') # NB: This block is unused when using one of our sb tgzs. It # may come in handy if people want a tgz that does not expand # to our expected sb_dir. # ##### # Handle glob if we use a .tgz that doesn't expand to sb_dir # sb_archive_dir = glob('/home/ubuntu/SparkBench*')[0] # SparkBench expects to live in ~/SparkBench, so put it there # Path(sb_archive_dir).rename(sb_dir) # ##### # Ensure users in the spark group can write to any subdirectory # of sb_dir (spark needs to write benchmark output there when # running in local modes). host.chownr(Path(sb_dir), 'ubuntu', 'spark', chowntopdir=True) for r, d, f in os.walk(sb_dir): os.chmod(r, 0o2775) unitdata.kv().set('spark_bench.installed', True) unitdata.kv().flush(True) # Configure the SB env every time this function is called. sb_conf = '{}/conf'.format(sb_dir) sb_env = Path(sb_conf) / 'env.sh' if not sb_env.exists(): (Path(sb_conf) / 'env.sh.template').copy(sb_env) # NB: A few notes on configuring SparkBench: # 1. Input data has been pregenerated and packed into the tgz. All # spark cluster members will have this data locally, which enables # us to execute benchmarks in the absense of HDFS. When spark is in # yarn mode, we'll need to generate and store this data in HDFS # so nodemanagers can access it (NMs obviously won't have SB # installed locally). Set DATA_HDFS to a local dir or common HDFS # location depending on our spark execution mode. # # 2. SB tries to SSH to spark workers to purge vmem caches. This # isn't possible in containers, nor is it possible in our env # because we don't distribute ssh keys among cluster members. # Set MC_LIST to an empty string to prevent this behavior. # # 3. Throughout SB, HADOOP_HOME/bin is used as the prefix for the # hdfs command. Bigtop's hdfs lives at /usr/bin/hdfs, so set the # SB HADOOP_HOME accordingly (it's not used for anything else). # # 4. Use our MASTER envar to set the SparkBench SPARK_MASTER url. # It is updated every time we (re)configure spark. mode = hookenv.config()['spark_execution_mode'] if mode.startswith('yarn'): sb_data_dir = "hdfs:///user/ubuntu/SparkBench" else: sb_data_dir = "file://{}".format(sb_dir) utils.re_edit_in_place(sb_env, { r'^DATA_HDFS *=.*': 'DATA_HDFS="{}"'.format(sb_data_dir), r'^DATASET_DIR *=.*': 'DATASET_DIR="{}/dataset"'.format(sb_dir), r'^MC_LIST *=.*': 'MC_LIST=""', r'.*HADOOP_HOME *=.*': 'HADOOP_HOME="/usr"', r'.*SPARK_HOME *=.*': 'SPARK_HOME="/usr/lib/spark"', r'^SPARK_MASTER *=.*': 'SPARK_MASTER="$MASTER"', }) else: # config[spark_bench_enabled] is false; remove it Path(sb_dir).rmtree_p() unitdata.kv().set('spark_bench.installed', False) unitdata.kv().flush(True)
def install_ssh_key(user, ssh_key): sshdir = Path('/home/%s/.ssh' % user) if not sshdir.exists(): host.mkdir(sshdir, owner=user, group='hadoop', perms=0o755) Path(sshdir / 'authorized_keys').write_text(ssh_key, append=True) host.chownr(sshdir, user, 'hadoop')