def put_guest(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, local_is_path): """ Upload file to a guest container """ pre = self.ftp.getcwd() pre = pre if pre else "" if local_is_path and self.isdir(remote_path): basename = os.path.basename(local_path) remote_path = posixpath.join(remote_path, basename) if output.running: print( "[%s] put: %s -> %s" % (env.host_string, local_path if local_is_path else "<file obj>", posixpath.join(pre, remote_path)) ) # Have to bounce off FS if doing file-like objects fd, real_local_path = None, local_path if not local_is_path: fd, real_local_path = tempfile.mkstemp() old_pointer = local_path.tell() local_path.seek(0) file_obj = os.fdopen(fd, "wb") file_obj.write(local_path.read()) file_obj.close() local_path.seek(old_pointer) # Use temporary file with a unique name on the host machine guest_path = remote_path hasher = hashlib.sha1() hasher.update(env.host_string) hasher.update(name_or_ctid) hasher.update(guest_path) host_path = hasher.hexdigest() # Upload the file to host machine rattrs = self.ftp.put(real_local_path, host_path) # Copy file to the guest container with settings(hide("everything"), cwd=""): _orig_run_command( 'cat "%s" | vzctl exec "%s" \'cat - > "%s"\'' % (host_path, name_or_ctid, guest_path), sudo=True ) # Revert to original remote_path for return value's sake remote_path = guest_path # Clean up if not local_is_path: os.remove(real_local_path) # Handle modes if necessary if (local_is_path and mirror_local_mode) or (mode is not None): lmode = os.stat(local_path).st_mode if mirror_local_mode else mode lmode = lmode & 07777 rmode = rattrs.st_mode & 07777 if lmode != rmode: with hide("everything"): sudo('chmod %o "%s"' % (lmode, remote_path)) return remote_path
def install(): """ Install a Parental Control on OpenWrt router """ # Set Shell environment env.shell = "/bin/ash -l -c" # Install USB required packages openwrt.update_index() openwrt.install([ 'tinyproxy', ]) print ("Set configuration") with settings(hide('running', 'warnings', 'stdout')): run('mkdir -p /var/etc/tinyproxy') run('echo "youporn" >> /var/etc/tinyproxy/filter') run('touch /var/log/tinyproxy.log') run('chown nobody:nogroup /var/log/tinyproxy.log') openwrt.uci_set([ "tinyproxy.@tinyproxy[0].Allow=192.168.253.0/24", "tinyproxy.@tinyproxy[0].Filter=/var/etc/tinyproxy/filter", "tinyproxy.@tinyproxy[0].enabled=1", "tinyproxy.@tinyproxy[0].Syslog=0", ]) with settings(hide('running', 'warnings', 'stdout'), warn_only=True): run('/etc/init.d/tinyproxy enable') update()
def get_node_backup_files(self, snapshot, incremental_backups): if snapshot.keyspaces: keyspace_globs = snapshot.keyspaces.split() else: keyspace_globs = ['*'] if snapshot.table: table_glob = snapshot.table else: table_glob = '*' files = [] for keyspace_glob in keyspace_globs: path = [ self.cassandra_data_path, keyspace_glob, table_glob ] if incremental_backups: path += ['backups'] else: path += ['snapshots', snapshot.name] path += ['*'] with hide('output'): path = sudo('python -c "import os; print os.path.join(*%s)"' % path) logging.info('list files to backup matching %s path', path) with hide('output'): glob_results = sudo("python -c \"import glob; print '\\n'.join(glob.glob('%s'))\"" % path) files.extend([f.strip() for f in glob_results.split("\n")]) logging.info("found %d files", len(files)) return files
def ensure(self): parent = fs.Directory(self.path.rsplit("/", 1)[0]) parent.ensure() if self.exists() and self.is_empty(): self.remove() if not self.exists(): if not self.python: self.python = "`which python`" self.log("Creating virtualenv at {}".format(self.path)) # Get virtualenv version with hide("stdout"): version = run("virtualenv --version") version = version.split(".") venv_args = "" if version < (1, 7): if not self.system_site_packages: venv_args = " --no-site-packages" else: venv_args = " --{}-site-packages".format(self.system_site_packages and "system" or "no") cmd = "virtualenv{} -p {} {}".format(venv_args, self.python, self.path) if not env.dry_run: output_ctrl = settings(hide("stdout"), warn_only=True) with output_ctrl, parent.temp_ownership(recursive=True): result = run(cmd) if result.return_code != 0: abort("Virtualenv creation failed:\n{}".format(result)) super(VirtualEnv, self).ensure()
def run_action(self, action_dict, target): env_settings = self._fab_settings() if isinstance(target, list): api.env.hosts = target else: env_settings['host_string'] = target settings = api.settings(**env_settings) if self.args.get('debug') is True: hide = api.hide() else: hide = api.hide('running', 'stdout', 'output', 'warnings') ad = action_dict.copy() method = ad.pop('method') # grab the api method from fabric method = getattr(api, method) # Run all the things with hide, settings: result = method(**ad) if hasattr(result, 'return_code'): # Check for Failures. code = result.return_code # TODO(kevin) Make this do something smart if code != 0: pass else: return result
def get_entry_details(self): with settings(hide('everything')): all_jobs = run('SHOW QUEUE /BATCH /ALL | SEA SYS$PIPE %s' % self.name) entries = {line.split()[0]: None for line in all_jobs.splitlines() if self.name in line.upper()} def find_start_params(lines, tag): for (lineno, line) in enumerate(lines): if tag.lower() in line.lower(): return lineno for entry_id in entries: with settings(hide('everything')): this_entry = run('SHOW ENTRY {} /FULL'.format(entry_id)) this_entry = this_entry.splitlines() this_name = this_entry[-1].split()[1][1:] this_param = ''.join( [line.strip() for line in this_entry[find_start_params(this_entry, 'submitted'):-1] ] ) this_param = this_param.split('/')[1:] entries[entry_id] = {'name': this_name, 'params': this_param} return entries
def write_configfile(remote_path, content=None, filename=None): _info('attempting to write {}...'.format(remote_path)) rm_file = False if not filename: _, filename = tempfile.mkstemp() rm_file = True with open(filename, 'w') as f: f.write(content) _, old = tempfile.mkstemp() with hide('running', 'stdout', 'stderr'): if exists(remote_path): get(remote_path, old) with settings(hide('warnings'), warn_only=True): res = local('diff {} {}'.format(old, filename), capture=True) if res.failed: _bad('files differ') puts(res, show_prefix=False) if prompt('update file? [y/n]') == 'y': _info('writing new {}...'.format(remote_path)) put(filename, remote_path, use_sudo=True, mode=0644) else: _good('files already match') else: _good('no remote file exists, writing now') put(filename, remote_path, use_sudo=True, mode=0644) # remove files os.remove(old) if rm_file: os.remove(filename)
def users(self, min_uid=None, max_uid=None, use_sudo=True): """ Return a dict of all users:: {'user1': userstruct(user1), 'user2': userstruct(user2)} """ with settings(hide('everything'), warn_only=True): content = self.execute(self.users_cmd, use_sudo=use_sudo) if content.failed: raise PlatformError(content) users = {} for line in content.splitlines(): name, _, uid, gid, comment, home, shell = line.strip().split(':') uid = int(uid) # Skip over users outside specified min/max uid. if min_uid is not None and uid < min_uid: continue if max_uid is not None and uid > max_uid: continue with settings(hide('everything'), warn_only=True): content = self.execute(self.userget_groups_cmd % name, use_sudo=use_sudo) if content.failed: raise PlatformError(content) content = content.strip().split(' ') group, groups = content[0], set(content[1:]) user = userstruct(name, uid, int(gid), group, groups, comment, home, shell) users[name] = user return users
def ismounted(device): """ Check if partition is mounted Example:: from fabtools.disk import ismounted if ismounted('/dev/sda1'): print ("disk sda1 is mounted") """ # Check filesystem with settings(hide('running', 'stdout')): res = run_as_root('mount') for line in res.splitlines(): fields = line.split() if fields[0] == device: return True # Check swap with settings(hide('running', 'stdout')): res = run_as_root('swapon -s') for line in res.splitlines(): fields = line.split() if fields[0] == device: return True return False
def install(): prj = PROJECT_PATH print(blue("Create project folders")) run("mkdir -p {0}/cfg {0}/env {0}/logs {0}/pids {0}/src".format(prj)) print(blue("Setup virtualenv")) run("virtualenv -p python --system-site-packages {0}/env".format(prj)) print(blue("Install system packages")) syspackages = "supervisor==3.0 gunicorn==18.0" with hide('stdout'): run("{0}/env/bin/pip install {1}".format(prj, syspackages)) print(blue("Install requirements")) req_path = os.path.join(os.path.dirname(CWD), 'requirements') req_packages = open(req_path, 'r').read().replace('\n', ' ') with hide('stdout'): run("{0}/env/bin/pip install {1}".format(prj, req_packages)) print(blue("Upload source")) src_local = os.path.join(os.path.dirname(CWD), PROJECT_NAME) src_remote = os.path.join(prj, "src") with cd(src_remote): put(src_local, ".") run("chmod 755 -R {src_remote}".format(src_remote=src_remote)) print(blue("Create symlink for static")) run("ln -s {src}/{prj_name}/static {prj}/static".format( src=src_remote, prj=prj, prj_name=PROJECT_NAME)) upload_configs() nginx_ctl("reload") print(blue("Start supervisord")) run("{0}/env/bin/supervisord -c {0}/cfg/supervisord.conf".format(prj))
def main(argv): env.user = '******' env.disable_known_hosts = True env.no_keys = True # Disable Output env.debug = False try: opts, args = getopt.getopt(argv,"hH:p:",["hostname=","password="******"--hostname"): env.hosts = [arg] elif opt in ("-p", "--password"): env.password = arg with settings( try: hide('warnings', 'running', 'stdout', 'stderr', 'everything') ): execute(check_status)
def install_jenkins(*args, **kwargs): home = run('echo $HOME') version = kwargs.get('version', 'latest') init = os.path.join(home,'init') jenkins_base_dir = os.path.join(home, 'jenkins') jenkins_init = os.path.join(init, 'jenkins') port = kwargs.get('port') if not exists(jenkins_base_dir): run('mkdir ' + jenkins_base_dir) if not exists(os.path.join(jenkins_base_dir, 'jenkins.war')): with hide('output'): run('wget http://mirrors.jenkins-ci.org/war/%s/jenkins.war -O ~/jenkins/jenkins.war' % version) if not exists(os.path.join(jenkins_base_dir, 'org.jenkinsci.main.modules.sshd.SSHD.xml')): with hide('output'): run('wget https://templates.wservices.ch/jenkins/org.jenkinsci.main.modules.sshd.SSHD.xml -O ~/jenkins/org.jenkinsci.main.modules.sshd.SSHD.xml') if not exists(init): run('mkdir ~/init') if not exists(jenkins_init): with hide('output'): run('wget https://templates.wservices.ch/jenkins/jenkins.init -O ~/init/jenkins') run('chmod 750 ~/init/jenkins') sed(jenkins_init, 'PORT=HTTP_PORT', 'PORT=%s' % port) run('~/init/jenkins start') else: run('~/init/jenkins restart')
def parser(options): demo_name = options.demo_name create_env = options.create_env verbose = options.verbose hidden_output = [] if verbose < 1: hidden_output.append("stdout") if verbose < 2: hidden_output.append("running") print(green("Setting up the '{0}' demo...".format(demo_name))) env_path = "." full_env_path = None # Check for virtualenv if create_env: env_path = "{0}-env".format(demo_name) full_env_path = os.path.abspath(env_path) @virtualenv(full_env_path) def install_deps(): print(green("- Downloading and installing dependencies")) local("pip install -r requirements.txt") @virtualenv(full_env_path) def collect_static(): print(green("- Collecting static files")) local("make collect") with hide(*hidden_output): if create_env: create_virtualenv(env_path) # Download the demo download_demo(demo_name) # Change into project directory for next set of commands.. os.chdir(demo_name) # Ensure manage.py is executable.. managepy_chmod() with hide("running"): install_deps() with hide(*hidden_output): collect_static() print(green("\nComplete! Copy and paste the following in your shell:\n")) if create_env: print(green("cd {0}/{1}\nsource ../bin/activate".format(env_path, demo_name))) else: print(green("cd {0}".format(demo_name))) print(green("./bin/manage.py runserver")) print(green("\nOpen up a web browser and go to: http://localhost:8000\n"))
def start_btclient(conf): torrent = conf['experiment_options']['bt_torrent_path'] if not torrent: return with settings(hide('warnings'),warn_only=True): sudo('rm -rf /tmp/dtach_bt /tmp/btclient.nontor.data') run('mkdir /tmp/dtach_bt') # start up tor+bittorrent for i in xrange(state.current.config['clients_per_host']): port = 2000 + i with settings(hide('warnings'),warn_only=True): sudo('rm -rf /tmp/btclient.%d.data /var/lib/tor/%d/btclient' % (i, i)) run('dtach -n /tmp/dtach_bt/%(id)d bt_runner %(torrent)s /tmp/btclient.%(id)d.data %(log)s %(port)d' % { 'id': i, 'port': port, 'torrent': torrent, 'log': '/var/lib/tor/%d/btclient' % i }) # ensure that the non-tor btclients can't talk to the seeders and are # forced to talk only to the tor clients sudo('iptables -A OUTPUT -p tcp -m multiport --dports 6881:6889 -j REJECT') # start up bittorrent w/o tor run('dtach -n /tmp/dtach_bt/nontor btdownloadheadless %s --saveas /tmp/btclient.nontor.data --display_interval 30' % torrent)
def provision(common='master'): """Provision master with salt-master and salt-minion.""" require('environment') # Install salt minion with settings(warn_only=True): with hide('running', 'stdout', 'stderr'): installed = run('which salt-call') if not installed: # resolve salt hostnames to localhost sudo('echo "127.0.0.1 salt master" >> /etc/hosts') sudo('echo "salt" >> /etc/hostname') sudo('hostname salt') # install salt-master and salt-minion on master sudo('apt-get update -q -y') sudo('apt-get install python-software-properties -q -y') sudo('add-apt-repository ppa:saltstack/salt -y') sudo('apt-get update -q') sudo('apt-get install salt-master salt-minion -q -y') # temporarily stop minon and master with settings(warn_only=True): sudo('service salt-minion stop') sudo('service salt-master stop') # pre-seed the master's minion with accepted key with cd('/etc/salt/pki/minion/'): sudo('salt-key --gen-keys=minion') with cd('/etc/salt/pki/master/minions/'): sudo('cp /etc/salt/pki/minion/minion.pub /etc/salt/pki/master/minions/salt') # make sure git is installed for gitfs with settings(warn_only=True): with hide('running', 'stdout', 'stderr'): installed = run('which git') if not installed: sudo('apt-get install python-pip git-core -q -y') sudo('pip install -U GitPython') update_salt_config()
def _configure_loggly(self): # Temporary check to install syslog-ng until the AMI is updated logger.info("Configuring Loggly") with hide(*fab_output_hides): install_check = run('which syslog-ng', shell=True) if install_check.return_code != 0: # syslog-ng isn't installed sudo('apt-get -y --force-yes install syslog-ng') # Update the config context = env.loggly_inputs upload_template( '../config/tpl/syslog_ng/syslog-ng.conf', '/etc/syslog-ng/syslog-ng.conf', context, use_sudo=True) if self._needs_syslog_ng_restart(): logger.info("Restarting syslog-ng") with hide(*fab_output_hides): result = sudo('/etc/init.d/syslog-ng restart') if result.failed: logger.critical("Failed to restart syslog-ng") logger.critical(result) exit(1)
def setup_env(): """ Set up the directory structure at env.host_site_path """ print('+ Creating directory structure') if files.exists(env.host_site_path): if console.confirm('Remove existing directory %s' % env.host_site_path): with hide('running', 'stdout'): run('rm -rf %s' % env.host_site_path) else: print('+ Directory not removed and recreated') return with hide('running', 'stdout'): run('mkdir -p %s' % env.host_site_path) with cd(env.host_site_path): with hide('stdout'): run('mkdir changesets files private') print('+ Cloning repository: %s' % env.repo_url) run('ssh-keyscan -H github.com >> ~/.ssh/known_hosts') run('ssh-keyscan -H bitbucket.org >> ~/.ssh/known_hosts') run('%s clone --quiet %s private/repo' % (env.repo_type, env.repo_url)) run('chmod g+w private/repo') print('+ Site directory structure created at: %s' % env.host_site_path)
def fix_folder_perms(self): """ Fix folder permissions that can break while restoring. """ logger.info("Fixing EBS volume folder permissions") if self.is_local_db(): with hide(*fab_quiet): sudo('chown -R mysql:mysql /var/lib/mysql') sudo('chown -R mysql:adm /var/log/mysql') with hide(*fab_quiet): sudo('chown -R %s /var/log/uwsgi' % F_CHOWN) sudo('chown -R %s /var/log/celery' % F_CHOWN) sudo('chown -R %s /var/log/pstat' % F_CHOWN) # Ensure the pstat log dir is writable by root sudo('chmod -R g+w /var/log/pstat') # One-off fix for wrong permissions on /etc/cron.d/calabard_monitor sudo('chown root:root /etc/cron.d/calabard_monitor') # Ensure the media storage directory exists sudo('mkdir %s --parents' % MEDIA_STORAGE_ROOT) sudo('chown -R %s %s' % (F_CHOWN, MEDIA_STORAGE_ROOT)) sudo('chmod -R u+rw,g+rw,o+r,o-w %s' % MEDIA_STORAGE_ROOT)
def _activate_new_source(self, source_dir, active_version_symlinks): """ Create a symlink at each of `active_version_symlinks` towards the actual `source_dir`. The `source_dir` will probably be in a timestamped directory like `/opt/pstat/versions/<timestamp>`. Example symlinks: [`/opt/pstat/versions/current`, `/var/www/pstatbeta.com`] """ # Switch the symlink and use our new project logger.info("Activating new source via symlinks") for symlink in active_version_symlinks: logger.info("Symlinking %s", symlink) symlink_dir, _ = os.path.split(symlink) with hide(*fab_output_hides): sudo('mkdir -p %s' % symlink_dir) sudo('rm -f %s' % symlink) sudo('ln -s %s %s' % (source_dir, symlink)) # Clean out any stale pycs that may have been generated by queued # up processes that were using the old symlink with hide(*fab_output_hides): sudo('find %s -name "*.pyc" -delete' % source_dir)
def _setup_venv(app_name): with hide('output'): sudo('apt-get install -y python-virtualenv') run('mkdir -p ~/virtualenvs') with cd('~/virtualenvs'): if not files.exists(os.path.join(app_name, 'bin', 'activate')): run('virtualenv {}'.format(app_name)) requirements_file = '~/projects/{}/requirements.txt'.format(app_name) if not files.exists(requirements_file): if not console.confirm("File {} was not found. Do you want to continue?".format(requirements_file), default=False): abort('File {} not found.'.format(requirements_file)) with prefix('source ~/virtualenvs/{}/bin/activate'.format(app_name)): run('pip install -r {}'.format(requirements_file)) run('pip install gunicorn') with hide('output'): with cd(os.path.join('~', 'virtualenvs', app_name)): venv_dir = run('pwd') sudo('mkdir -p /var/virtualenvs') with cd('/var/virtualenvs'): if not files.exists(app_name): sudo('ln -s {} {}'.format(venv_dir, app_name))
def client1(): with settings(warn_only=True): print colored('##########################', 'blue') print colored('######## CLIENT 1 ########', 'blue') print colored('##########################', 'blue') sudo('yum clean all') sudo('yum install -y python-devel vim net-tools sudo openssh-server openssh-clients wget') sudo('yum install -y epel-release rsync') print colored('######################################', 'blue') print colored('END FIREWALL - NAT TABLE STATUS: ', 'blue') print colored('######################################', 'blue') with hide('output'): fw = sudo('iptables -t nat -L') print colored(fw, 'red') print colored('######################################', 'blue') print colored('END FIREWALL - FILTER TABLE STATUS: ', 'blue') print colored('######################################', 'blue') with hide('output'): fw = sudo('iptables -L') print colored(fw, 'red') print colored('##########################', 'blue') print colored('## NETWORK CONFIGURATION #', 'blue') print colored('##########################', 'blue') with hide('output'): netconf = sudo('ip addr show') print colored(netconf, 'yellow')
def bootstrap(action=''): """ Bootstrap the environment. """ with hide('running', 'stdout'): exists = run('if [ -d "{virtualenv_dir}" ]; then echo 1; fi'\ .format(**env)) if exists and not action == 'force': puts('Assuming {host} has already been bootstrapped since ' '{virtualenv_dir} exists.'.format(**env)) return # run('mkvirtualenv {project_name}'.format(**env)) with hide('running', 'stdout'): project_git_exists = run('if [ -d "{project_dir}" ]; then echo 1; fi'\ .format(**env)) if not project_git_exists: run('mkdir -p {0}'.format(posixpath.dirname(env.virtualenv_dir))) run('git clone {repository} {project_dir}'.format(**env)) # sudo('{virtualenv_dir}/bin/pip install -e {project_dir}'.format(**env)) # with cd(env.virtualenv_dir): # sudo('chown -R {user} .'.format(**env)) # fix_permissions() requirements() puts('Bootstrapped {host} - database creation needs to be done manually.'\ .format(**env))
def put(self, local_path, remote_path, use_sudo, mirror_local_mode, mode, local_is_path, temp_dir): from fabric.api import sudo, hide pre = self.ftp.getcwd() pre = pre if pre else '' if local_is_path and self.isdir(remote_path): basename = os.path.basename(local_path) remote_path = posixpath.join(remote_path, basename) if output.running: print("[%s] put: %s -> %s" % ( env.host_string, _format_local(local_path, local_is_path), posixpath.join(pre, remote_path) )) # When using sudo, "bounce" the file through a guaranteed-unique file # path in the default remote CWD (which, typically, the login user will # have write permissions on) in order to sudo(mv) it later. if use_sudo: target_path = remote_path hasher = hashlib.sha1() hasher.update(env.host_string) hasher.update(target_path) remote_path = posixpath.join(temp_dir, hasher.hexdigest()) # Read, ensuring we handle file-like objects correct re: seek pointer putter = self.ftp.put if not local_is_path: old_pointer = local_path.tell() local_path.seek(0) putter = self.ftp.putfo rattrs = putter(local_path, remote_path) if not local_is_path: local_path.seek(old_pointer) # Handle modes if necessary if (local_is_path and mirror_local_mode) or (mode is not None): lmode = os.stat(local_path).st_mode if mirror_local_mode else mode # Cast to octal integer in case of string if isinstance(lmode, basestring): lmode = int(lmode, 8) lmode = lmode & 07777 rmode = rattrs.st_mode # Only bitshift if we actually got an rmode if rmode is not None: rmode = (rmode & 07777) if lmode != rmode: if use_sudo: # Temporarily nuke 'cwd' so sudo() doesn't "cd" its mv # command. (The target path has already been cwd-ified # elsewhere.) with settings(hide('everything'), cwd=""): sudo('chmod %o \"%s\"' % (lmode, remote_path)) else: self.ftp.chmod(remote_path, lmode) if use_sudo: # Temporarily nuke 'cwd' so sudo() doesn't "cd" its mv command. # (The target path has already been cwd-ified elsewhere.) with settings(hide('everything'), cwd=""): sudo("mv \"%s\" \"%s\"" % (remote_path, target_path)) # Revert to original remote_path for return value's sake remote_path = target_path return remote_path
def dtop(dstat_file='/tmp/dstat',dstat_opts='-Tclmdrn',interval='1'): """ fab -H 192.168.1.1,localhost dtop """ index = env.hosts.index(env.host) max_host_len = _get_max_host_len() start_line = 3 while True: try: with settings( hide('warnings', 'running', 'stdout', 'stderr'), warn_only=True): result = run(dstat_file + ' ' + dstat_opts + ' ' + interval +' 1') if result.failed: result = run('dstat ' + dstat_opts + ' ' + interval + ' 1') with settings( hide('warnings', 'running', 'stderr'), warn_only=True): lines = result.stdout.splitlines() try: host_stat = lines.pop() _init_console() _display_header(lines, max_host_len) except IndexError: host_stat = 'Can not run dstat! Run "fab dtop.put_dstat" task.' # ホストリストの順に描画行を決定して、そこにカーソルを移動する line_control = '\x1b[' + str(index + start_line) + ';0H' print line_control, print '%s%s%s' % (env.host.ljust(max_host_len), '\t', host_stat) except KeyboardInterrupt: line_control = '\x1b[0;0B' # 最後の行に移動 print line_control, break
def __install_site(): """ Instala os arquivos de configuração no supervisor, monit, nginx, logrotate, etc """ # Supervisor sudo('mv -f %s/releases/%s/conf/supervisor/*.conf ' ' /etc/supervisor/conf.d/' % (env.project_path, env.release)) sudo('supervisorctl -c /etc/supervisor/supervisord.conf reread') sudo('supervisorctl -c /etc/supervisor/supervisord.conf update') # Monit sudo('mv -f %s/releases/%s/conf/monit/*.conf ' '/etc/monit/conf.d/' % (env.project_path, env.release)) sudo('/etc/init.d/monit restart') # Nginx sudo('mv -f {0}/releases/{1}/conf/nginx/{2}.conf ' ' /etc/nginx/sites-available/{2}'.format(env.project_path, env.release, env.project_name)) with settings(hide('warnings', 'stdout', 'stderr'), warn_only=True): sudo('unlink /etc/nginx/sites-enabled/default') sudo('ln -s /etc/nginx/sites-available/{0} /etc/nginx/sites-enabled/{0}'.format(env.project_name)) # Log Rotate with settings(hide('warnings', 'stdout', 'stderr'), warn_only=True): with cd(env.project_path): sudo('mv -f %s/releases/%s/conf/logrotate/* ' '/etc/logrotate.d/' % (env.project_path, env.release)) sudo('chmod 644 /etc/logrotate.d/*') # Must ensure permission __green(u'Todas as configurações foram instaladas')
def allocated_ports(excluded_programs=None): if excluded_programs is None: excluded_programs = [] # run_gunicorn old syntax options = " ".join("--exclude={}.conf".format(name) for name in excluded_programs) cmd = "grep {} gunicorn /etc/supervisor/conf.d/*.conf | cut -d':' -f3 | cut -d' ' -f1".format(options) with hide('running', 'stdout'): res = run(cmd) for port in res.split(): try: yield int(port) except ValueError: pass # gunicorn new syntax options = " ".join("--exclude={}.conf".format(name) for name in excluded_programs) cmd = "grep {} gunicorn /etc/supervisor/conf.d/*.conf | cut -d':' -f4 | cut -d' ' -f1".format(options) with hide('running', 'stdout'): res = run(cmd) for port in res.split(): try: yield int(port) except ValueError: pass
def last(): """w;last""" with hide('output'): lst = run('last') _write_file('last', lst) with hide('output'): who = run('w') _write_file('w', who)
def user_list(): """getent passwd""" with hide('output'): ulist = run('getent passwd') _write_file('userlist', ulist) with hide('output'): glist = run('getent group') _write_file('grouplist', glist)
def ss(): """ss; ss -s""" with hide('output'): sss = run('ss -s') _write_file('ss-s', sss) with hide('output'): ssp = run('ss') _write_file('ss', ssp)
def _start_sphinx(stop_first=False): start_cmd = "start sphinxsearch" with hide(*fab_output_hides): if stop_first: with hide(*fab_quiet): sudo("stop sphinxsearch") sudo(start_cmd)
def hostnames_for_role(role): with hide('running', 'stdout'): result = salt( cmd='grains.item fqdn --output=yaml', target='-G "roles:%s"' % role) return [v['fqdn'] for v in yaml.safe_load(result.stdout).values()]
def service_enabled(name): """Check if an upstart service is enabled.""" with settings(warn_only=True): with hide('running', 'stdout', 'stderr'): return sudo('service %s status' % name).succeeded
def install_openstack(settings_dict, envs=None, verbose=None, url_script=None, prepare=False, force=False): """ Install OS with COI with script provided by Chris on any host(s) :param settings_dict: settings dictionary for Fabric :param envs: environment variables to inject when executing job :param verbose: if to hide all output or print everything :param url_script: URl of Cisco installer script from Chris :param force: Use if you don't connect via interface you gonna bridge later :return: always true """ envs = envs or {} verbose = verbose or [] if settings_dict['user'] != 'root': use_sudo_flag = True run_func = sudo else: use_sudo_flag = False run_func = run with settings(**settings_dict), hide(*verbose), shell_env(**envs): # TODO: check statuses of commands with cd("/root/"): warn_if_fail(run_func("apt-get update")) ## avoid grub and other prompts warn_if_fail(run_func('DEBIAN_FRONTEND=noninteractive apt-get -y ' '-o Dpkg::Options::="--force-confdef" -o ' 'Dpkg::Options::="--force-confold" dist-upgrade')) warn_if_fail(run_func("apt-get install -y git")) warn_if_fail(run_func("git config --global user.email '*****@*****.**';" "git config --global user.name 'Test Node'")) warn_if_fail(sed("/etc/hosts", "127.0.1.1.*", "127.0.1.1 all-in-one all-in-one.domain.name", use_sudo=use_sudo_flag)) warn_if_fail(put(StringIO("all-in-one"), "/etc/hostname", use_sudo=use_sudo_flag)) warn_if_fail(run_func("hostname all-in-one")) if use_sudo_flag: append("/etc/sudoers", "{user} ALL=(ALL) NOPASSWD: ALL".format(user=settings_dict['user']), use_sudo=True) warn_if_fail(run_func("git clone -b icehouse " "https://github.com/CiscoSystems/puppet_openstack_builder")) if not force and not prepare: with cd("puppet_openstack_builder"): ## run the latest, not i.0 release sed("/root/puppet_openstack_builder/install-scripts/cisco.install.sh", "icehouse/snapshots/i.0", "icehouse-proposed", use_sudo=use_sudo_flag) sed("/root/puppet_openstack_builder/data/hiera_data/vendor/cisco_coi_common.yaml", "/snapshots/i.0", "-proposed", use_sudo=use_sudo_flag) with cd("install-scripts"): run_func("./install.sh") result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False) tries = 1 error = "Error:" while error in result and tries <= APPLY_LIMIT: time.sleep(60) print >> sys.stderr, "Some errors found, running apply again" result = run_func('puppet apply -v /etc/puppet/manifests/site.pp', pty=False) tries += 1 if exists('/root/openrc'): get('/root/openrc', "./openrc") else: print (red("No openrc file, something went wrong! :(")) print (green("Copying logs and configs")) collect_logs(run_func=run_func, hostname="aio", clean=True) print (green("Finished!")) return True if force and not prepare: shell_envs = ";".join(["export " + k + "=" + v for k, v in envs.iteritems()]) or "" sudo_mode = "sudo " if use_sudo_flag else '' if not settings_dict['gateway']: local("{shell_envs}; ssh -t -t -i {id_rsa} {user}@{host} \ '/bin/bash /root/install_icehouse_cisco.sh'".format( shell_envs=shell_envs, id_rsa=settings_dict['key_filename'], user=settings_dict['user'], host=settings_dict['host_string'])) local("scp -i {id_rsa} {user}@{host}:/root/openrc ./openrc".format( id_rsa=settings_dict['key_filename'], user=settings_dict['user'], host=settings_dict['host_string'])) else: local('ssh -t -t -i {id_rsa} {user}@{gateway} \ "{shell_envs}; ssh -t -t -i {id_rsa} {user}@{host} \ \'{sudo_mode}/bin/bash /root/install_icehouse_cisco.sh\'"'.format( shell_envs=shell_envs, id_rsa=settings_dict['key_filename'], user=settings_dict['user'], host=settings_dict['host_string'], gateway=settings_dict['gateway'], sudo_mode=sudo_mode)) local('scp -Cp -o "ProxyCommand ssh {user}@{gateway} ' 'nc {host} 22" root@{host}:/root/openrc ./openrc'.format( user=settings_dict['user'], host=settings_dict['host_string'], gateway=settings_dict['gateway'], )) print (green("Finished!")) return True
def pwd(local=False): with hide('everything'): if local: return lrun('pwd', capture=True) return env.run('pwd', capture=True)
def instalar_pxp(): with settings(hide('stderr'), warn_only=True): question = raw_input("La conexion se realizara por un proxy? (s/n) : ") if question == 's': question = raw_input( "Ingrese la cadena de conexion del servidor proxy (proxyuser:proxypwd@server:port o server:port) : " ) proxy = question else: proxy = "" run("yum -y install wget") with cd('/tmp'): run('ls') version = run("grep -o release.. /etc/redhat-release") if (version == 'release 7'): # postgres de rpm de postgres 9.5# run("wget https://download.postgresql.org/pub/repos/yum/reporpms/EL-7-x86_64/pgdg-redhat-repo-latest.noarch.rpm" ) else: # postgres de rpm de postgres 9.5# run("wget http://yum.postgresql.org/9.5/redhat/rhel-6-x86_64/pgdg-redhat95-9.5-3.noarch.rpm" ) # configuraicon de archivos de centos-base.repo agregando una linea # s = open("/etc/yum.repos.d/CentOS-Base.repo", 'a') s.write("exclude=postgresql*\n\n") s.close() if (version == 'release 7'): run("rpm -Uvh --replacepkgs pgdg-redhat-repo-latest.noarch.rpm") else: run("rpm -Uvh --replacepkgs pgdg-redhat95-9.5-3.noarch.rpm") # instalacion de postgres y la primera corrida # S_pgsql = "service postgresql-9.5" I_pgsql = "postgresql95" sudo( "yum -y install postgresql11-server postgresql11-docs postgresql11-contrib postgresql11-plperl postgresql11-plpython postgresql11-pltcl postgresql11-test rhdb-utils gcc-objc postgresql11-devel " ) if (version == 'release 7'): run("/usr/pgsql-11/bin/postgresql-11-setup initdb") run("systemctl start postgresql-11") run("systemctl enable postgresql-11") else: run("service postgresql-9.5 initdb") run("service postgresql-9.5 start") run("chkconfig postgresql-9.5 on") # instalacion del php y apache mas la primera corrida # sudo( "yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm" ) sudo( "yum install -y https://rpms.remirepo.net/enterprise/remi-release-7.rpm" ) sudo("yum install -y yum-utils") sudo("yum-config-manager --enable remi-php72") sudo( "yum -y install httpd php72-php mod_ssl mod_auth_pgsql php72-php-pear php72-php-bcmath php72-php-cli php72-php-ldap php72-php-pdo php72-php-pgsql php72-php-gd php72-php-mbstring php72-php-pecl-zip" ) if (version == 'release 7'): run("systemctl start httpd") run("systemctl enable httpd") else: run("service httpd start") run("chkconfig httpd on") #Creacion de archivos para bitacoras archi = open("/usr/local/lib/phx.c", 'w') archi.write('#include "postgres.h"\n') archi.write('#include <string.h>\n') archi.write('#include "fmgr.h"\n') archi.write('#include "utils/geo_decls.h"\n') archi.write('#include <stdio.h>\n') archi.write('#ifdef PG_MODULE_MAGIC\n') archi.write('PG_MODULE_MAGIC;\n') archi.write('#endif\n') archi.write('/* by value */\n') archi.write('PG_FUNCTION_INFO_V1(monitor_phx);\n') archi.write('Datum\n') archi.write('monitor_phx(PG_FUNCTION_ARGS)\n') archi.write('{\n') archi.write(' int32 arg = PG_GETARG_INT32(0);\n') archi.write(' system("sudo /usr/local/lib/./phxbd.sh");\n') archi.write(' PG_RETURN_INT32(arg);\n') archi.write('}') archi.close() run("gcc -I /usr/local/include -I /usr/pgsql-11/include/server/ -fpic -c /usr/local/lib/phx.c" ) run("gcc -I /usr/local/include -I /usr/pgsql-11/include/server/ -shared -o /usr/local/lib/phx.so phx.o" ) run("chown root.postgres /usr/local/lib/phx.so") run("chmod 750 /usr/local/lib/phx.so") archi = open("/usr/local/lib/phxbd.sh", 'w') archi.write('!/bin/bash\n') archi.write( 'top -b -n 1 | grep -e postgres -e httpd | awk \'{print $1","$12","$2","$9","$10","$5""""}\' > /tmp/procesos.csv\n' ) archi.write('chown root.postgres /tmp/procesos.csv\n') archi.write('chmod 740 /tmp/procesos.csv') sudo("chown root.postgres /usr/local/lib/phxbd.sh") sudo("sudo chmod 700 /usr/local/lib/phxbd.sh") f = open("/etc/sudoers", 'r') chain = f.read() chain = chain.replace("Defaults requiretty", "#Defaults requiretty") chain = chain.replace( "root ALL=(ALL) ALL", "root ALL=(ALL) ALL\n postgres ALL=NOPASSWD: /usr/local/lib/phxbd.sh" ) f.close() f = open("/etc/sudoers", 'w') f.write(chain) f.close() #Instalacion de mcrypt para servicios rest if (version == 'release 7'): run("wget http://dl.fedoraproject.org/pub/epel/7/x86_64/Packages/e/epel-release-7-11.noarch.rpm" ) run("wget http://rpms.remirepo.net/enterprise/remi-release-7.rpm") #run("rpm -Uvh remi-release-7*.rpm epel-release-7*.rpm") #else: # #run("wget http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm") # #run("wget http://rpms.famillecollet.com/enterprise/remi-release-6.rpm") # #sudo("rpm -Uvh remi-release-6*.rpm epel-release-6*.rpm") # #Comentaqdo pro que en los prerequisitos estamos instalado EPEL run("yum -y update") run("yum -y install php72-php-mcrypt*") # cambio de los archivos pg_hba y postgres.config# archi = open("/var/lib/pgsql/11/data/pg_hba.conf", 'w') archi.write( "# TYPE DATABASE USER ADDRESS METHOD\n\n" ) archi.write("# 'local' is for Unix domain socket connections only\n") archi.write( "local all postgres,dbkerp_conexion trust\n" ) archi.write( "local all all md5\n" ) archi.write("# IPv4 local connections:\n") archi.write( "host all all 127.0.0.1/32 md5\n" ) archi.write( "host all all 192.168.0.0/16 md5\n" ) archi.write("# IPv6 local connections:\n") archi.write( "host all all ::1/128 md5\n" ) archi.close() f = open("/var/lib/pgsql/11/data/postgresql.conf", 'r') chain = f.read() chain = chain.replace("pg_catalog.english", "pg_catalog.spanish") chain = chain.replace("log_destination = 'stderr'", "log_destination = 'csvlog'") chain = chain.replace("log_filename = 'postgresql-%a.log'", "log_filename = 'postgresql-%Y-%m-%d.log'") chain = chain.replace("log_truncate_on_rotation = on", "log_truncate_on_rotation = off") chain = chain.replace("#log_error_verbosity = default", "log_error_verbosity = verbose") chain = chain.replace("#log_statement = 'none'", "log_statement = 'mod'") chain = chain.replace("iso, mdy", "iso, dmy") f.close() otro = open("/var/lib/pgsql/11/data/postgresql.conf", 'w') otro.write(chain) otro.close() s = open("/var/lib/pgsql/11/data/postgresql.conf", 'a') s.write("listen_addresses = '*'\n") s.write("bytea_output = 'escape'\n") s.close() db_pass = "******" sudo('psql -c "ALTER USER postgres WITH ENCRYPTED PASSWORD E\'%s\'"' % (db_pass), user='******') sudo('psql -c "CREATE DATABASE dbkerp WITH ENCODING=\'UTF-8\';"', user='******') sudo( 'psql -c "CREATE USER dbkerp_conexion WITH PASSWORD \'dbkerp_conexion\';"', user='******') sudo('psql -c "ALTER ROLE dbkerp_conexion SUPERUSER;"', user='******') sudo( 'psql -c "CREATE USER dbkerp_admin WITH PASSWORD \'a1a69c4e834c5aa6cce8c6eceee84295\';"', user='******') sudo('psql -c "ALTER ROLE dbkerp_admin SUPERUSER;"', user='******') if (version == 'release 7'): run('systemctl restart postgresql-11') else: run('service postgresql-9.5 restart') # instalacion de git para poder bajar el repositoriio pxp y moviendo a la carpeta /var/www/html/kerp/# sudo("yum -y install git-core") run("mkdir /var/www/html/kerp") run("mkdir /var/www/html/kerp/pxp") #Si existe proxy se configura github para el proxy if (proxy != ""): run("git config --global http.proxy http://" + proxy) run("git config --global https.proxy https://" + proxy) run("git clone https://github.com/ofepbolivia/pxp.git /var/www/html/kerp/pxp" ) run("chown -R apache.apache /var/www/html/kerp/") run("chmod 700 -R /var/www/html/kerp/") # haciendo una copia de datosgenerales.samples.php y modificando archivo# f = open("/var/www/html/kerp/pxp/lib/DatosGenerales.sample.php") g = open("/var/www/html/kerp/pxp/lib/DatosGenerales.php", "w") linea = f.readline() while linea != "": g.write(linea) linea = f.readline() g.close() f.close() #TODO VOLVER VARIABLE LA CARPETA PRINCIPAL KERP f = open("/var/www/html/kerp/pxp/lib/DatosGenerales.php", 'r') chain = f.read() chain = chain.replace("/web/lib/lib_control/", "/kerp/pxp/lib/lib_control/") chain = chain.replace("/kerp-boa/", "/kerp/") chain = chain.replace("/var/lib/pgsql/9.1/data/pg_log/", "/var/lib/pgsql/11/data/pg_log/") f.close() otro = open("/var/www/html/kerp/pxp/lib/DatosGenerales.php", 'w') otro.write(chain) otro.close() run("ln -s /var/www/html/kerp/pxp/lib /var/www/html/kerp/lib") run("ln -s /var/www/html/kerp/pxp/index.php /var/www/html/kerp/index.php" ) run("ln -s /var/www/html/kerp/pxp/sis_generador /var/www/html/kerp/sis_generador" ) run("ln -s /var/www/html/kerp/pxp/sis_organigrama /var/www/html/kerp/sis_organigrama" ) run("ln -s /var/www/html/kerp/pxp/sis_parametros /var/www/html/kerp/sis_parametros" ) run("ln -s /var/www/html/kerp/pxp/sis_seguridad /var/www/html/kerp/sis_seguridad" ) run("ln -s /var/www/html/kerp/pxp/sis_workflow /var/www/html/kerp/sis_workflow" ) archi = open('/var/www/html/kerp/sistemas.txt', 'w') archi.close() run("mkdir /var/www/html/kerp/reportes_generados") sudo( "setfacl -R -m u:apache:wrx /var/www/html/kerp/reportes_generados") # sudo("yum -y install rpm-build") sudo("setfacl -R -m u:postgres:wrx /var/www/html") sudo("chcon -Rv --type=httpd_sys_rw_content_t /var/www/html/kerp/") sudo("setsebool -P httpd_can_network_connect_db=1") sudo("setsebool -P httpd_can_network_connect 1") # iptables if (version == 'release 6'): run("iptables --flush") run("iptables -P INPUT ACCEPT") run("iptables -P OUTPUT ACCEPT") run("iptables -P FORWARD ACCEPT") #Interfaz local aceptar run("iptables -A INPUT -i lo -j ACCEPT") #Comunicaciones establecidas aceptar run("iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT" ) #Ping Aceptar run("iptables -A INPUT -p icmp --icmp-type echo-request -j ACCEPT") #Ssh Aceptar run("iptables -A INPUT -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT" ) #http y https aceptar run("iptables -A INPUT -p tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT" ) run("iptables -A INPUT -p tcp --dport 443 -m state --state NEW,ESTABLISHED -j ACCEPT" ) #websocket run("iptables -A INPUT -p tcp --dport 8010 -m state --state NEW,ESTABLISHED -j ACCEPT" ) #postgres aceptar run("iptables -A INPUT -p tcp --dport 5432 -m state --state NEW,ESTABLISHED -j ACCEPT" ) run("iptables -P INPUT DROP") run("service iptables save") run("service iptables restart") else: run("firewall-cmd --permanent --add-port=22/tcp") run("firewall-cmd --permanent --add-port=80/tcp") run("firewall-cmd --permanent --add-port=5432/tcp") run("firewall-cmd --permanent --add-port=8010/tcp") run("firewall-cmd --reload") sudo( "php /var/www/html/kerp/lib/ratchet/pxp-Server.php > /dev/null 2>&1 &" ) sudo("setfacl -R -m u:postgres:wrx /root") prompts = [] prompts += expect('Ingrese una opcion.*', '1') prompts += expect('Ingrese el nombre de la BD.*', 'dbkerp') prompts += expect('Desea obtener un backup de la BD.*', 'NO') prompts += expect('los datos de prueba.*', 'n') with expecting(prompts): sudo( "/var/www/html/kerp/pxp/utilidades/restaurar_bd/./restaurar_todo.py", user="******")
def get(self, remote_path, local_path, use_sudo, local_is_path, rremote=None, temp_dir=""): from fabric.api import sudo, hide # rremote => relative remote path, so get(/var/log) would result in # this function being called with # remote_path=/var/log/apache2/access.log and # rremote=apache2/access.log rremote = rremote if rremote is not None else remote_path # Handle format string interpolation (e.g. %(dirname)s) path_vars = { 'host': env.host_string.replace(':', '-'), 'basename': os.path.basename(rremote), 'dirname': os.path.dirname(rremote), 'path': rremote } if local_is_path: # Naive fix to issue #711 escaped_path = re.sub(r'(%[^()]*\w)', r'%\1', local_path) local_path = os.path.abspath(escaped_path % path_vars) # Ensure we give ssh.SFTPCLient a file by prepending and/or # creating local directories as appropriate. dirpath, filepath = os.path.split(local_path) if dirpath and not os.path.exists(dirpath): os.makedirs(dirpath) if os.path.isdir(local_path): local_path = os.path.join(local_path, path_vars['basename']) if output.running: print("[%s] download: %s <- %s" % (env.host_string, _format_local(local_path, local_is_path), remote_path)) # Warn about overwrites, but keep going if local_is_path and os.path.exists(local_path): msg = "Local file %s already exists and is being overwritten." warn(msg % local_path) # When using sudo, "bounce" the file through a guaranteed-unique file # path in the default remote CWD (which, typically, the login user will # have write permissions on) in order to sudo(cp) it. if use_sudo: target_path = remote_path hasher = hashlib.sha1() hasher.update(env.host_string) hasher.update(target_path) target_path = posixpath.join(temp_dir, hasher.hexdigest()) # Temporarily nuke 'cwd' so sudo() doesn't "cd" its mv command. # (The target path has already been cwd-ified elsewhere.) with settings(hide('everything'), cwd=""): sudo('cp -p "%s" "%s"' % (remote_path, target_path)) # Only root and the user has the right to read the file sudo('chmod %o "%s"' % (0404, target_path)) remote_path = target_path try: # File-like objects: reset to file seek 0 (to ensure full overwrite) # and then use Paramiko's getfo() directly getter = self.ftp.get if not local_is_path: local_path.seek(0) getter = self.ftp.getfo getter(remote_path, local_path) finally: # try to remove the temporary file after the download if use_sudo: with settings(hide('everything'), cwd=""): sudo('rm -f "%s"' % remote_path) # Return local_path object for posterity. (If mutated, caller will want # to know.) return local_path
def test_lint(): with hide('running', 'stdout', 'stderr', 'warnings', 'aborts'): _print("Checking syntax...") local('flake8 . --exclude=*/migrations/* --max-line-length=120 --ignore=F403', capture=True) print "done"
def controllerLTE(enable_controller=1): fh = StringIO(); [tx_count_,rx_ack_count_]=shm_reader(); count_round=0 fp=open('/tmp/controllerLTE.log','w') if enable_controller=='2': with fab.hide('output','running','warnings'), fab.settings(warn_only=True): fab.run('bytecode-manager -a 1') else: print("ENABLE TDMA") with fab.hide('output','running','warnings'), fab.settings(warn_only=True): fab.run('bytecode-manager -a 2') fab.run('bytecode-manager --set-tdma-mask 1111111111') while True: count_round=count_round+1 [tx_count,rx_ack_count]=shm_reader() dtx=numpy.mod(tx_count-tx_count_,0xFFFF) dack=numpy.mod(rx_ack_count-rx_ack_count_,0xFFFF) tx_count_=tx_count rx_ack_count_=rx_ack_count psucc=numpy.divide(dack,dtx) for i in range(0,len(psucc)): if numpy.isinf(psucc[i]): psucc[i]=0 continue if numpy.isnan(psucc[i]): psucc[i]=0 continue psucc_tot=numpy.divide(numpy.sum(dack),numpy.sum(dtx)) print("rx_ack = {}".format(dack)) print("tx = {}".format(dtx)) numpy.set_printoptions(precision=1) print("psucc = {}".format(psucc)) print("psucc_tot={}".format(psucc_tot)) print("count_round={}".format(count_round)) mask="" for x in psucc: if x > 0.5: maskval=1 elif numpy.isinf(x): maskval=0 else: maskval=0 mask="{}{}".format(maskval,mask) mask_int = [ int(x) for x in list(mask)] mask_sum=0 for x in list(mask_int): mask_sum+=x if mask=="0000000000": mask="1111111111" EST_SLOT=4 L=10 print(mask) p_insert=numpy.random.rand(); if mask_sum < EST_SLOT: p_insert=1; mask_int=update_pattern(mask_int,L,mask_sum+1) # else: # p_insert=0 # if p_insert > 0.01: # mask_int=update_pattern(mask_int,L,mask_sum+1) mask="" for x in mask_int: mask="{}{}".format(mask,x) #count_round=0 print(mask_int) if enable_controller=='0': mask="1111111111" if enable_controller == '0' or enable_controller =='1': with fab.hide('output','running','warnings'), fab.settings(warn_only=True): fab.run('bytecode-manager --set-tdma-mask {}'.format(mask)) json_msg={ 'time':time.time(), 'psucc':psucc_tot, 'mask':mask,'enable_controller':enable_controller,'mask_sum':mask_sum, 'psucc_list':list(psucc)} print(json_msg) fp.write(json.dumps(json_msg)) fp.write("\n") fp.flush() time.sleep(1) close(fp)
def prep(): with fab.hide('status', 'aborts', 'running', 'warnings', 'stdout', 'stderr', 'user'), fab.settings(warn_only=True): fab.put('gethttp.py', '~')
def get_branch(path): with env.cd(path): with hide('everything'): cmd = "hg branch" return env.run(cmd, capture=True)
def branches(): "returns a list of available branches" with hide('everything'): cmd = 'hg branches' res = env.run(cmd, capture=True) return [bit[:bit.find(' ')] for bit in res.split('\n')]
def remote_branch(): ''' Get the current branch of the remote server. ''' with hide('everything'): result = run('git rev-parse --abbrev-ref HEAD') return result.strip()
def get_hostname(): """ Get the fully qualified hostname. """ with settings(hide('running', 'stdout')): return run('hostname --fqdn')
def remote_listdir(path): with hide('everything'): output = run('ls %s' % path) if not output.startswith('ls: cannot access'): return output.split() return []
def dir_exists(dirname): with settings(warn_only=True), hide('warnings'): # pylint: disable=not-context-manager res = run('[ -d {} ]'.format(dirname)) return res.return_code == 0
def get_host_date(host_string): # output here should be hidden anyway. with fab.settings(fab.hide('running', 'stdout')): return run_command_on_host('date +%Y%m%dT%H%M%S', host_string).stdout
def _invalid_file_obj_situations(self, remote_path): with settings(hide('running'), warn_only=True): get(remote_path, StringIO()) assert_contains('is a glob or directory', sys.stderr.getvalue())
def test_get_returns_none_for_stringio(self): """ get() should return None if local_path is a StringIO """ with hide('everything'): eq_([], get('/file.txt', StringIO()))
def run_cmd(cmd): with hide('running', 'stdout', 'stderr'): if self.use_sudo: sudo(cmd) else: run(cmd)
def mute(): """Run a fabric command without reporting any responses to the user. """ with settings(warn_only='true'): with hide('running', 'stdout', 'stderr', 'warnings'): yield
def load_config(env, remote=None, config_user='******', debug=False): # pylint: disable=too-complex,too-many-branches,too-many-statements env['sudo_prefix'] += '-H ' # Load the json file try: with open(f'{settings.SITE_ROOT}/server.json', 'r', encoding='utf-8') as json_data: config = json.load(json_data) except Exception as e: print(e) raise Exception( 'Something is wrong with the server.json file, make sure it exists and is valid JSON.' ) # Define current host from settings in server config # First check if there is a single remote or multiple. if 'remotes' not in config or not config['remotes']: raise Exception('No remotes specified in config.') # Prompt for a host selection. remote_keys = list(config['remotes'].keys()) if len(remote_keys) == 1: remote_prompt = remote_keys[0] elif remote: remote_prompt = remote if remote_prompt not in remote_keys: raise Exception('Invalid remote name `{}`.'.format(remote)) else: print('Available hosts: {}'.format(', '.join( config['remotes'].keys()))) remote_prompt = prompt('Please enter a remote: ', default=remote_keys[0], validate=lambda x: x in remote_keys) remote = config['remotes'][remote_prompt] env.host_string = remote['server']['ip'] config['remote_name'] = remote_prompt env.user = config_user env.disable_known_hosts = True env.reject_unknown_hosts = False # If is_aws is explicitly declared, trust it. if 'is_aws' in remote: aws_check = remote['is_aws'] # Try to guess it from the hostname. elif 'amazonaws.com' in env.host_string: aws_check = True # Dunno, ask the user. else: aws_check = confirm('Is this host on AWS?', default=False) if aws_check: if 'initial_user' in remote['server']: env.user = remote['server']['initial_user'] else: env.user = '******' if 'identity_file' in remote['server']: if remote['server']['identity_file']: env.key_filename = remote['server']['identity_file'] else: key = prompt('Please enter the path to the AWS key pair: ') if key: env.key_filename = key else: if sys.argv[1] == 'deploy' and 'initial_user' in remote['server']: env.user = remote['server']['initial_user'] elif 'deploy_user' in remote['server']: env.user = remote['server']['deploy_user'] # Make sure we can connect to the server with hide('output', 'running', 'warnings'): with fabric_settings(warn_only=True): if not run('whoami'): print('Failed to connect to remote server') exit() if not debug: # Change the output to be less verbose. fabric.state.output['stdout'] = False fabric.state.output['running'] = False # Return the server config return config, remote
def site(config_name, template_contents=None, template_source=None, enabled=True, check_config=True, **kwargs): """ Require an Apache site. You must provide a template for the site configuration, either as a string (*template_contents*) or as the path to a local template file (*template_source*). :: from fabtools import require CONFIG_TPL = ''' <VirtualHost *:%(port)s> ServerName %(hostname})s DocumentRoot %(document_root)s <Directory %(document_root)s> Options Indexes FollowSymLinks MultiViews AllowOverride All Order allow,deny allow from all </Directory> </VirtualHost> ''' require.apache.site( 'example.com', template_contents=CONFIG_TPL, port=80, hostname='www.example.com', document_root='/var/www/mysite', ) .. seealso:: :py:func:`fabtools.require.files.template_file` """ server() config_filename = '/etc/apache2/sites-available/%s' % _get_config_name( config_name) context = { 'port': 80, } context.update(kwargs) context['config_name'] = config_name template_file(config_filename, template_contents, template_source, context, use_sudo=True) if enabled: enable_site(config_name) else: disable_site(config_name) if check_config: with settings(hide('running', 'warnings'), warn_only=True): if run_as_root('apache2ctl configtest').failed: disable_site(config_name) message = red( "Error in %(config_name)s apache site config (disabling for safety)" % locals()) abort(message) reload_service('apache2')
def upload_template(filename, destination, context=None, use_jinja=False, template_dir=None, use_sudo=False, backup=True, mirror_local_mode=False, mode=None, pty=None, keep_trailing_newline=False, temp_dir=''): """ Render and upload a template text file to a remote host. Returns the result of the inner call to `~fabric.operations.put` -- see its documentation for details. ``filename`` should be the path to a text file, which may contain `Python string interpolation formatting <http://docs.python.org/library/stdtypes.html#string-formatting>`_ and will be rendered with the given context dictionary ``context`` (if given.) Alternately, if ``use_jinja`` is set to True and you have the Jinja2 templating library available, Jinja will be used to render the template instead. Templates will be loaded from the invoking user's current working directory by default, or from ``template_dir`` if given. The resulting rendered file will be uploaded to the remote file path ``destination``. If the destination file already exists, it will be renamed with a ``.bak`` extension unless ``backup=False`` is specified. By default, the file will be copied to ``destination`` as the logged-in user; specify ``use_sudo=True`` to use `sudo` instead. The ``mirror_local_mode``, ``mode``, and ``temp_dir`` kwargs are passed directly to an internal `~fabric.operations.put` call; please see its documentation for details on these two options. The ``pty`` kwarg will be passed verbatim to any internal `~fabric.operations.run`/`~fabric.operations.sudo` calls, such as those used for testing directory-ness, making backups, etc. The ``keep_trailing_newline`` kwarg will be passed when creating Jinja2 Environment which is False by default, same as Jinja2's behaviour. .. versionchanged:: 1.1 Added the ``backup``, ``mirror_local_mode`` and ``mode`` kwargs. .. versionchanged:: 1.9 Added the ``pty`` kwarg. .. versionchanged:: 1.11 Added the ``keep_trailing_newline`` kwarg. .. versionchanged:: 1.11 Added the ``temp_dir`` kwarg. """ func = use_sudo and sudo or run if pty is not None: func = partial(func, pty=pty) # Normalize destination to be an actual filename, due to using StringIO with settings(hide('everything'), warn_only=True): if func('test -d %s' % _expand_path(destination)).succeeded: sep = "" if destination.endswith('/') else "/" destination += sep + os.path.basename(filename) # Use mode kwarg to implement mirror_local_mode, again due to using # StringIO if mirror_local_mode and mode is None: mode = os.stat(apply_lcwd(filename, env)).st_mode # To prevent put() from trying to do this # logic itself mirror_local_mode = False # Process template text = None if use_jinja: try: template_dir = template_dir or os.getcwd() template_dir = apply_lcwd(template_dir, env) from jinja2 import Environment, FileSystemLoader jenv = Environment(loader=FileSystemLoader(template_dir), keep_trailing_newline=keep_trailing_newline) text = jenv.get_template(filename).render(**context or {}) # Force to a byte representation of Unicode, or str()ification # within Paramiko's SFTP machinery may cause decode issues for # truly non-ASCII characters. text = text.encode('utf-8') except ImportError: import traceback tb = traceback.format_exc() abort(tb + "\nUnable to import Jinja2 -- see above.") else: if template_dir: filename = os.path.join(template_dir, filename) filename = apply_lcwd(filename, env) with open(os.path.expanduser(filename)) as inputfile: text = inputfile.read() if context: text = text % context # Back up original file if backup and exists(destination): func("cp %s{,.bak}" % _expand_path(destination)) # Upload the file. return put(local_path=StringIO(text), remote_path=destination, use_sudo=use_sudo, mirror_local_mode=mirror_local_mode, mode=mode, temp_dir=temp_dir)
def editfile(editor, rfile, usesudo=False): try: if len(hosts) > 1: hostsdict = dict() index = 1 for host in hosts: hostsdict[index] = host index += 1 for key in hostsdict.keys(): print '%s%s) %s%s' % (termcolors.YELLOW, str(key), hostsdict[key], termcolors.END) try: hostkey = int( raw_input( '%sPlease choose the host to perform the edit action: %s' % (termcolors.BLUE, termcolors.END))) if hostkey in hostsdict.keys(): host = hostsdict[hostkey] else: nothingtodo() return except: nothingtodo() return else: host = hosts[0] filename = taningiashelltmpdir + '-%s' % rfile.replace('/', '-') fabric.env.host_string = host fabric.env.warn_only = True if usesudo: with fabric.hide('running', 'output'): fabric.sudo('cp %s /tmp/%s' % (rfile, sessionid)) with fabric.show('running', 'output'): fabric.get('/tmp/%s' % sessionid, filename) fabric.sudo('rm /tmp/%s' % sessionid) else: fabric.get(rfile, filename) os.system('%s %s' % (editor, filename)) print '''%s1) Push the file to all %s hosts. 2) Push the file to the host you choosed only. 3) Do nothing. %s''' % (termcolors.YELLOW, len(hosts), termcolors.END) try: editaction = int( raw_input( '%sPlease choose the action you want to do for %s: %s' % (termcolors.BLUE, rfile, termcolors.END))) if editaction == 1: for host in hosts: fabric.env.host_string = host fabric.env.warn_only = True fabric.put(filename, rfile, use_sudo=usesudo) elif editaction == 2: fabric.put(filename, rfile, use_sudo=usesudo) else: nothingtodo() filemd5sum = hashlib.md5(open(filename).read()).hexdigest() shutil.move(filename, taningiashelltmpdir + '/' + filemd5sum) if usesudo: commands[len(commands) + 1] = (rfile, filemd5sum, 'SUDO') else: commands[len(commands) + 1] = (rfile, filemd5sum) except: nothingtodo() except fabexceptions.NetworkError: networkexception()
def as_sudo(self, command, show=True): if show: self.print_command(command) with hide("running"): return sudo(command)
def sed(filename, before, after, limit='', use_sudo=False, backup='.bak', flags='', shell=False): """ Run a search-and-replace on ``filename`` with given regex patterns. Equivalent to ``sed -i<backup> -r -e "/<limit>/ s/<before>/<after>/<flags>g" <filename>``. Setting ``backup`` to an empty string will, disable backup file creation. For convenience, ``before`` and ``after`` will automatically escape forward slashes, single quotes and parentheses for you, so you don't need to specify e.g. ``http:\/\/foo\.com``, instead just using ``http://foo\.com`` is fine. If ``use_sudo`` is True, will use `sudo` instead of `run`. The ``shell`` argument will be eventually passed to `run`/`sudo`. It defaults to False in order to avoid problems with many nested levels of quotes and backslashes. However, setting it to True may help when using ``~fabric.operations.cd`` to wrap explicit or implicit ``sudo`` calls. (``cd`` by it's nature is a shell built-in, not a standalone command, so it should be called within a shell.) Other options may be specified with sed-compatible regex flags -- for example, to make the search and replace case insensitive, specify ``flags="i"``. The ``g`` flag is always specified regardless, so you do not need to remember to include it when overriding this parameter. .. versionadded:: 1.1 The ``flags`` parameter. .. versionadded:: 1.6 Added the ``shell`` keyword argument. """ func = use_sudo and sudo or run # Characters to be escaped in both for char in "/'": before = before.replace(char, r'\%s' % char) after = after.replace(char, r'\%s' % char) # Characters to be escaped in replacement only (they're useful in regexen # in the 'before' part) for char in "()": after = after.replace(char, r'\%s' % char) if limit: limit = r'/%s/ ' % limit context = { 'script': r"'%ss/%s/%s/%sg'" % (limit, before, after, flags), 'filename': _expand_path(filename), 'backup': backup } # Test the OS because of differences between sed versions with hide('running', 'stdout'): platform = run("uname", shell=False, pty=False) if platform in ('NetBSD', 'OpenBSD', 'QNX'): # Attempt to protect against failures/collisions hasher = hashlib.sha1() hasher.update(env.host_string) hasher.update(filename) context['tmp'] = "/tmp/%s" % hasher.hexdigest() # Use temp file to work around lack of -i expr = r"""cp -p %(filename)s %(tmp)s \ && sed -r -e %(script)s %(filename)s > %(tmp)s \ && cp -p %(filename)s %(filename)s%(backup)s \ && mv %(tmp)s %(filename)s""" else: context['extended_regex'] = '-E' if platform == 'Darwin' else '-r' expr = r"sed -i%(backup)s %(extended_regex)s -e %(script)s %(filename)s" command = expr % context return func(command, shell=shell)
def _check_pgp_key(path, keyid): with settings(hide('everything')): return not run( 'gpg --with-colons %(path)s | cut -d: -f 5 | grep -q \'%(keyid)s$\'' % locals())
def create(): """ Creates the environment needed to host the project. The environment consists of: system locales, virtualenv, database, project files, SSL certificate, and project-specific Python requirements. """ # Generate project locale locale = env.locale.replace("UTF-8", "utf8") with hide("stdout"): if locale not in run("locale -a"): sudo("locale-gen %s" % env.locale) sudo("update-locale %s" % env.locale) sudo("service postgresql restart") run("exit") # Create project path run("mkdir -p %s" % env.proj_path) # Set up virtual env run("mkdir -p %s" % env.venv_home) with cd(env.venv_home): if exists(env.proj_name): if confirm("Virtualenv already exists in host server: %s" "\nWould you like to replace it?" % env.proj_name): run("rm -rf %s" % env.proj_name) else: abort() run("virtualenv %s" % env.proj_name) # Upload project files if env.deploy_tool in env.vcs_tools: vcs_upload() else: rsync_upload() # Create DB and DB user pw = db_pass() user_sql_args = (env.proj_name, pw.replace("'", "\'")) user_sql = "CREATE USER %s WITH ENCRYPTED PASSWORD '%s';" % user_sql_args psql(user_sql, show=False) shadowed = "*" * len(pw) print_command(user_sql.replace("'%s'" % pw, "'%s'" % shadowed)) psql("CREATE DATABASE %s WITH OWNER %s ENCODING = 'UTF8' " "LC_CTYPE = '%s' LC_COLLATE = '%s' TEMPLATE template0;" % (env.proj_name, env.proj_name, env.locale, env.locale)) # Set up SSL certificate if not env.ssl_disabled: conf_path = "/etc/nginx/conf" if not exists(conf_path): sudo("mkdir %s" % conf_path) with cd(conf_path): crt_file = env.proj_name + ".crt" key_file = env.proj_name + ".key" if not exists(crt_file) and not exists(key_file): try: crt_local, = glob(join("deploy", "*.crt")) key_local, = glob(join("deploy", "*.key")) except ValueError: parts = (crt_file, key_file, env.domains[0]) sudo("openssl req -new -x509 -nodes -out %s -keyout %s " "-subj '/CN=%s' -days 3650" % parts) else: upload_template(crt_local, crt_file, use_sudo=True) upload_template(key_local, key_file, use_sudo=True) # Install project-specific requirements upload_template_and_reload("settings") with project(): if env.reqs_path: pip("-r %s/%s" % (env.proj_path, env.reqs_path)) pip("gunicorn setproctitle psycopg2 " "django-compressor python-memcached") # Bootstrap the DB manage("createdb --noinput --nodata") python( "from django.conf import settings;" "from django.contrib.sites.models import Site;" "Site.objects.filter(id=settings.SITE_ID).update(domain='%s');" % env.domains[0]) for domain in env.domains: python("from django.contrib.sites.models import Site;" "Site.objects.get_or_create(domain='%s');" % domain) if env.admin_pass: pw = env.admin_pass user_py = ("from django.contrib.auth import get_user_model;" "User = get_user_model();" "u, _ = User.objects.get_or_create(username='******');" "u.is_staff = u.is_superuser = True;" "u.set_password('%s');" "u.save();" % pw) python(user_py, show=False) shadowed = "*" * len(pw) print_command(user_py.replace("'%s'" % pw, "'%s'" % shadowed)) return True
def run_command(self, command, show=True): if show: self.print_command(command) with hide("running"): return run(command)
def is_type(fstype, mountpoint): with hide("everything"): res = run("df -t %s | awk 'NR>=2{print $NF}'" % (fstype)) return mountpoint in res.split()