def upload_kits(version, upload_type="ent", remote_dir="/var/www/downloads.voltdb.com/technologies/server", kit_dir="/home/test/releases/released", dry_run='False'): """ Upload kits to the voltdb download site. Args: version - version number such as 7.1.2 upload_type - upload types are "ent" (only ent kit) and "all" for all kits remote_dir - directory to upload to kits_dir - directory to pull kits from dry_run - if true, the upload will not be done """ if not version: exit ("FATAL: You must specify a version number that exists in ~tests/releases/released") #Any upload_type starting with ent will work #This allows the Jenkins selector to be "ent_only" upload_type = upload_type[:3] if upload_type not in ("ent", "all"): exit("FATAL: Upload upload_type must be enterprise or all") dry_run = str_option_to_bool(dry_run) ent_kits = ["voltdb-ent-%s.tar.gz","voltdb-ent-%s.SHA256SUM","LINUX-voltdb-ent-%s.tar.gz","MAC-voltdb-ent-%s.tar.gz"] all_kits = ent_kits + ["voltdb-pro-%s.tar.gz","voltdb-pro-%s.SHA256SUM", "voltdb-%s.tar.gz", "voltdb-community-%s.tar.gz"] kits_home = kit_dir + "/voltdb-" + version count=0 with quiet(): dircheck = local("ls -d " + kits_home, capture=True) if dircheck.failed: exit ("FATAL: " + dircheck.stderr) #For all the files in ent_kits or all_kits upload the files for k in eval(upload_type + "_kits"): f = os.path.join(kits_home, k) % version #print "====testing" + f with quiet(): filecheck = local("ls " + f) if filecheck.succeeded: count += 1 if dry_run: print "DRYRUN:" + f + " --> " + remote_dir else: with cd(remote_dir): put(f, k % version) if not count: print "Maybe the kits in kits_home do not match version " + version print "This can happen if your tag does not match your version.txt" print print "Maybe these files need to be renamed:" local("ls " + kits_home) exit ("FATAL: No files uploaded")
def ssh_cmd(host, cmd, options={}): if host == "localhost": with settings(quiet()): output = local(cmd, capture=True) else: with settings(quiet(), use_ssh_config = True, host_string = host): output = run(cmd) return output
def generate_ssh_keys(self): for ssh_key_name, ssh_keygen_args in sorted(self.ssh_keys): if not exists(self.custom_template_path): os.mkdir(self.custom_template_path) ssh_key = join(self.custom_template_path, ssh_key_name) if exists(ssh_key): continue with settings(quiet(), warn_only=True): result = local("ssh-keygen %s -f %s -N ''" % (ssh_keygen_args, ssh_key), capture=True) if result.failed: continue with settings(quiet()): fingerprint = local("ssh-keygen -lf %s" % ssh_key, capture=True).split()[1] print("Generated %s with fingerprint %s." % (ssh_key_name, fingerprint))
def jobs(): local_jobs_dir = mkdtemp() # Download all job infos TODO: is there better way? with quiet(): rsync_project(remote_dir=env.jobs_path + '/', local_dir=local_jobs_dir, upload=False) table = [] table.append(['project', 'spider', 'job', 'state', 'items', 'requests', 'errors']) running = set() with quiet(): docker_ps = run('sudo docker ps | grep scrapy---') for line in docker_ps.splitlines(): container_id = line.split()[-1] _, project, spider, job = container_id.split('---') running.add((project, spider, job)) for project in sorted(listdir(local_jobs_dir)): project_path = join(local_jobs_dir, project) for spider in sorted(listdir(project_path)): spider_path = join(project_path, spider) for job in sorted(map(int, listdir(spider_path)), reverse=True): job = str(job) items = '-' requests = '-' errors = '-' state = 'unknown' job_path = join(spider_path, job) with open(job_path) as f: try: job_info = json.load(f) except ValueError: pass else: stats = job_info['stats'] if 'finish_reason' in stats: state = 'done ({})'.format(stats['finish_reason']) elif (project, spider, job) in running: state = 'running' items = stats.get('item_scraped_count', 0) requests = stats.get('response_received_count', 0) errors = stats.get('log_count/ERROR', 0) table.append([project, spider, job, state, items, requests, errors]) pprint_table(sys.stdout, table)
def check_java_checkstyle(file_path, config_path=None): if not file_path.endswith(".java"): return True, None if not os.path.exists(CHECKSTYLE_JAR_PATH): with lcd(os.path.join(OTTERTUNE_DIR, "client/controller")): # pylint: disable=not-context-manager local("gradle downloadJars") options = '' if config_path is None else '-c ' + config_path with quiet(): res = local("java -jar {} {} {}".format(CHECKSTYLE_JAR_PATH, options, file_path), capture=True) lines = res.stdout.split('\n') assert len(lines) >= 2 and lines[0] == "Starting audit..." and lines[-1] == "Audit done." if len(lines) == 2: return True, None output = [] for line in lines[1:-1]: parts = line.strip().split(':') line_number = int(parts[1]) text, code = parts[-1].rsplit('[', 1) text = text.strip() code = code[:-1] output.append(format_message(os.path.basename(file_path), line_number, text, code)) output = ''.join(output) return False, output
def check_pylint(file_path, config_path=None): if not file_path.endswith(".py"): return True, None options = [ '--output-format=json', '--reports=yes', ] if config_path is not None: options.append('--rcfile=' + config_path) with settings(warn_only=True), quiet(): res = local('pylint {} {}'.format(' '.join(options), file_path), capture=True) if res.stdout == '': assert res.return_code == 0, 'return_code={}, expected=0\n{}'.format( res.return_code, res.stderr) return True, None output = [] errors = json.loads(res.stdout) for entry in errors: # Remove extra whitespace and hints msg = entry['message'].replace('^', '').replace('|', '') msg = re.sub(' +', ' ', msg) msg = msg.strip() output.append(format_message(os.path.basename(file_path), entry['line'], msg, entry['symbol'])) output = ''.join(output) return res.return_code == 0, output
def _validate_puppet(self, recurse=False): """ when recurse==True, all puppet under _ymir_service_root/puppet will be checked otherwise, only validate the files mentioned in SETUP_LIST / PROVISION_LIST """ errs = [] pdir = self._puppet_dir if not os.path.exists(pdir): msg = 'puppet directory does not exist @ {0}' msg = msg.format(pdir) errs.append(msg) else: with quiet(): parser = self._template_data().get('puppet_parser', '') validation_cmd = 'puppet parser {0} validate '.format( '--parser {0}'.format(parser) if parser else '') result = local('find {0}|grep .pp$'.format(pdir), capture=True) for filename in result.split('\n'): (" .. validating {0}".format(filename)) result = local('{1} {0}'.format( filename, validation_cmd), capture=True) error = result.return_code != 0 if error: short_fname = filename.replace(os.getcwd(), '.') error = "running `{1} {0}'".format( short_fname, validation_cmd) errs.append(error) return errs
def test(): 'Test differences in rendering.' reference = 'reference.txt' output = 'output.txt' with lcd(TESTS_DIR): def curl(path): local('curl http://localhost:5000%s >> %s' % (path, output)) stdout.write('.') stdout.flush() with quiet(): local('echo > ' + output) curl('/') curl('/feed') curl('/posts') for root, dirnames, filenames in walk(POSTS_DIR): for filename in sorted(filenames): if filename.startswith('.'): continue if not filename.endswith('.md'): continue curl('/posts/%s' % filename[:-len('.md')]) stdout.write('\n') stdout.flush() local('git diff --no-index -- %s %s' % (reference, output))
def initialize_project(self): logger.info('Intializing local worker environment') with quiet(): local('virtualenv -p python2.7 env') local('PATH=/usr/lib/ccache:/usr/lib64/ccache/bin:$PATH ' 'env/bin/pip install ' '--download-cache /tmp/pip -r requirements.txt')
def _hotswap(file, type, keywords): file_with_full_path = '/app/online/{}{}'.format(GAME, file) file_path = os.path.dirname(file_with_full_path) #local('rsync -aqP {}/{{hotswap.zip,md5.txt}} {}@{}:{}/'.format(file_path, env.user, env.host_string, REMOTE_DIR)) run('mkdir -p {}'.format(REMOTE_DIR)) with lcd(file_path): put('hotswap.zip', REMOTE_DIR) put('md5.txt', REMOTE_DIR) with cd(REMOTE_DIR): run('dos2unix md5.txt && md5sum -c md5.txt') run('unzip -q hotswap.zip') run('cd hotswap && chmod +x attach remote update') ret_value = {} for gameServer in LOCATE_GAME_SRVS[env.host_string]: with cd('/app/{}/backend/logs'.format(gameServer)): run('echo >start.out') with cd('{}/hotswap'.format(REMOTE_DIR)): pid = gameServer_pid(gameServer) run('./{} {}'.format(type, pid)) with cd('/app/{}/backend/logs'.format(gameServer)): for each_keyword in keywords.split(','): with quiet(): do_hotswap_success = run('grep --color=never -E -A 20 "reload.*{}" {} | grep --color=never "reload succ"'.format(each_keyword, 'start.out')).succeeded if not do_hotswap_success: ret_value[gameServer] = False break else: ret_value[gameServer] = True return ret_value
def realmem(self): import math with settings(quiet()): realmem = run("sysctl -n hw.realmem").strip() realmem = float(realmem) / 1024 / 1024 return 2 ** int(math.ceil(math.log(realmem, 2)))
def _all_gameServer_info(self): """ Get all self.game info. It will get a dict like: { 'astd_17wan_1': '10.6.120.23', 'astd_37wan_98': '10.4.5.5', . . . 'astd_37wan_8': '10.4.5.15'} """ #with quiet(): # result_info = local('''/app/opbin/work/global/getserverlist -g %s|awk -F@ -v OFS=@ '{print $1,$2}' ''' % self.game, capture=True) #info_list = result_info.splitlines() #info_dict = { each.split('@')[0]:each.split('@')[1] for each in info_list } with quiet(): result_info = local('''/app/opbin/work/bible/main.py serverlist -g %s -l %s -s '.*' ''' % (self.game, self.region), capture=True) info_list = result_info.splitlines() info_dict = {} for each in info_list: name, ip = each.split('@') info_dict['{}_{}'.format(self.game, name)] = ip return info_dict
def run(self, src='prod', dest='local', *args, **kwargs): """ Synchronizes the unversioned folders from one environment to another. (src: prod, dest: local) Typically this is used to download the WordPress uploads folder from the production server, as well as any other folders defined in the UNVERSIONED_FOLDERS config value. Note that this function DOES NOT copy/transfer any of the application code - that must be done instead using the "deploy" task. Example usage: - `fab rsync # Default params, same as following command.` - `fab rsync:prod,local # Downloads unversioned files from the production to the local server.` - `fab rsync:local,prod # NOT RECOMMENDED - have not developed/tested this yet.` - `fab rsync:local,dev # NOT RECOMMENDED - have not developed/tested this yet.` - `fab rsync:prod,dev # NOT RECOMMENDED - have not tested this, nor is it necessary UNLESS the dev server is on a different server than the prod server. Also, not sure rsync supports one remote to another.` """ for dir in env.conf.unversioned_folders: cmd_vars = { 'src_host': env[src]['hostname'], 'dest_host': env[dest]['hostname'], 'root': env[src]['root'], 'dest_root': env[dest]['root'], 'dir': dir, 'extra_options': '--cvs-exclude', } if src == 'local': cmd = 'rsync -ravz %(extra_options)s %(root)s/%(dir)s/ %(dest_host)s:%(dest_root)s/%(dir)s' % cmd_vars else: cmd = 'rsync -ravz %(extra_options)s %(src_host)s:%(root)s/%(dir)s/ %(dest_root)s/%(dir)s' % cmd_vars print('Syncing unversioned files...') with quiet(): local(cmd)
def _local_resolver(domain): with quiet(): ret_value = local('''nslookup %s |grep "^Address" |grep -v '#'|awk '{print $2}' ''' % domain, capture=True) if ret_value: return ret_value else: raise Exception('Fail to resolve {} to ip address'.format(domain))
def main(): parser = argparse.ArgumentParser( prog='pyrsnapshot', description='''\ Pyrsnapshot will back up the contents of the current directory to the remote server and save it under the path given. Use the executing user's .ssh/ssh_config to configure the user and private key for the connection. ''', ) help = 'Keep how many {} backups (default: {})' for adjective, default in zip(remote.ADJECTIVES, remote.DEFAULTS): parser.add_argument( '--{}'.format(adjective), type=int, default=default, help=help.format(adjective, default) ) parser.add_argument( 'remote', metavar='host:path', help='Backup to the remote host at the given path' ) try: args = parser.parse_args() host, root = args.remote.split(':') if not root: raise Exception("You must specify the remote path to backup to.") with quiet(): env.use_ssh_config = True execute(remote.pyrsnapshot, root, args, hosts=[host]) finally: disconnect_all()
def check_working_copy(): """ Check if working copy is clean. """ with quiet(): if local('git diff --quiet').failed: abort(red('Working copy is not clean'))
def release(new_version): with quiet(): local('git commit -am "new version {}"'.format(new_version)) local('git tag -a v{0} -m \'new version {0}\''.format(new_version)) local('git push origin master --tags') local("python setup.py register") local("python setup.py sdist upload -r pypi")
def check_bucket_permissions(): with quiet(): result = local( 'aws lambda get-policy' + ' --function-name ' + LAMBDA_FUNCTION_NAME + AWS_CLI_STANDARD_OPTIONS, capture=True ) if result.failed or result == '': return False result_decoded = json.loads(result) if not isinstance(result_decoded, dict): return False policy = json.loads(result_decoded.get('Policy', '{}')) if not isinstance(policy, dict): return False statements = policy.get('Statement', []) for s in statements: if s.get('Sid', '') == BUCKET_PERMISSION_SID: return True return False
def own_django_log(): """ make sure logs/django.log is owned by www-data""" require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) with quiet(): log_path = os.path.join(env.path, 'logs', 'django.log') if run('ls {}'.format(log_path)).succeeded: sudo('chown www-data:www-data {}'.format(log_path))
def test_enable_password_auth(sshd_config): from fabtools.ssh import enable_password_auth enable_password_auth(sshd_config=SSHD_CONFIG) with quiet(): assert contains(SSHD_CONFIG, 'PasswordAuthentication yes', exact=True) assert not contains(SSHD_CONFIG, 'PasswordAuthentication no', exact=True)
def decorator(*args, **kwargs): # with shell_env(**config): # if 'environ' in inspect.getargspec(f)[0]: # kwargs['environ'] = config # with quiet(): # return f(*args, **kwargs) with quiet(): return f(*args, **kwargs)
def sh_mktemp(base_path, directory=True, **kwargs): with quiet(): args = ['--tmpdir={0}'.format(base_path)] if directory: args.append('-d') args = ' '.join(args) return run_cmd('mktemp {0}'.format(args), **kwargs)
def reset(self, remote=None, branch=None, hard=False, files=None): with quiet(): args = ["%s/%s" % (remote, branch)] if hard: args.append("--hard") if files: args.append(" ".join(files)) self.git("reset", args)
def logs(): '''delete cassandra logs''' with quiet(): sudo("rm -f {0}/*".format(LOG_FOLDER)) for i in [YCSB_RUN_OUT_FILE, YCSB_RUN_ERR_FILE, YCSB_LOAD_OUT_FILE, YCSB_LOAD_ERR_FILE]: sudo("rm {0}".format(i)) sudo("rm -f {0} {1}".format(DSTAT_SERVER,DSTAT_YCSB))
def process_untracked_files(self): with quiet(): untracked_files = local(self.git("ls-files --other --exclude-standard"), capture=True) if untracked_files: answer = prompt(cyan("You have untracked files. Add them?(y/n):"), default="n") if answer == "y": self.git("add", untracked_files.splitlines()) self.commit()
def ubuntu_update(): """Update Ubuntu """ with quiet(): run('aptitude update') upgrade = run('apt-get -y upgrade') if upgrade.failed is False: puts('Distribution up to date')
def upload_file(local_file, remote_path, mode=0o644, owner="root:root", temp_dir="", changelog=False): """Upload a file to a remote host. ``local_file`` can be a filename or a seekable file-like object. Globbing is not supported. ``remote_file`` should be a full filename, not just the directory. ``mode`` can be an integer (e.g. 0o755). ``changelog``, if True, adds a changelog message of the form "uploaded {filename}". Bug: doesn't handle ``with cd(...):`` or ``with lcd(...):``. Probably. Bug: doesn't set mode/ownership if the file exists and has the same content but different mode/ownership. Warning: is not suitable for uploading secrets (changes the mode after uploading the file), unless you take care to specify ``temp_dir`` to point to a non-world-readable area. Undocumented features that are subject to change without notice: ``mode`` can be a string or None; ``owner`` can be None. """ if isinstance(mode, int): mode = '{:o}'.format(mode) assert_shell_safe(remote_path, mode or '', temp_dir) assert_shell_safe(owner or '', extra_allow=':') local_is_path = not callable(getattr(local_file, 'read', None)) if isinstance(local_file, StringIO) and not getattr(local_file, 'name', None): local_file.name = os.path.basename(remote_path) with closing(SFTP(env.host_string)) as ftp: if env.get('cwd'): home = ftp.normalize('.') temp_dir = posixpath.join(home, temp_dir) tmp_path = posixpath.join( temp_dir, hashlib.sha1(env.host_string + remote_path).hexdigest()) assert_shell_safe(tmp_path) ftp.put(local_file, tmp_path, use_sudo=False, mirror_local_mode=False, mode=None, local_is_path=local_is_path, temp_dir="") with quiet(): same = sudo("test -f {realfile} && cmp -s {tempfile} {realfile}".format( tempfile=tmp_path, realfile=remote_path)).succeeded if same: sudo("rm {tempfile}".format(tempfile=tmp_path)) return False else: if mode is not None: sudo('chmod {mode} {tempfile}'.format(mode=mode, tempfile=tmp_path)) if owner: sudo('chown {owner} {tempfile}'.format(owner=owner, tempfile=tmp_path)) sudo("mv {tempfile} {realfile}".format(tempfile=tmp_path, realfile=remote_path)) if changelog: changelog_append("# updated {}".format(remote_path)) return True
def get_postfix_setting(setting): """Get the current value of a postfix setting""" assert_shell_safe(setting) with quiet(): current_setting = run("postconf -h {setting}".format(setting=setting)) if current_setting.startswith('postconf: warning:'): # assume "postconf: warning: {setting}: unknown parameter" current_setting = '' return current_setting
def prepare_server(): has_docker = False with quiet(): has_docker = run('docker --version').succeeded if not has_docker: run('sudo apt-get install -y docker.io') run('mkdir -p {} {} {} {} {}'.format( env.project_path, env.items_path, env.logs_path, env.jobs_path, env.files_path))
def package_installed(package): """Check if the specified packages is installed.""" assert_shell_safe(package) # XXX: doing this in a loop is slow :( with quiet(): # XXX idea: return exists('/var/lib/dpkg/info/{}.list'.format(package)) # caveats: libnss-myhostname:amd64.list :/ status = run("dpkg-query -W --showformat='${Status}' %s" % package) return status == "install ok installed"
def attach(self, instance, user="******"): """ attach volume or snapshot """ if isinstance(instance, str): instance = aws.get(instance, collections=aws.ec2.instances) fab.env.host_string = instance.public_ip_address fab.env.user = user volume = aws.get(self.name, collections=aws.ec2.volumes) if volume: # validate volume if volume.availability_zone != \ instance.placement["AvailabilityZone"]: raise Exception("volume and instance must be in same " "availability zone") else: # create volume from snapshot snapshot = self.latest_snapshot() if snapshot is None: raise Exception("No volume or snapshot found " "for %s" % self.name) r = aws.client.create_volume( SnapshotId=snapshot.id, AvailabilityZone=instance.placement["AvailabilityZone"], VolumeType="gp2") volume = aws.ec2.Volume(r["VolumeId"]) aws.set_name(volume, self.name) # remove existing attachment if volume.attachments: self.detach() # wait until available while True: item = aws.client.describe_volumes( VolumeIds=[volume.id])["Volumes"][0] if item["State"] == "available": break log.info("waiting until volume available") sleep(15) log.info("volume available") # attach instance.attach_volume(VolumeId=volume.id, Device='/dev/xvdf') # wait until usable. while True: with fab.quiet(): if fab.sudo("ls -l /dev/xvdf").succeeded: break log.info("waiting until volume visible") sleep(1) log.info("volume attached")
def own_django_log(): """ make sure logs/django.log is owned by www-data""" if is_vagrant(): return require('srvr', 'path', provided_by=env.servers) with quiet(): log_path = os.path.join(env.path, 'logs', 'django.log') if exists(log_path): sudo('chown www-data:www-data {}'.format(log_path)) sudo('chmod g+rw {}'.format(log_path))
def wait_ssh(): """ wait for ssh server """ log.info("waiting for ssh server") while True: with fab.quiet(): try: fab.sudo("ls") break except: pass sleep(1) log.info("ssh connected %s" % fab.env.host_string)
def generate_ssl_keystore(self, root_certificate, keystore_file, storepass, worker_home): logger.info('Generating SSL keystore') remote_keystore = "{}/perfrunner/{}".format(worker_home, keystore_file) remote_root_cert = "{}/perfrunner/{}".format(worker_home, root_certificate) put(root_certificate, remote_root_cert) with quiet(): run("keytool -delete -keystore {} -alias couchbase -storepass storepass" .format(remote_keystore)) run("keytool -importcert -file {} -storepass {} -trustcacerts " "-noprompt -keystore {} -alias couchbase" .format(remote_root_cert, storepass, remote_keystore))
def update_index(force=False): """ Update pkgin package definitions. """ manager = MANAGER if force: with quiet(): # clean the package cache run_as_root("%(manager)s clean" % locals()) run_as_root("%(manager)s -f update" % locals()) else: run_as_root("%(manager)s update" % locals())
def test_abort_message_only_printed_once(): """ abort()'s SystemExit should not cause a reprint of the error message """ # No good way to test the implicit stderr print which sys.exit/SystemExit # perform when they are allowed to bubble all the way to the top. So, we # invoke a subprocess and look at its stderr instead. with quiet(): result = local("fab -f tests/support/aborts.py kaboom", capture=True) # When error in #1318 is present, this has an extra "It burns!" at end of # stderr string. eq_(result.stderr, "Fatal error: It burns!\n\nAborting.")
def create_virtualenv(): require('srvr', 'path', 'within_virtualenv', provided_by=env.servers) env_vpath = get_virtual_env_path() with quiet(): if run('ls {}'.format(env_vpath)).succeeded: print(green( 'virtual environment at [{}] exists'.format(env_vpath))) return # All we need is a .venv dir in the project folder; # 'pipenv install' will set it up first time print(yellow('setting up virtual environment in [{}]'.format(env_vpath))) run('mkdir {}'.format(env_vpath))
def unmount(self): """ unmount """ with fab.quiet(): r = fab.sudo("umount /v1") if r.succeeded: log.info("volume dismounted") else: log.warning("dismount failed. trying to force.") r = fab.sudo("fuser -km /v1") if r.succeeded: log.info("volume dismounted") else: log.warning("failed to force dismount")
def status_rabbitmq(): with settings(warn_only=True), quiet(): #res = sudo('rabbitmqctl status', pty=False) res = local('sudo rabbitmqctl status') if res.return_code == 2 or res.return_code == 69: status = STATUS.STOPPED elif res.return_code == 0: status = STATUS.RUNNING else: raise Exception("Rabbitmq: unknown status " + str(res.return_code)) print status print_status(status, 'rabbitmq') return status
def allow(user, port): path = os.path.join('/etc/authbind/byport', str(port)) needsUpdate = True with quiet(): state = run('/usr/bin/stat -c %U:%a {}'.format(path)) needsUpdate = state.strip().split(':') != [user, '500'] if needsUpdate: if not hasSudoCapabilities(): abort('Trying to give {} access to port {} but have insufficient ' 'capabilities.'.format(user, port)) sudo('/bin/touch {}'.format(path)) sudo('/bin/chown {0}:{0} {1}'.format(user, path)) sudo('/bin/chmod 0500 {}'.format(path))
def docker_check_service_base(): """ Check if the consul image exists, build it if not. .. versionadded:: 0.2 """ with quiet(): log.info('Checking if we have a consul image ...') env.arch = get_local_arch() state = sudo('%(docker)s inspect service-base:test' % env) if state.return_code == 1: docker_generate_service_base()
def sh_chown(path, user=None, group=None, recursive=False, **kwargs): if user: mask = '{0}'.format(user) if group: mask = '{0}:{1}'.format(mask, group) with quiet(): extra_args = '' if recursive: extra_args = ' -R' return run_cmd('chown {0} {1} {2}'.format(extra_args, mask, path), capture=True, **kwargs)
def up_to_date(env, cmd, version, args=None, stdout_flag=None): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): out = env.safe_run_output(cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag) else: iversion = out.strip() return LooseVersion(iversion) >= LooseVersion(version)
def get_commit_id(): '''Find the commit id of the checked out repo we are in.''' id = '' if fs.sh_which('svn'): with quiet(): content = run_cmd('svn info') if content: detected = text.safefind('(?m)^Revision: ([0-9]+)', content) if detected: id = detected if not id: if fs.sh_which('git'): with quiet(): content = run_cmd('git svn info') if content: detected = text.safefind('(?m)^Revision: ([0-9]+)', content) if detected: id = detected return id
def install_devices(self): mounts = self.mounts with settings(quiet()): cd_device = env.instance.config.get('bootstrap-cd-device', 'cd0') if '/dev/{dev} on /rw/cdrom'.format(dev=cd_device) not in mounts: run('test -e /dev/{dev} && mount_cd9660 /dev/{dev} /cdrom || true' .format(dev=cd_device)) usb_device = env.instance.config.get('bootstrap-usb-device', 'da0a') if '/dev/{dev} on /rw/media'.format(dev=usb_device) not in mounts: run('test -e /dev/{dev} && mount -o ro /dev/{dev} /media || true' .format(dev=usb_device)) return [cd_device, usb_device]
def __init__(self, filename, remote_dir=REMOTE_DIR): self.filename = filename self.dir = run('pwd') with quiet(): has_the_file = run('test -f {}'.format(filename)).succeeded if not has_the_file: raise Exception('File {}/{} NOT exists'.format(self.dir, filename)) tmp_tag = self.filename.split('/')[0] backup_dir = '{}/{}'.format(remote_dir, tmp_tag) run('[ -d {0} ] || mkdir -p {0}'.format(backup_dir)) run('cp {} {}/'.format(self.filename, backup_dir))
def wait_ssh(): """ wait for ssh server """ apps.setdebug() log.info("waiting for ssh server") while True: with fab.quiet(): try: r = fab.sudo("ls") except: pass if r.succeeded: break sleep(1) log.info("ssh connected %s"%fab.env.host_string)
def release(): with quiet(): version_row = local("grep 'VERSION = ' setup.py", capture=True) version = version_row.split(' = ')[1].strip() version = version[1:-1].split('.') version[-1] = str(int(version[-1]) + 1) new_version = '.'.join(version) local("sed -isetup.py 's/VERSION =.*/VERSION = \"{0}\"/g' setup.py". format(new_version)) local('git commit -am "new version {0}"'.format(new_version)) local('git tag -a v{0} -m \'new version {0}\''.format(new_version)) local('git push origin master --tags') local("python setup.py register") local("python setup.py sdist upload -r pypi")
def __init__(self, filename, remote_dir=REMOTE_DIR): self.filename = filename with quiet(): has_the_file = run('test -f {}'.format(filename)).succeeded if not has_the_file: raise Exception('File {} NOT exists under backend/apps/'.format(filename)) self.dir = run('pwd') gameServer = self.dir.split('/')[2] backup_dir = '{}/{}'.format(remote_dir, gameServer) run('[ -d {0} ] || mkdir -p {0}'.format(backup_dir)) run('cp {} {}/'.format(self.filename, backup_dir))
def pull_db(dump='backup'): app_env = honcho.environ.parse(open('.env').read()) remote_db = dj_database_url.parse(app_env['REAL_DATABASE_URL']) local_db = dj_database_url.parse(app_env['DATABASE_URL']) # Make and download database dump if dump == 'direct': # Dump directly from remote database with local pg_dump print('Making database dump...') local(DUMP_COMMAND % remote_db) elif dump == 'app': # Alternative: dump to app-server than rsync here, # useful with slow or flaky internet connection. print('Making database dump...') run(DUMP_COMMAND % remote_db) print('Downloading dump...') local( 'rsync -avP stargeo:/home/ubuntu/app/stargeo.sql.gz stargeo.sql.gz' ) run('rm stargeo.sql.gz') elif dump == 'backup': # Alternative: fetch latests db backup local( 'rsync -avP stargeo:/home/ubuntu/db-backups/stargeo.sql.gz stargeo.sql.gz' ) elif dump == 'local': print('Using local dump...') if not os.path.exists('stargeo.sql.gz'): print( red('Local database dump not found (stargeo.sql.gz).\n' 'Please use "remote" or "app" dump.')) return print('Dropping %(NAME)s...' % local_db) with quiet(): local('psql -Upostgres -c "drop database if exists %(NAME)s"' % local_db) # Check if database is deleted if local('psql -Upostgres -d %(NAME)s -c ""' % local_db, capture=True).succeeded: print( red('Database not dropped.\n' 'Disconnect all the clients and retry with "fab pull_db:local"' )) return # Load dump local('gzip -cd stargeo.sql.gz | psql -Upostgres -f -')
def set_docker_folder(folder="/var/lib"): """ set location of docker images and containers for xdrive volume = "/v1" """ # create daemon.json settings config = '{"graph":"%s/docker"}' % folder fab.sudo("mkdir -p /etc/docker") fab.put(io.StringIO(config), "/etc/docker/daemon.json", use_sudo=True) # create target folder fab.sudo(f"mkdir -p {folder}/docker") # restart to activate new target folder with fab.quiet(): fab.sudo("service docker restart")
def run(params): """ run container """ with fab.quiet(): r = fab.sudo("nvidia-smi") # gpu if r.succeeded: # nvidia-docker run and save drivers fab.run(f"nvidia-docker run {params}") volumepath = "/v1/var/lib/nvidia-docker/volumes" if exists(volumepath): fab.sudo("cp -r --parents /var/lib/nvidia-docker/volumes /v1") else: # cpu fab.run(f"docker run {params}")
def _download_file(): for game_server in locate_game_servers[env.host_string]: local_path = '{}/{}/'.format(local_root_path, game_server) local('su - astd -c "mkdir -p {}"'.format(local_path)) target_file = '/app/{}/{}'.format(game_server, remote_file) with quiet(): target_file_exists = run( 'test -f {}'.format(target_file)).succeeded if target_file_exists: get(target_file, local_path) else: raise Exception('File {} NOT exists on {}'.format( target_file, game_server)) local('chown -R astd.astd {}'.format(local_root_path))
def matchServer_exists(matchServer, ip): with quiet(): exists = local( '''grep "\\b{}\\b" /etc/hosts '''.format(matchServer)).succeeded if exists: raise Exception( '''The match server {} already exists in /etc/hosts'''.format( matchServer)) else: matchServer_dir_exists = execute(remote_dir_exists, '/app/{}'.format(matchServer), hosts=[ip])[ip] if matchServer_dir_exists: raise Exception( '''The match dir: /app/{} already exists on {}'''.format( matchServer, ip))
def start_workers(): '''starts compute + site workers in the background, use view_workers to look at the output''' with quiet(): if not local('tmux -V').succeeded: print 'Please install tmux before running this command, i.e. "brew install tmux"' return _print("Starting tmux...") if local('tmux has-session -t codalab_workers').succeeded: print green("session already started! fab view_workers to view it.") return if local('./tmux.sh').succeeded: print green("done") else: print red("could not start workers!")
def start(): '''start cassandra in all nodes in order''' with cd(CODE_DIR): for i in range(MAX_RETRIES): cassandra_bin = path.join(CODE_DIR, "bin", "cassandra") sudo("screen -d -m {0} -f".format(cassandra_bin), pty=False) time.sleep(2) with quiet(): res = sudo("pgrep -f 'java.*c[a]ssandra'") if not res.failed: break print('Starting cassandra failed at {0}. Retry {1}.'.format( env.host_string, i)) else: abort('starting cassandra failed at node {0}.'.format( env.host_string))
def distroName(): """ Get the name of the distro. """ with quiet(): lsb = run('/usr/bin/lsb_release --id --short', warn_only=True) if lsb.succeeded: return lsb.lower() distros = [ ('centos', '/etc/centos-release'), ('fedora', '/etc/fedora-release'), ] for distro, sentinel in distros: if succeeds('/usr/bin/test -f {}'.format(sentinel)): return distro
def uninstall(): """ Uninstall Presto after stopping the services on all nodes """ stop() # currently we have two rpm names out so we need this retry with quiet(): ret = sudo('rpm -e presto') if ret.succeeded: print('Package uninstalled successfully on: ' + env.host) return ret = sudo('rpm -e presto-server-rpm') if ret.succeeded: print('Package uninstalled successfully on: ' + env.host)
def checksum(filename, *files_or_dirs): paths = ' '.join(files_or_dirs) # check whether the files have changed (or the checksum file does not exist at all) with quiet(): if not sudo( f'find {paths} -type f -print0 | sort -z | xargs -0 tar cf - | shasum -c {filename}' ).failed: modified = False else: modified = True yield modified # compute checksum for specified paths if modified: sudo( f'find {paths} -type f -print0 | sort -z | xargs -0 tar cf - | shasum > {filename}' )
def schedule(spider, args=''): jobs_dir = join(env.jobs_path, env.project, spider) with quiet(): run('mkdir -p {}'.format(jobs_dir)) all_jobs = run('ls -1 {}'.format(jobs_dir)) jobs = map(int, re.findall('\d+', all_jobs)) if jobs: latest_job = max(jobs) else: latest_job = 0 job = latest_job + 1 job_path = join(jobs_dir, str(job)) run('touch {}'.format(job_path)) items_dir = join(env.items_path, env.project, spider) run('mkdir -p {}'.format(items_dir)) items_path = join(items_dir, str(job) + '.jl') run('touch {}'.format(items_path)) logs_dir = join(env.logs_path, env.project, spider) run('mkdir -p {}'.format(logs_dir)) logs_path = join(logs_dir, str(job) + '.log') run('touch {}'.format(logs_path)) files_dir = join(env.files_path, env.project, spider, str(job)) run('mkdir -p {}'.format(files_dir)) run('sudo docker run -d -t -i ' '--name scrapy---{project}---{spider}---{job} ' '-v {job_path}:/data/job ' '-v {items_path}:/data/items.jl ' '-v {logs_path}:/data/log.log ' '-v {files_dir}:/data/files ' 'scrapy-{project} ' 'scrapy crawl {spider} {args} ' '-s FILES_DIR=/data/files ' '-s JOB_PATH=/data/job ' '-o /data/items.jl ' '--logfile /data/log.log'.format(job_path=job_path, items_path=items_path, logs_path=logs_path, files_dir=files_dir, project=env.project, spider=spider, job=job, args=args))