def ext_virtualenv(abs_source, dest, **kwargs): requirements=[] # Create virtualenv virtual_env = get_filename(dest) subprocess.check_call(['virtualenv', virtual_env]) # Set up environment pip_env = os.environ.copy() pip_env['VIRTUAL_ENV'] = virtual_env pip_env['PATH'] = '{}/bin:{}'.format(virtual_env, pip_env['PATH']) if 'PYTHONHOME' in pip_env: del pip_env['PYTHONHOME'] # Install requirements pip_bin = os.path.join(virtual_env, 'bin', 'pip') pip = subprocess.Popen([pip_bin, 'install', '-r', abs_source], env=pip_env) ret_code = pip.wait() if ret_code != 0: raise subprocess.CalledProcessError(ret_code, pip_bin) return virtual_env
def ext_template(inputhash, relpath, loader, dest, **kwargs): """ Renders a ~tpl file""" output = render(relpath, inputhash, loader) finalfile = get_filename(dest) with open(finalfile, 'w') as outfile: outfile.write(output) return finalfile
def ext_download(loader, inputhash, abs_source, dest, **kwargs): """ Downloads a ~download file""" template = loader.load_template(abs_source) output = template.render(inputhash, loader=loader) # split on whitespace (url, checksum) = output.split(None, 3) finalfile = get_filename(dest) urllib.urlretrieve(url, finalfile) return finalfile
def ext_extract(abs_source, dest, inputhash, loader, **kwargs): contents = yaml.load(render(abs_source, inputhash, loader)) jsonschema.validate(contents, SCHEMA) finalfile = get_filename(dest) url = contents['url'] local_filename = url.split('/')[-1] tmpfile = tempfile.NamedTemporaryFile(prefix=local_filename, suffix='.download', delete=False) to_delete = [tmpfile.name] try: auth = None if 'username' in contents and 'password' in contents: auth = (contents['username'], contents['password']) _download_file(url, tmpfile, auth) if 'type' in contents: extractor = EXTRACTORS[contents['type']] else: for k in EXTRACTORS.keys(): if local_filename.endswith('.' + k): extractor = EXTRACTORS[k] break else: raise TypeError( 'No extractor found for {}'.format(local_filename)) if 'persist' in contents and not contents['persist']: extract_dest = tempfile.mkdtemp(prefix=local_filename, suffix='.sq') to_delete.append(extract_dest) persist = False else: extract_dest = finalfile persist = True extractor(tmpfile.name, extract_dest) if 'copy' in contents: _copy_files(extract_dest, finalfile, contents) return finalfile finally: for f in to_delete: if os.path.isdir(f): shutil.rmtree(f) else: os.remove(f)
def ext_download(loader, inputhash, abs_source, dest, **kwargs): """ Downloads a ~download file""" contents = yaml.load(render(abs_source, inputhash, loader)) jsonschema.validate(contents, SCHEMA) finalfile = get_filename(dest) handle = open(finalfile, 'w') auth = None if 'username' in contents and 'password' in contents: auth = (contents['username'], contents['password']) _download_file(contents['url'], handle, auth) return finalfile
def ext_git(abs_source, dest, inputhash, loader, resources, **kwargs): """ Clones a git repository """ contents = json.loads(render(abs_source, inputhash, loader)) finalfile = get_filename(dest) jsonschema.validate(contents, SCHEMA) url = contents['url'] refspec = None sshkey = None args = '' if 'refspec' in contents: refspec = contents['refspec'] if 'sshkey' in contents: sshkey = contents['sshkey'] if 'args' in contents: args = contents['args'] if sshkey: key = resources[sshkey]() if 'GIT_SSH' in os.environ: git_ssh = os.environ['GIT_SSH'] reset_to_env = True else: git_ssh = 'ssh' reset_to_env = False keyfile = write_temp_file(key, '.key', False) wrapper = write_temp_file(SSH_WRAPPER.format(git_ssh, keyfile), '.sh', True) old_environ = os.environ.copy() try: os.environ['GIT_SSH'] = wrapper repo = _clone_repo(url, finalfile, args) finally: if reset_to_env: os.environ['GIT_SSH'] = git_ssh else: del os.environ['GIT_SSH'] os.remove(keyfile) os.remove(wrapper) else: repo = _clone_repo(url, finalfile, args) if refspec: repo.git.checkout(refspec) return finalfile
def ext_git(abs_source, dest, inputhash, loader, resources, **kwargs): """ Clones a git repository """ contents = yaml.load(render(abs_source, inputhash, loader)) finalfile = get_filename(dest) jsonschema.validate(contents, SCHEMA) url = contents['url'] refspec = None sshkey = None args = '' if 'refspec' in contents: refspec = contents['refspec'] if 'sshkey' in contents: sshkey = contents['sshkey'] if 'args' in contents: args = contents['args'] if sshkey: key = resources[sshkey]() if 'GIT_SSH' in os.environ: git_ssh = os.environ['GIT_SSH'] reset_to_env = True else: git_ssh = 'ssh' reset_to_env = False keyfile = write_temp_file(key, '.key', False) wrapper = write_temp_file(SSH_WRAPPER.format(git_ssh, keyfile), '.sh', True) old_environ = os.environ.copy() try: os.environ['GIT_SSH'] = wrapper repo = _clone_repo(url, finalfile, args) finally: if reset_to_env: os.environ['GIT_SSH'] = git_ssh else: del os.environ['GIT_SSH'] os.remove(keyfile) os.remove(wrapper) else: repo = _clone_repo(url, finalfile, args) if refspec: repo.git.checkout(refspec) return finalfile
def ext_extract(abs_source, dest, inputhash, loader, **kwargs): contents = json.loads(render(abs_source, inputhash, loader)) jsonschema.validate(contents, SCHEMA) url = contents['url'] local_filename = url.split('/')[-1] tmpfile = tempfile.NamedTemporaryFile(prefix=local_filename, suffix='.download', delete=False) to_delete = [tmpfile.name] try: _download_file(url, tmpfile) if 'type' in contents: extractor = EXTRACTORS[contents['type']] else: for k in EXTRACTORS.keys(): if local_filename.endswith('.' + k): extractor = EXTRACTORS[k] break else: raise UserException('No extractor found for {}'.format(local_filename)) if 'persist' in contents and not contents['persist']: extract_dest = tempfile.mkdtemp(prefix=local_filename, suffix='.sq') to_delete.append(extract_dest) persist = False else: extract_dest = dest persist = True extractor(tmpfile.name, extract_dest) if 'copy' in contents: _copy_files(extract_dest, dest, contents) if persist: finalfile = get_filename(dest) return finalfile else: return None finally: for f in to_delete: if os.path.isdir(f): shutil.rmtree(f) else: os.remove(f)
def ext_virtualenv(abs_source, dest, **kwargs): requirements = [] # Create virtualenv virtual_env = get_filename(dest) subprocess.check_call(['virtualenv', virtual_env]) # Set up environment pip_env = os.environ.copy() pip_env['VIRTUAL_ENV'] = virtual_env pip_env['PATH'] = '{}/bin:{}'.format(virtual_env, pip_env['PATH']) if 'PYTHONHOME' in pip_env: del pip_env['PYTHONHOME'] # Install requirements pip_bin = os.path.join(virtual_env, 'bin', 'pip') pip = subprocess.Popen([pip_bin, 'install', '-r', abs_source], env=pip_env) ret_code = pip.wait() if ret_code != 0: raise subprocess.CalledProcessError(ret_code, pip_bin) return virtual_env
def ext_dir(dest, **kwargs): finalfile = get_filename(dest) os.mkdir(finalfile) return finalfile