def ext_virtualenv(abs_source, dest, **kwargs):
requirements=[]
# Create virtualenv
virtual_env = get_filename(dest)
subprocess.check_call(['virtualenv', virtual_env])
# Set up environment
pip_env = os.environ.copy()
pip_env['VIRTUAL_ENV'] = virtual_env
pip_env['PATH'] = '{}/bin:{}'.format(virtual_env, pip_env['PATH'])
if 'PYTHONHOME' in pip_env:
del pip_env['PYTHONHOME']
# Install requirements
pip_bin = os.path.join(virtual_env, 'bin', 'pip')
pip = subprocess.Popen([pip_bin, 'install', '-r', abs_source],
env=pip_env)
ret_code = pip.wait()
if ret_code != 0:
raise subprocess.CalledProcessError(ret_code, pip_bin)
return virtual_env
def ext_template(inputhash, relpath, loader, dest, **kwargs):
""" Renders a ~tpl file"""
output = render(relpath, inputhash, loader)
finalfile = get_filename(dest)
with open(finalfile, 'w') as outfile:
outfile.write(output)
return finalfile
def ext_template(inputhash, relpath, loader, dest, **kwargs):
""" Renders a ~tpl file"""
output = render(relpath, inputhash, loader)
finalfile = get_filename(dest)
with open(finalfile, 'w') as outfile:
outfile.write(output)
return finalfile
def ext_download(loader, inputhash, abs_source, dest, **kwargs):
""" Downloads a ~download file"""
template = loader.load_template(abs_source)
output = template.render(inputhash, loader=loader)
# split on whitespace
(url, checksum) = output.split(None, 3)
finalfile = get_filename(dest)
urllib.urlretrieve(url, finalfile)
return finalfile
def ext_extract(abs_source, dest, inputhash, loader, **kwargs):
contents = yaml.load(render(abs_source, inputhash, loader))
jsonschema.validate(contents, SCHEMA)
finalfile = get_filename(dest)
url = contents['url']
local_filename = url.split('/')[-1]
tmpfile = tempfile.NamedTemporaryFile(prefix=local_filename,
suffix='.download',
delete=False)
to_delete = [tmpfile.name]
try:
auth = None
if 'username' in contents and 'password' in contents:
auth = (contents['username'], contents['password'])
_download_file(url, tmpfile, auth)
if 'type' in contents:
extractor = EXTRACTORS[contents['type']]
else:
for k in EXTRACTORS.keys():
if local_filename.endswith('.' + k):
extractor = EXTRACTORS[k]
break
else:
raise TypeError(
'No extractor found for {}'.format(local_filename))
if 'persist' in contents and not contents['persist']:
extract_dest = tempfile.mkdtemp(prefix=local_filename,
suffix='.sq')
to_delete.append(extract_dest)
persist = False
else:
extract_dest = finalfile
persist = True
extractor(tmpfile.name, extract_dest)
if 'copy' in contents:
_copy_files(extract_dest, finalfile, contents)
return finalfile
finally:
for f in to_delete:
if os.path.isdir(f):
shutil.rmtree(f)
else:
os.remove(f)
def ext_download(loader, inputhash, abs_source, dest, **kwargs):
""" Downloads a ~download file"""
contents = yaml.load(render(abs_source, inputhash, loader))
jsonschema.validate(contents, SCHEMA)
finalfile = get_filename(dest)
handle = open(finalfile, 'w')
auth = None
if 'username' in contents and 'password' in contents:
auth = (contents['username'], contents['password'])
_download_file(contents['url'], handle, auth)
return finalfile
def ext_download(loader, inputhash, abs_source, dest, **kwargs):
""" Downloads a ~download file"""
contents = yaml.load(render(abs_source, inputhash, loader))
jsonschema.validate(contents, SCHEMA)
finalfile = get_filename(dest)
handle = open(finalfile, 'w')
auth = None
if 'username' in contents and 'password' in contents:
auth = (contents['username'], contents['password'])
_download_file(contents['url'], handle, auth)
return finalfile
def ext_git(abs_source, dest, inputhash, loader, resources, **kwargs):
""" Clones a git repository """
contents = json.loads(render(abs_source, inputhash, loader))
finalfile = get_filename(dest)
jsonschema.validate(contents, SCHEMA)
url = contents['url']
refspec = None
sshkey = None
args = ''
if 'refspec' in contents:
refspec = contents['refspec']
if 'sshkey' in contents:
sshkey = contents['sshkey']
if 'args' in contents:
args = contents['args']
if sshkey:
key = resources[sshkey]()
if 'GIT_SSH' in os.environ:
git_ssh = os.environ['GIT_SSH']
reset_to_env = True
else:
git_ssh = 'ssh'
reset_to_env = False
keyfile = write_temp_file(key, '.key', False)
wrapper = write_temp_file(SSH_WRAPPER.format(git_ssh, keyfile), '.sh', True)
old_environ = os.environ.copy()
try:
os.environ['GIT_SSH'] = wrapper
repo = _clone_repo(url, finalfile, args)
finally:
if reset_to_env:
os.environ['GIT_SSH'] = git_ssh
else:
del os.environ['GIT_SSH']
os.remove(keyfile)
os.remove(wrapper)
else:
repo = _clone_repo(url, finalfile, args)
if refspec:
repo.git.checkout(refspec)
return finalfile
def ext_git(abs_source, dest, inputhash, loader, resources, **kwargs):
""" Clones a git repository """
contents = yaml.load(render(abs_source, inputhash, loader))
finalfile = get_filename(dest)
jsonschema.validate(contents, SCHEMA)
url = contents['url']
refspec = None
sshkey = None
args = ''
if 'refspec' in contents:
refspec = contents['refspec']
if 'sshkey' in contents:
sshkey = contents['sshkey']
if 'args' in contents:
args = contents['args']
if sshkey:
key = resources[sshkey]()
if 'GIT_SSH' in os.environ:
git_ssh = os.environ['GIT_SSH']
reset_to_env = True
else:
git_ssh = 'ssh'
reset_to_env = False
keyfile = write_temp_file(key, '.key', False)
wrapper = write_temp_file(SSH_WRAPPER.format(git_ssh, keyfile), '.sh', True)
old_environ = os.environ.copy()
try:
os.environ['GIT_SSH'] = wrapper
repo = _clone_repo(url, finalfile, args)
finally:
if reset_to_env:
os.environ['GIT_SSH'] = git_ssh
else:
del os.environ['GIT_SSH']
os.remove(keyfile)
os.remove(wrapper)
else:
repo = _clone_repo(url, finalfile, args)
if refspec:
repo.git.checkout(refspec)
return finalfile
def ext_extract(abs_source, dest, inputhash, loader, **kwargs):
contents = json.loads(render(abs_source, inputhash, loader))
jsonschema.validate(contents, SCHEMA)
url = contents['url']
local_filename = url.split('/')[-1]
tmpfile = tempfile.NamedTemporaryFile(prefix=local_filename, suffix='.download', delete=False)
to_delete = [tmpfile.name]
try:
_download_file(url, tmpfile)
if 'type' in contents:
extractor = EXTRACTORS[contents['type']]
else:
for k in EXTRACTORS.keys():
if local_filename.endswith('.' + k):
extractor = EXTRACTORS[k]
break
else:
raise UserException('No extractor found for {}'.format(local_filename))
if 'persist' in contents and not contents['persist']:
extract_dest = tempfile.mkdtemp(prefix=local_filename, suffix='.sq')
to_delete.append(extract_dest)
persist = False
else:
extract_dest = dest
persist = True
extractor(tmpfile.name, extract_dest)
if 'copy' in contents:
_copy_files(extract_dest, dest, contents)
if persist:
finalfile = get_filename(dest)
return finalfile
else:
return None
finally:
for f in to_delete:
if os.path.isdir(f):
shutil.rmtree(f)
else:
os.remove(f)
def ext_virtualenv(abs_source, dest, **kwargs):
requirements = []
# Create virtualenv
virtual_env = get_filename(dest)
subprocess.check_call(['virtualenv', virtual_env])
# Set up environment
pip_env = os.environ.copy()
pip_env['VIRTUAL_ENV'] = virtual_env
pip_env['PATH'] = '{}/bin:{}'.format(virtual_env, pip_env['PATH'])
if 'PYTHONHOME' in pip_env:
del pip_env['PYTHONHOME']
# Install requirements
pip_bin = os.path.join(virtual_env, 'bin', 'pip')
pip = subprocess.Popen([pip_bin, 'install', '-r', abs_source], env=pip_env)
ret_code = pip.wait()
if ret_code != 0:
raise subprocess.CalledProcessError(ret_code, pip_bin)
return virtual_env
def ext_dir(dest, **kwargs):
finalfile = get_filename(dest)
os.mkdir(finalfile)
return finalfile
def ext_dir(dest, **kwargs):
finalfile = get_filename(dest)
os.mkdir(finalfile)
return finalfile
