def test_build_proc_yaml_file(self, gridfs):
# Test that the proc.yaml file that gets deployed has the correct
# information.
config_name = 'test'
hostname = 'somehost'
proc = 'web'
port = 8000
with tmpdir():
# Generate the proc.yaml file the same way that
# server.vr.server.tasks.deploy() does; then yaml.load() it
# and compare with the local info.
with io.open('proc.yaml', 'w+') as f:
info = build_proc_info(self.release, config_name, hostname,
proc, port)
yaml.safe_dump(info,
stream=f,
default_flow_style=False,
encoding=None)
f.seek(0)
written_info = yaml.load(f.read())
assert info == written_info
def deploy(release_id, config_name, hostname, proc, port, swarm_trace_id=None):
with unlock_port_ctx(hostname, port):
release = Release.objects.get(id=release_id)
logger.info('[%s] Deploy %s-%s-%s to %s', swarm_trace_id, release,
proc, port, hostname)
msg_title = '%s-%s-%s' % (release.build.app.name, release.build.tag,
proc)
msg = 'deploying %s-%s-%s to %s' % (release, proc, port, hostname)
send_event(title=msg_title,
msg=msg,
tags=['deploy'],
swarm_id=swarm_trace_id)
assert release.build.file, "Build %s has no file" % release.build
assert release.hash, "Release %s has not been hashed" % release
with tmpdir():
# write the proc.yaml locally
with open('proc.yaml', 'wb') as f:
info = build_proc_info(release, config_name, hostname, proc,
port)
f.write(yaml.safe_dump(info, default_flow_style=False))
with remote_settings(hostname):
with always_disconnect(hostname):
remote.deploy_proc('proc.yaml')
def test_version_in_fragment():
rev = '16c1dba07ee78d5dbee1f965d91d3d61942ccb67'
url = 'https://github.com/btubbs/vr_python_example.git#' + rev
with tmpdir():
bp = BuildPack('bp', url, 'git')
bp.clone()
bp.update()
assert bp.version == rev
def cmd_build(build_data, runner_cmd='run', make_tarball=True):
# runner_cmd may be 'run' or 'shell'.
saver = OutputSaver() if make_tarball else NullSaver()
with tmpdir():
app_folder = _cmd_build(build_data, runner_cmd, saver)
saver.make_tarball(app_folder, build_data)
def download_file(url, path):
with tmpdir():
print("Downloading %s" % url)
base = os.path.basename(path)
with open(base, 'wb') as f:
resp = requests.get(url, stream=True)
resp.raise_for_status()
shutil.copyfileobj(resp.raw, f)
shutil.move(base, path)
def untar(tarpath, outfolder, owners=None, overwrite=True, fixperms=True):
"""
Unpack tarpath to outfolder. Make a guess about the compression based on
file extension (.gz or .bz2).
The unpacking of the tarfile is done in a temp directory and moved into
place atomically at the end (assuming /tmp is on the same filesystem as
outfolder).
If 'owners' is provided, it should be a tuple in the form
(username, groupname), and the contents of the unpacked folder will be set
with that owner and group.
If outfolder already exists, and overwrite=True (the default), the existing
outfolder will be deleted before the new one is put in place. If outfolder
already exists and overwrite=False, IOError will be raised.
"""
# We don't use fixperms at all
_ignored = fixperms # noqa
# make a folder to untar to
with tmpdir():
_, _, ext = tarpath.rpartition('.')
if ext not in ('gz', 'bz2', 'xz'):
raise ValueError('tarpath must point to a .gz, .bz2, or .xz file')
tf = tarfile.open(tarpath, 'r:' + ext)
try:
os.mkdir('contents')
tf.extractall('contents')
finally:
tf.close()
if owners is not None:
contents = path.Path('contents')
for item in contents.walk():
if item.isdir():
# chown user:group
item.chown(*owners)
# chmod ug+xr
item.chmod('ug+xr')
if item.isfile() and not item.islink():
# chown user:group
item.chown(*owners)
# chmod ug+rw
item.chmod('ug+rw')
if os.path.isdir(outfolder):
if overwrite:
shutil.rmtree(outfolder)
else:
raise IOError(('Cannot untar %s because %s already exists and '
'overwrite=False') % (tarfile, outfolder))
shutil.move('contents', outfolder)
def build_image(image_id, callback=None):
logger.info("Build image %s start", image_id)
image = OSImage.objects.get(id=image_id)
image_yaml = ImageData({
'base_image_url': image.base_image_url,
'base_image_name': image.base_image_name,
'new_image_name': image.name,
'script_url': image.provisioning_script_url,
}).as_yaml()
img_msg = "Started image build %s" % image + '\n\n' + image_yaml
send_event(str(image), img_msg, tags=['buildimage'])
t0 = time.time()
with tmpdir():
try:
with open('image.yaml', 'wb') as f:
f.write(image_yaml)
with remote_settings('localhost'):
remote.build_image('image.yaml')
# We should now have <image_name>.tar.gz and <image_name>.log
# locally.
img_file = image.name + '.tar.gz'
img_storage_path = 'images/' + img_file
with open(img_file, 'rb') as localfile:
image.file.save(img_storage_path, File(localfile))
image.active = True
finally:
logfile = image.name + '.log'
try:
# grab and store the compile log.
with open(logfile, 'rb') as f:
logname = 'images/' + logfile
image.build_log.save(logname, File(f))
except Exception:
logger.info('Could not retrieve ' + logfile)
raise
finally:
image.save()
elapsed_time = time.time() - t0
send_event(str(image),
"Completed image %s in %d seconds" % (image, elapsed_time),
tags=['buildimage', 'success'])
# start callback if there is one.
if callback is not None:
subtask(callback).delay()
def _do_build(build, build_yaml):
with timing.Stopwatch() as watch:
# enter a temp folder
with tmpdir():
try:
_try_build(build, build_yaml)
except Exception:
logger.exception('Build failed')
build.status = 'failed'
# Don't raise, or we'll mask the real error
try_get_compile_log(build, re_raise=False)
raise
finally:
build.end_time = timezone.now()
build.save()
msg = "Completed build %s in %s" % (build, watch.elapsed)
send_event(str(build), msg, tags=['build', 'success'])
def run_image(*args, **kwargs):
outfolder = path.Path.getcwd()
with tmpdir():
_run_image(outfolder, *args, **kwargs)
def test_git_clone(self):
url = 'https://github.com/btubbs/vr_python_example.git'
with tmpdir():
gitrepo = repo.Repo('gitrepo', url, 'git')
gitrepo.clone()
assert gitrepo.get_url() == url
def test_git_folder_detection(self):
with tmpdir():
folder = os.path.abspath('.git')
run('mkdir -p %s' % folder)
assert repo.guess_folder_vcs(os.getcwd()) == 'git'
def test_hg_clone(self):
url = 'https://bitbucket.org/btubbs/vr_python_example'
with tmpdir():
hgrepo = repo.Repo('hgrepo', url, 'hg')
hgrepo.clone()
assert hgrepo.get_url() == url
def test_hg_folder_detection():
with tmpdir():
folder = os.path.abspath('.hg')
run('mkdir -p %s' % folder)
assert repo.guess_folder_vcs(os.getcwd()) == 'hg'