def test_working_dir(self):
build_data = default_build_data()
build_data['build_info']['channels'] = ['foo']
build_data['build_item_info']['instructions']['build_targets'] = {
'files': 'output_file',
}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
with open(script_filename, 'r') as script_file:
script_content = script_file.read()
self.assertIn("BUILD_ENV_PATH=", script_content)
line = [
line for line in script_content.splitlines()
if 'BUILD_ENV_PATH=' in line
]
build_env_path = line[0].split('=')[-1].strip()
if os.name == 'nt':
self.assertEqual(build_env_path, '%WORKING_DIR%\env"')
else:
self.assertEqual(build_env_path, '"${WORKING_DIR}/env"')
def test_instructions_failure2(self):
build_data = default_build_data()
build_data['build_item_info']['instructions'][
'script'] = 'invalid_command'
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
output = p0.stdout.read().decode()
self.assertEqual(return_code, 12)
self.assertIn("Exit BINSTAR_BUILD_RESULT=failure", output)
self.assertInOrdered([
'UNIQUE INSTALL MARKER',
'UNIQUE TEST MARKER',
'UNIQUE AFTER FAILURE MARKER',
'UNIQUE AFTER SCRIPT MARKER',
], output)
p0.stdout.close()
def test_instructions_success(self):
build_data = default_build_data()
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
self.assertEqual(
return_code,
0,
)
output = p0.stdout.read().decode()
self.assertIn("Exit BINSTAR_BUILD_RESULT=success", output)
self.assertInOrdered([
'UNIQUE INSTALL MARKER',
'UNIQUE TEST MARKER',
'UNIQUE BEFORE SCRIPT MARKER',
'UNIQUE SCRIPT MARKER',
'UNIQUE AFTER SUCCESS MARKER',
'UNIQUE AFTER SCRIPT MARKER',
], output)
p0.stdout.close()
def test_bad_tarball(self):
build_data = default_build_data()
script_filename = gen_build_script(build_data, ignore_setup_build=True)
self.addCleanup(os.unlink, script_filename)
build_tarball = path.join(path.dirname(__file__), 'data', 'does_not_exist.tar.bz2')
p0 = Popen([script_filename, '--build-tarball', build_tarball], stdout=PIPE, stderr=STDOUT)
self.assertEqual(p0.wait(), 11)
p0.stdout.close()
def test_bad_tarball(self):
build_data = default_build_data()
script_filename = gen_build_script(build_data, ignore_setup_build=True)
self.addCleanup(os.unlink, script_filename)
build_tarball = path.join(path.dirname(__file__), 'data',
'does_not_exist.tar.bz2')
p0 = Popen([script_filename, '--build-tarball', build_tarball],
stdout=PIPE,
stderr=STDOUT)
self.assertEqual(p0.wait(), 11)
def test_install_channels(self):
working_dir = tempfile.mkdtemp()
try:
script = script_generator.gen_build_script(working_dir,
default_build_data())
with open(script, 'r') as f:
contents = f.read()
self.assertIn('--add channels r', contents)
self.assertIn('--add channels python', contents)
self.assertIn('--add channels other_channel', contents)
finally:
shutil.rmtree(working_dir)
def test_install_channels(self):
working_dir = tempfile.mkdtemp()
try:
script = script_generator.gen_build_script(working_dir,
default_build_data())
with open(script, 'r') as f:
contents = f.read()
self.assertIn('--add channels r', contents)
self.assertIn('--add channels python', contents)
self.assertIn('--add channels other_channel', contents)
finally:
shutil.rmtree(working_dir)
def test_env_envvars(self):
'Test env or envvars can be used in .binstar.yml'
build_data = default_build_data()
for name in ('env', 'envvars'):
build_data['build_item_info'][name] = {'ENVIRONMENT_VARIABLE': '1'}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
contents = open(script_filename).read()
self.assertIn('ENVIRONMENT_VARIABLE=', contents)
build_data['build_item_info'].pop(name)
def test_env_envvars(self):
'Test env or envvars can be used in .binstar.yml'
build_data = default_build_data()
for name in ('env', 'envvars'):
build_data['build_item_info'][name] = {'ENVIRONMENT_VARIABLE': '1'}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
contents = open(script_filename).read()
self.assertIn('ENVIRONMENT_VARIABLE=', contents)
build_data['build_item_info'].pop(name)
def test_auto_install_channels(self):
working_dir = tempfile.mkdtemp()
try:
build_data = default_build_data()
build_data['build_item_info']['instructions'][
'install_channels'] = []
build_data['build_item_info']['engine'] = 'r'
script = script_generator.gen_build_script(working_dir, build_data)
with open(script, 'r') as f:
contents = f.read()
self.assertIn('--add channels r', contents)
finally:
shutil.rmtree(working_dir)
def test_auto_install_channels(self):
working_dir = tempfile.mkdtemp()
try:
build_data = default_build_data()
build_data['build_item_info']['instructions']['install_channels'] = []
build_data['build_item_info']['engine'] = 'r'
script = script_generator.gen_build_script(working_dir,
build_data)
with open(script, 'r') as f:
contents = f.read()
self.assertIn('--add channels r', contents)
finally:
shutil.rmtree(working_dir)
def test_build_channels(self):
build_data = default_build_data()
build_data['build_info']['channels'] = ['foo']
build_data['build_item_info']['instructions']['build_targets'] = {
'files': 'output_file',
}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
with open(script_filename, 'r') as script_file:
script_content = script_file.read()
self.assertIn("--label foo", script_content)
def test_build_channels(self):
build_data = default_build_data()
build_data['build_info']['channels'] = ['foo']
build_data['build_item_info']['instructions']['build_targets'] = {
'files': 'output_file',
}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
with open(script_filename, 'r') as script_file:
script_content = script_file.read()
self.assertIn("--label foo", script_content)
def test_conda_npy(self):
build_data = default_build_data()
build_data['build_item_info']['engine'] = 'numpy=1.9'
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=False,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
output = p0.stdout.read().decode()
lines = output.splitlines()
conda_npy = [line for line in lines if "CONDA_NPY" in line]
self.assertTrue(len(conda_npy) > 0)
conda_npy_read = conda_npy[0].strip().replace('CONDA_NPY=', '')
self.assertEqual(conda_npy_read, '19')
def test_conda_npy(self):
build_data = default_build_data()
build_data['build_item_info']['engine'] = 'numpy=1.9'
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=False,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
output = p0.stdout.read().decode()
lines = output.splitlines()
conda_npy = [line for line in lines if "CONDA_NPY" in line]
self.assertTrue(len(conda_npy) > 0)
conda_npy_read = conda_npy[0].strip().replace('CONDA_NPY=', '')
self.assertEqual(conda_npy_read, '19')
def test_conda_npy(self):
build_data = default_build_data()
build_data['build_item_info']['engine'] = 'numpy=1.9'
tempdir = tempfile.mkdtemp()
script_filename = gen_build_script(tempdir,
tempdir,
build_data,
ignore_setup_build=False,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
with open(script_filename, 'r') as f:
script_lines = [_.strip() for _ in f.readlines()]
relates_to_npy = [
idx for idx, line in enumerate(script_lines)
if 'NUMPY' in line or 'NPY' in line
]
# Test that the export showed up at top
if script_lines[relates_to_npy[0]].startswith(('export', 'set')):
exported = relates_to_npy.pop(0)
self.assertIn('=19', script_lines[exported])
self.assertIn('CONDA_NPY', script_lines[exported])
# Test that the other type of CONDA_NPY identification
# can run without error
other_numpy = "\n".join(
script_lines[min(relates_to_npy):max(relates_to_npy) + 1])
script_filename = os.path.join(tempdir, 'numpy_script')
if os.name == 'nt':
script_filename += '.bat'
args = ['cmd', '/c', 'call', script_filename]
else:
script_filename += '.sh'
args = ['bash', script_filename]
with open(script_filename, 'w') as f:
f.write(other_numpy)
self.addCleanup(os.unlink, script_filename)
proc = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=tempdir)
output = proc.stdout.read().decode().splitlines()
npy = len([line for line in output if 'CONDA_NPY=' in line])
self.assertTrue(npy >= 1)
def test_instructions_error(self):
build_data = default_build_data()
build_data['build_item_info']['instructions']['install'] = 'invalid_command'
script_filename = gen_build_script(build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
output = p0.stdout.read().decode()
self.assertEqual(return_code, 11)
self.assertIn("Exit BINSTAR_BUILD_RESULT=error", output)
self.assertInOrdered([
'UNIQUE AFTER ERROR MARKER',
'UNIQUE AFTER SCRIPT MARKER',
], output)
p0.stdout.close()
def test_instructions_error(self):
build_data = default_build_data()
build_data['build_item_info']['instructions'][
'install'] = 'invalid_command'
script_filename = gen_build_script(build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
output = p0.stdout.read()
self.assertEqual(return_code, 11)
self.assertIn("Exit BINSTAR_BUILD_RESULT=error", output)
self.assertInOrdered([
'UNIQUE AFTER ERROR MARKER',
'UNIQUE AFTER SCRIPT MARKER',
], output)
def test_instructions_success(self):
build_data = default_build_data()
script_filename = gen_build_script(build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT)
return_code = p0.wait()
self.assertEqual(return_code, 0,)
output = p0.stdout.read().decode()
self.assertIn("Exit BINSTAR_BUILD_RESULT=success", output)
self.assertInOrdered(['UNIQUE INSTALL MARKER',
'UNIQUE TEST MARKER',
'UNIQUE BEFORE SCRIPT MARKER',
'UNIQUE SCRIPT MARKER',
'UNIQUE AFTER SUCCESS MARKER',
'UNIQUE AFTER SCRIPT MARKER',
], output)
p0.stdout.close()
def test_working_dir(self):
build_data = default_build_data()
build_data['build_info']['channels'] = ['foo']
build_data['build_item_info']['instructions']['build_targets'] = {
'files': 'output_file',
}
script_filename = gen_build_script(tempfile.mkdtemp(),
build_data,
ignore_setup_build=True,
ignore_fetch_build_source=True)
self.addCleanup(os.unlink, script_filename)
with open(script_filename, 'r') as script_file:
script_content = script_file.read()
self.assertIn("BUILD_ENV_PATH=", script_content)
line = [line for line in script_content.splitlines() if 'BUILD_ENV_PATH=' in line]
build_env_path = line[0].split('=')[-1].strip()
if os.name == 'nt':
self.assertEqual(build_env_path, '%WORKING_DIR%\env"')
else:
self.assertEqual(build_env_path, '"${WORKING_DIR}/env"')
def build(self, job_data):
"""
Run a single build
"""
job_id = job_data['job']['_id']
if 'envvars' in job_data['build_item_info']:
job_data['build_item_info']['env'] = job_data[
'build_item_info'].pop('envvars')
working_dir = self.working_dir(job_data)
staging_dir = self.staging_dir(job_data)
# -- Clean --
log.info("Removing previous build dir: {0}".format(staging_dir))
rm_rf(staging_dir)
log.info("Creating working dir: {0}".format(staging_dir))
os.makedirs(staging_dir)
quiet = job_data['build_item_info'].get('instructions',
{}).get('quiet', False)
build_log = BuildLog(
self.bs,
self.config.username,
self.config.queue,
self.worker_id,
job_id,
filename=self.build_logfile(job_data),
quiet=quiet,
)
build_log.update_metadata({'section': 'dequeue_build'})
with build_log:
instructions = job_data['build_item_info'].get('instructions')
msg = "Building on worker {0} (platform {1})\n".format(
self.config.hostname, self.config.platform)
build_log.writeline(msg.encode('utf-8', errors='replace'))
msg = "Starting build {0} at {1}\n".format(
job_data['job_name'], job_data['BUILD_UTC_DATETIME'])
build_log.writeline(msg.encode('utf-8', errors='replace'))
# build_log.flush()
script_filename = script_generator.gen_build_script(
staging_dir,
working_dir,
job_data,
conda_build_dir=self.args.conda_build_dir)
iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT)
timeout = self.args.timeout
api_token = job_data['upload_token']
git_oauth_token = job_data.get('git_oauth_token')
if not job_data.get('build_info', {}).get('github_info'):
build_filename = self.download_build_source(
staging_dir, job_id)
else:
build_filename = None
exit_code = self.run(
job_data,
script_filename,
build_log,
timeout,
iotimeout,
api_token,
git_oauth_token,
build_filename,
instructions=instructions,
build_was_stopped_by_user=build_log.terminated)
log.info("Build script exited with code {0}".format(exit_code))
if exit_code == script_generator.EXIT_CODE_OK:
failed = False
status = 'success'
log.info('Build {0} Succeeded'.format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_ERROR:
failed = True
status = 'error'
log.error("Build {0} errored".format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_FAILED:
failed = True
status = 'failure'
log.error("Build {0} failed".format(job_data['job_name']))
else: # Unknown error
failed = True
status = 'error'
log.error("Unknown build exit status {0} for build {1}".format(
exit_code, job_data['job_name']))
return failed, status
def build(self, job_data):
"""
Run a single build
"""
job_id = job_data['job']['_id']
if 'envvars' in job_data['job']:
job_data['job']['env'] = job_data.pop('envvars')
working_dir = self.working_dir(job_data)
log.info("Removing previous build dir: {0}".format(working_dir))
rm_rf(working_dir)
log.info("Creating working dir: {0}".format(working_dir))
os.makedirs(working_dir)
raw_build_log = BuildLog(
self.bs,
self.config.username,
self.config.queue,
self.worker_id,
job_id,
filename=self.build_logfile(job_data)
)
build_log = io.BufferedWriter(raw_build_log)
with build_log:
instructions = job_data['build_item_info'].get('instructions')
msg = "Building on worker {0} (platform {1})\n".format(
self.config.hostname, self.config.platform)
build_log.write(msg.encode('utf-8', errors='replace'))
msg = "Starting build {0}\n".format(job_data['job_name'])
build_log.write(msg.encode('utf-8', errors='replace'))
build_log.flush()
script_filename = script_generator.gen_build_script(working_dir,
job_data, conda_build_dir=self.args.conda_build_dir)
iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT)
timeout = self.args.timeout
api_token = job_data['upload_token']
git_oauth_token = job_data.get('git_oauth_token')
if not job_data.get('build_info', {}).get('github_info'):
build_filename = self.download_build_source(working_dir, job_id)
else:
build_filename = None
exit_code = self.run(
job_data, script_filename, build_log, timeout, iotimeout, api_token,
git_oauth_token, build_filename, instructions=instructions,
build_was_stopped_by_user=raw_build_log.terminated)
log.info("Build script exited with code {0}".format(exit_code))
if exit_code == script_generator.EXIT_CODE_OK:
failed = False
status = 'success'
log.info('Build {0} Succeeded'.format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_ERROR:
failed = True
status = 'error'
log.error("Build {0} errored".format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_FAILED:
failed = True
status = 'failure'
log.error("Build {0} failed".format(job_data['job_name']))
else: # Unknown error
failed = True
status = 'error'
log.error("Unknown build exit status {0} for build {1}".format(
exit_code, job_data['job_name']))
return failed, status
