def test_working_dir(self): build_data = default_build_data() build_data['build_info']['channels'] = ['foo'] build_data['build_item_info']['instructions']['build_targets'] = { 'files': 'output_file', } script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) with open(script_filename, 'r') as script_file: script_content = script_file.read() self.assertIn("BUILD_ENV_PATH=", script_content) line = [ line for line in script_content.splitlines() if 'BUILD_ENV_PATH=' in line ] build_env_path = line[0].split('=')[-1].strip() if os.name == 'nt': self.assertEqual(build_env_path, '%WORKING_DIR%\env"') else: self.assertEqual(build_env_path, '"${WORKING_DIR}/env"')
def test_instructions_failure2(self): build_data = default_build_data() build_data['build_item_info']['instructions'][ 'script'] = 'invalid_command' script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() output = p0.stdout.read().decode() self.assertEqual(return_code, 12) self.assertIn("Exit BINSTAR_BUILD_RESULT=failure", output) self.assertInOrdered([ 'UNIQUE INSTALL MARKER', 'UNIQUE TEST MARKER', 'UNIQUE AFTER FAILURE MARKER', 'UNIQUE AFTER SCRIPT MARKER', ], output) p0.stdout.close()
def test_instructions_success(self): build_data = default_build_data() script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() self.assertEqual( return_code, 0, ) output = p0.stdout.read().decode() self.assertIn("Exit BINSTAR_BUILD_RESULT=success", output) self.assertInOrdered([ 'UNIQUE INSTALL MARKER', 'UNIQUE TEST MARKER', 'UNIQUE BEFORE SCRIPT MARKER', 'UNIQUE SCRIPT MARKER', 'UNIQUE AFTER SUCCESS MARKER', 'UNIQUE AFTER SCRIPT MARKER', ], output) p0.stdout.close()
def test_bad_tarball(self): build_data = default_build_data() script_filename = gen_build_script(build_data, ignore_setup_build=True) self.addCleanup(os.unlink, script_filename) build_tarball = path.join(path.dirname(__file__), 'data', 'does_not_exist.tar.bz2') p0 = Popen([script_filename, '--build-tarball', build_tarball], stdout=PIPE, stderr=STDOUT) self.assertEqual(p0.wait(), 11) p0.stdout.close()
def test_bad_tarball(self): build_data = default_build_data() script_filename = gen_build_script(build_data, ignore_setup_build=True) self.addCleanup(os.unlink, script_filename) build_tarball = path.join(path.dirname(__file__), 'data', 'does_not_exist.tar.bz2') p0 = Popen([script_filename, '--build-tarball', build_tarball], stdout=PIPE, stderr=STDOUT) self.assertEqual(p0.wait(), 11)
def test_install_channels(self): working_dir = tempfile.mkdtemp() try: script = script_generator.gen_build_script(working_dir, default_build_data()) with open(script, 'r') as f: contents = f.read() self.assertIn('--add channels r', contents) self.assertIn('--add channels python', contents) self.assertIn('--add channels other_channel', contents) finally: shutil.rmtree(working_dir)
def test_env_envvars(self): 'Test env or envvars can be used in .binstar.yml' build_data = default_build_data() for name in ('env', 'envvars'): build_data['build_item_info'][name] = {'ENVIRONMENT_VARIABLE': '1'} script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) contents = open(script_filename).read() self.assertIn('ENVIRONMENT_VARIABLE=', contents) build_data['build_item_info'].pop(name)
def test_auto_install_channels(self): working_dir = tempfile.mkdtemp() try: build_data = default_build_data() build_data['build_item_info']['instructions'][ 'install_channels'] = [] build_data['build_item_info']['engine'] = 'r' script = script_generator.gen_build_script(working_dir, build_data) with open(script, 'r') as f: contents = f.read() self.assertIn('--add channels r', contents) finally: shutil.rmtree(working_dir)
def test_auto_install_channels(self): working_dir = tempfile.mkdtemp() try: build_data = default_build_data() build_data['build_item_info']['instructions']['install_channels'] = [] build_data['build_item_info']['engine'] = 'r' script = script_generator.gen_build_script(working_dir, build_data) with open(script, 'r') as f: contents = f.read() self.assertIn('--add channels r', contents) finally: shutil.rmtree(working_dir)
def test_build_channels(self): build_data = default_build_data() build_data['build_info']['channels'] = ['foo'] build_data['build_item_info']['instructions']['build_targets'] = { 'files': 'output_file', } script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) with open(script_filename, 'r') as script_file: script_content = script_file.read() self.assertIn("--label foo", script_content)
def test_conda_npy(self): build_data = default_build_data() build_data['build_item_info']['engine'] = 'numpy=1.9' script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=False, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() output = p0.stdout.read().decode() lines = output.splitlines() conda_npy = [line for line in lines if "CONDA_NPY" in line] self.assertTrue(len(conda_npy) > 0) conda_npy_read = conda_npy[0].strip().replace('CONDA_NPY=', '') self.assertEqual(conda_npy_read, '19')
def test_conda_npy(self): build_data = default_build_data() build_data['build_item_info']['engine'] = 'numpy=1.9' tempdir = tempfile.mkdtemp() script_filename = gen_build_script(tempdir, tempdir, build_data, ignore_setup_build=False, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) with open(script_filename, 'r') as f: script_lines = [_.strip() for _ in f.readlines()] relates_to_npy = [ idx for idx, line in enumerate(script_lines) if 'NUMPY' in line or 'NPY' in line ] # Test that the export showed up at top if script_lines[relates_to_npy[0]].startswith(('export', 'set')): exported = relates_to_npy.pop(0) self.assertIn('=19', script_lines[exported]) self.assertIn('CONDA_NPY', script_lines[exported]) # Test that the other type of CONDA_NPY identification # can run without error other_numpy = "\n".join( script_lines[min(relates_to_npy):max(relates_to_npy) + 1]) script_filename = os.path.join(tempdir, 'numpy_script') if os.name == 'nt': script_filename += '.bat' args = ['cmd', '/c', 'call', script_filename] else: script_filename += '.sh' args = ['bash', script_filename] with open(script_filename, 'w') as f: f.write(other_numpy) self.addCleanup(os.unlink, script_filename) proc = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=tempdir) output = proc.stdout.read().decode().splitlines() npy = len([line for line in output if 'CONDA_NPY=' in line]) self.assertTrue(npy >= 1)
def test_instructions_error(self): build_data = default_build_data() build_data['build_item_info']['instructions']['install'] = 'invalid_command' script_filename = gen_build_script(build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() output = p0.stdout.read().decode() self.assertEqual(return_code, 11) self.assertIn("Exit BINSTAR_BUILD_RESULT=error", output) self.assertInOrdered([ 'UNIQUE AFTER ERROR MARKER', 'UNIQUE AFTER SCRIPT MARKER', ], output) p0.stdout.close()
def test_instructions_error(self): build_data = default_build_data() build_data['build_item_info']['instructions'][ 'install'] = 'invalid_command' script_filename = gen_build_script(build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() output = p0.stdout.read() self.assertEqual(return_code, 11) self.assertIn("Exit BINSTAR_BUILD_RESULT=error", output) self.assertInOrdered([ 'UNIQUE AFTER ERROR MARKER', 'UNIQUE AFTER SCRIPT MARKER', ], output)
def test_instructions_success(self): build_data = default_build_data() script_filename = gen_build_script(build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) p0 = Popen([script_filename], stdout=PIPE, stderr=STDOUT) return_code = p0.wait() self.assertEqual(return_code, 0,) output = p0.stdout.read().decode() self.assertIn("Exit BINSTAR_BUILD_RESULT=success", output) self.assertInOrdered(['UNIQUE INSTALL MARKER', 'UNIQUE TEST MARKER', 'UNIQUE BEFORE SCRIPT MARKER', 'UNIQUE SCRIPT MARKER', 'UNIQUE AFTER SUCCESS MARKER', 'UNIQUE AFTER SCRIPT MARKER', ], output) p0.stdout.close()
def test_working_dir(self): build_data = default_build_data() build_data['build_info']['channels'] = ['foo'] build_data['build_item_info']['instructions']['build_targets'] = { 'files': 'output_file', } script_filename = gen_build_script(tempfile.mkdtemp(), build_data, ignore_setup_build=True, ignore_fetch_build_source=True) self.addCleanup(os.unlink, script_filename) with open(script_filename, 'r') as script_file: script_content = script_file.read() self.assertIn("BUILD_ENV_PATH=", script_content) line = [line for line in script_content.splitlines() if 'BUILD_ENV_PATH=' in line] build_env_path = line[0].split('=')[-1].strip() if os.name == 'nt': self.assertEqual(build_env_path, '%WORKING_DIR%\env"') else: self.assertEqual(build_env_path, '"${WORKING_DIR}/env"')
def build(self, job_data): """ Run a single build """ job_id = job_data['job']['_id'] if 'envvars' in job_data['build_item_info']: job_data['build_item_info']['env'] = job_data[ 'build_item_info'].pop('envvars') working_dir = self.working_dir(job_data) staging_dir = self.staging_dir(job_data) # -- Clean -- log.info("Removing previous build dir: {0}".format(staging_dir)) rm_rf(staging_dir) log.info("Creating working dir: {0}".format(staging_dir)) os.makedirs(staging_dir) quiet = job_data['build_item_info'].get('instructions', {}).get('quiet', False) build_log = BuildLog( self.bs, self.config.username, self.config.queue, self.worker_id, job_id, filename=self.build_logfile(job_data), quiet=quiet, ) build_log.update_metadata({'section': 'dequeue_build'}) with build_log: instructions = job_data['build_item_info'].get('instructions') msg = "Building on worker {0} (platform {1})\n".format( self.config.hostname, self.config.platform) build_log.writeline(msg.encode('utf-8', errors='replace')) msg = "Starting build {0} at {1}\n".format( job_data['job_name'], job_data['BUILD_UTC_DATETIME']) build_log.writeline(msg.encode('utf-8', errors='replace')) # build_log.flush() script_filename = script_generator.gen_build_script( staging_dir, working_dir, job_data, conda_build_dir=self.args.conda_build_dir) iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT) timeout = self.args.timeout api_token = job_data['upload_token'] git_oauth_token = job_data.get('git_oauth_token') if not job_data.get('build_info', {}).get('github_info'): build_filename = self.download_build_source( staging_dir, job_id) else: build_filename = None exit_code = self.run( job_data, script_filename, build_log, timeout, iotimeout, api_token, git_oauth_token, build_filename, instructions=instructions, build_was_stopped_by_user=build_log.terminated) log.info("Build script exited with code {0}".format(exit_code)) if exit_code == script_generator.EXIT_CODE_OK: failed = False status = 'success' log.info('Build {0} Succeeded'.format(job_data['job_name'])) elif exit_code == script_generator.EXIT_CODE_ERROR: failed = True status = 'error' log.error("Build {0} errored".format(job_data['job_name'])) elif exit_code == script_generator.EXIT_CODE_FAILED: failed = True status = 'failure' log.error("Build {0} failed".format(job_data['job_name'])) else: # Unknown error failed = True status = 'error' log.error("Unknown build exit status {0} for build {1}".format( exit_code, job_data['job_name'])) return failed, status
def build(self, job_data): """ Run a single build """ job_id = job_data['job']['_id'] if 'envvars' in job_data['job']: job_data['job']['env'] = job_data.pop('envvars') working_dir = self.working_dir(job_data) log.info("Removing previous build dir: {0}".format(working_dir)) rm_rf(working_dir) log.info("Creating working dir: {0}".format(working_dir)) os.makedirs(working_dir) raw_build_log = BuildLog( self.bs, self.config.username, self.config.queue, self.worker_id, job_id, filename=self.build_logfile(job_data) ) build_log = io.BufferedWriter(raw_build_log) with build_log: instructions = job_data['build_item_info'].get('instructions') msg = "Building on worker {0} (platform {1})\n".format( self.config.hostname, self.config.platform) build_log.write(msg.encode('utf-8', errors='replace')) msg = "Starting build {0}\n".format(job_data['job_name']) build_log.write(msg.encode('utf-8', errors='replace')) build_log.flush() script_filename = script_generator.gen_build_script(working_dir, job_data, conda_build_dir=self.args.conda_build_dir) iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT) timeout = self.args.timeout api_token = job_data['upload_token'] git_oauth_token = job_data.get('git_oauth_token') if not job_data.get('build_info', {}).get('github_info'): build_filename = self.download_build_source(working_dir, job_id) else: build_filename = None exit_code = self.run( job_data, script_filename, build_log, timeout, iotimeout, api_token, git_oauth_token, build_filename, instructions=instructions, build_was_stopped_by_user=raw_build_log.terminated) log.info("Build script exited with code {0}".format(exit_code)) if exit_code == script_generator.EXIT_CODE_OK: failed = False status = 'success' log.info('Build {0} Succeeded'.format(job_data['job_name'])) elif exit_code == script_generator.EXIT_CODE_ERROR: failed = True status = 'error' log.error("Build {0} errored".format(job_data['job_name'])) elif exit_code == script_generator.EXIT_CODE_FAILED: failed = True status = 'failure' log.error("Build {0} failed".format(job_data['job_name'])) else: # Unknown error failed = True status = 'error' log.error("Unknown build exit status {0} for build {1}".format( exit_code, job_data['job_name'])) return failed, status