def run(self, build_data, script_filename, build_log, timeout, iotimeout,
api_token=None, git_oauth_token=None, build_filename=None, instructions=None):
owner = build_data['owner']['login']
package = build_data['package']['name']
working_dir = os.path.abspath(os.path.join('builds', owner, package))
rm_rf(working_dir)
os.makedirs(working_dir)
args = [os.path.abspath(script_filename), '--api-token', api_token]
if git_oauth_token:
args.extend(['--git-oauth-token', git_oauth_token])
elif build_filename:
args.extend(['--build-tarball', build_filename])
log.info("Running command: (iotimeout=%s)" % iotimeout)
log.info(" ".join(args))
if self.args.show_new_procs:
already_running_procs = get_my_procs()
p0 = BufferedPopen(args, stdout=build_log, iotimeout=iotimeout, cwd=working_dir)
try:
exit_code = p0.wait()
except BaseException:
log.error("Binstar build process caught an exception while waiting for the build to finish")
p0.kill_tree()
p0.wait()
raise
finally:
if self.args.show_new_procs:
currently_running_procs = get_my_procs()
new_procs = [psutil.Process(pid) for pid in currently_running_procs - already_running_procs]
if new_procs:
build_log.write("WARNING: There are processes that were started during the build and are still running\n")
for proc in new_procs:
build_log.write(" - Process name:%s pid:%s\n" % (proc.name, proc.pid))
try:
cmdline = ' '.join(proc.cmdline)
except:
pass
else:
build_log.write(" + %s\n" % cmdline)
if p0.stdout and not p0.stdout.closed:
log.info("Closing subprocess stdout PIPE")
p0.stdout.close()
return exit_code
def main():
parser = ArgumentParser(description=__doc__)
parser.parse_args()
root_env = get_conda_root_prefix()
build_root = os.path.join(root_env, 'conda-bld')
if os.path.isdir(build_root):
print("Removing conda build root %s" % build_root)
rm_rf(build_root)
else:
print("Conda build root %s does not exist" % build_root)
def main():
parser = ArgumentParser(description=__doc__)
parser.parse_args()
root_env = get_conda_root_prefix()
build_root = os.path.join(root_env, 'conda-bld')
has_access = os.access(build_root, os.W_OK)
if not has_access:
build_root = os.path.join(os.path.expanduser('~'), 'conda-bld')
if os.path.isdir(build_root):
print("Removing conda build root {}".format(build_root))
rm_rf(build_root)
else:
print("Conda build root {} does not exist".format(build_root))
def run(self,
build_data,
script_filename,
build_log,
timeout,
iotimeout,
api_token=None,
git_oauth_token=None,
build_filename=None,
instructions=None):
owner = build_data['owner']['login']
package = build_data['package']['name']
working_dir = os.path.abspath(os.path.join('builds', owner, package))
rm_rf(working_dir)
os.makedirs(working_dir)
args = [os.path.abspath(script_filename), '--api-token', api_token]
if git_oauth_token:
args.extend(['--git-oauth-token', git_oauth_token])
elif build_filename:
args.extend(['--build-tarball', build_filename])
log.info("Running command: (iotimeout=%s)" % iotimeout)
log.info(" ".join(args))
if self.args.show_new_procs:
already_running_procs = get_my_procs()
p0 = BufferedPopen(args,
stdout=build_log,
iotimeout=iotimeout,
cwd=working_dir)
try:
exit_code = p0.wait()
except BaseException:
log.error(
"Binstar build process caught an exception while waiting for the build to finish"
)
p0.kill_tree()
p0.wait()
raise
finally:
if self.args.show_new_procs:
currently_running_procs = get_my_procs()
new_procs = [
psutil.Process(pid)
for pid in currently_running_procs - already_running_procs
]
if new_procs:
build_log.write(
"WARNING: There are processes that were started during the build and are still running\n"
)
for proc in new_procs:
build_log.write(" - Process name:%s pid:%s\n" %
(proc.name, proc.pid))
try:
cmdline = ' '.join(proc.cmdline)
except:
pass
else:
build_log.write(" + %s\n" % cmdline)
return exit_code
def build(self, job_data):
"""
Run a single build
"""
job_id = job_data['job']['_id']
if 'envvars' in job_data['build_item_info']:
job_data['build_item_info']['env'] = job_data[
'build_item_info'].pop('envvars')
working_dir = self.working_dir(job_data)
staging_dir = self.staging_dir(job_data)
# -- Clean --
log.info("Removing previous build dir: {0}".format(staging_dir))
rm_rf(staging_dir)
log.info("Creating working dir: {0}".format(staging_dir))
os.makedirs(staging_dir)
quiet = job_data['build_item_info'].get('instructions',
{}).get('quiet', False)
build_log = BuildLog(
self.bs,
self.config.username,
self.config.queue,
self.worker_id,
job_id,
filename=self.build_logfile(job_data),
quiet=quiet,
)
build_log.update_metadata({'section': 'dequeue_build'})
with build_log:
instructions = job_data['build_item_info'].get('instructions')
msg = "Building on worker {0} (platform {1})\n".format(
self.config.hostname, self.config.platform)
build_log.writeline(msg.encode('utf-8', errors='replace'))
msg = "Starting build {0} at {1}\n".format(
job_data['job_name'], job_data['BUILD_UTC_DATETIME'])
build_log.writeline(msg.encode('utf-8', errors='replace'))
# build_log.flush()
script_filename = script_generator.gen_build_script(
staging_dir,
working_dir,
job_data,
conda_build_dir=self.args.conda_build_dir)
iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT)
timeout = self.args.timeout
api_token = job_data['upload_token']
git_oauth_token = job_data.get('git_oauth_token')
if not job_data.get('build_info', {}).get('github_info'):
build_filename = self.download_build_source(
staging_dir, job_id)
else:
build_filename = None
exit_code = self.run(
job_data,
script_filename,
build_log,
timeout,
iotimeout,
api_token,
git_oauth_token,
build_filename,
instructions=instructions,
build_was_stopped_by_user=build_log.terminated)
log.info("Build script exited with code {0}".format(exit_code))
if exit_code == script_generator.EXIT_CODE_OK:
failed = False
status = 'success'
log.info('Build {0} Succeeded'.format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_ERROR:
failed = True
status = 'error'
log.error("Build {0} errored".format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_FAILED:
failed = True
status = 'failure'
log.error("Build {0} failed".format(job_data['job_name']))
else: # Unknown error
failed = True
status = 'error'
log.error("Unknown build exit status {0} for build {1}".format(
exit_code, job_data['job_name']))
return failed, status
def build(self, job_data):
"""
Run a single build
"""
job_id = job_data['job']['_id']
if 'envvars' in job_data['job']:
job_data['job']['env'] = job_data.pop('envvars')
working_dir = self.working_dir(job_data)
log.info("Removing previous build dir: {0}".format(working_dir))
rm_rf(working_dir)
log.info("Creating working dir: {0}".format(working_dir))
os.makedirs(working_dir)
raw_build_log = BuildLog(
self.bs,
self.config.username,
self.config.queue,
self.worker_id,
job_id,
filename=self.build_logfile(job_data)
)
build_log = io.BufferedWriter(raw_build_log)
with build_log:
instructions = job_data['build_item_info'].get('instructions')
msg = "Building on worker {0} (platform {1})\n".format(
self.config.hostname, self.config.platform)
build_log.write(msg.encode('utf-8', errors='replace'))
msg = "Starting build {0}\n".format(job_data['job_name'])
build_log.write(msg.encode('utf-8', errors='replace'))
build_log.flush()
script_filename = script_generator.gen_build_script(working_dir,
job_data, conda_build_dir=self.args.conda_build_dir)
iotimeout = instructions.get('iotimeout', DEFAULT_IO_TIMEOUT)
timeout = self.args.timeout
api_token = job_data['upload_token']
git_oauth_token = job_data.get('git_oauth_token')
if not job_data.get('build_info', {}).get('github_info'):
build_filename = self.download_build_source(working_dir, job_id)
else:
build_filename = None
exit_code = self.run(
job_data, script_filename, build_log, timeout, iotimeout, api_token,
git_oauth_token, build_filename, instructions=instructions,
build_was_stopped_by_user=raw_build_log.terminated)
log.info("Build script exited with code {0}".format(exit_code))
if exit_code == script_generator.EXIT_CODE_OK:
failed = False
status = 'success'
log.info('Build {0} Succeeded'.format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_ERROR:
failed = True
status = 'error'
log.error("Build {0} errored".format(job_data['job_name']))
elif exit_code == script_generator.EXIT_CODE_FAILED:
failed = True
status = 'failure'
log.error("Build {0} failed".format(job_data['job_name']))
else: # Unknown error
failed = True
status = 'error'
log.error("Unknown build exit status {0} for build {1}".format(
exit_code, job_data['job_name']))
return failed, status
def build(self, job_data):
"""
Run a single build
"""
job_id = job_data['job']['_id']
working_dir = self.working_dir(job_data)
log.info("Creating working dir: %s" % working_dir)
rm_rf(working_dir)
os.makedirs(working_dir)
build_log = BuildLog(self.bs, self.args.username, self.args.queue, self.worker_id, job_id,
filename=self.build_logfile(job_data))
with build_log:
instructions = job_data['build_item_info'].get('instructions')
build_log.write("Building on worker %s (platform %s)\n" % (self.args.hostname, self.args.platform))
build_log.write("Starting build %s\n" % job_data['job_name'])
if not os.path.exists('build_scripts'):
os.mkdir('build_scripts')
script_filename = gen_build_script(job_data,
conda_build_dir=self.args.conda_build_dir)
iotimeout = instructions.get('iotimeout', 60)
timeout = self.args.timeout
api_token = job_data['upload_token']
files = [script_filename]
git_oauth_token = job_data.get('git_oauth_token')
if not job_data.get('build_info', {}).get('github_info'):
build_filename = self.download_build_source(job_id)
files.append(build_filename)
else:
build_filename = None
with remove_files_after(files):
exit_code = self.run(job_data, script_filename, build_log,
timeout, iotimeout,
api_token, git_oauth_token, build_filename,
instructions=instructions)
log.info("Build script exited with code %s" % exit_code)
if exit_code == EXIT_CODE_OK:
failed = False
status = 'success'
log.info('Build %s Succeeded' % (job_data['job_name']))
elif exit_code == EXIT_CODE_ERROR:
failed = True
status = 'error'
log.error("Build %s errored" % (job_data['job_name']))
elif exit_code == EXIT_CODE_FAILED:
failed = True
status = 'failure'
log.error("Build %s failed" % (job_data['job_name']))
else: # Unknown error
failed = True
status = 'error'
log.error("Unknown build exit status %s for build %s" % (exit_code, job_data['job_name']))
return failed, status