def main(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
# Force user auth
package_name = None
url = urlparse(args.url)
builds = get_gitrepo(url)
ghowner, ghrepo = builds['repo'].split('/', 1)
if not args.package:
package_name = ghrepo
log.info("Using repo name '%s' as the pkg name." % package_name)
user = binstar.user()
user_name = user['login']
args.package = PackageSpec(user_name, package_name)
binstar = get_binstar(args, cls=BinstarBuildAPI)
try:
_ = binstar.package(args.package.user, args.package.name)
except errors.NotFound:
raise errors.UserError("Package %s does not exist" % (args.package,))
log.info("Submitting the following repo for package creation: %s" % args.url)
# TODO: change channels= to labels=
binstar.add_ci(args.package.user, args.package.name,
ghowner=ghowner, ghrepo=ghrepo,
channels=args.labels, queue=args.queue, sub_dir=args.sub_dir,
email=args.email)
log.info("CI Added to package %s", args.package)
def main(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
# Force user auth
package_name = None
url = urlparse(args.url)
builds = get_gitrepo(url)
ghowner, ghrepo = builds['repo'].split('/', 1)
if not args.package:
package_name = ghrepo
log.info("Using repo name '%s' as the pkg name." % package_name)
user = binstar.user()
user_name = user['login']
args.package = PackageSpec(user_name, package_name)
binstar = get_binstar(args, cls=BinstarBuildAPI)
try:
_ = binstar.package(args.package.user, args.package.name)
except errors.NotFound:
raise errors.UserError("Package %s does not exist" % (args.package,))
log.info("Submitting the following repo for package creation: %s" % args.url)
# TODO: change channels= to labels=
binstar.add_ci(args.package.user, args.package.name,
ghowner=ghowner, ghrepo=ghrepo,
channels=args.labels, queue=args.queue, sub_dir=args.sub_dir,
email=args.email)
log.info("CI Added to package %s", args.package)
def search(args):
binstar = get_binstar()
log.info("Run 'binstar show <USER/PACKAGE>' to get more details:")
packages = binstar.search(args.name, package_type=args.package_type)
pprint_packages(packages)
log.info("Found %i packages" % len(packages))
def main(args):
args.conda_build_dir = args.conda_build_dir.format(args=args)
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.queue.count('/') == 1:
username, queue = args.queue.split('/', 1)
args.username = username
args.queue = queue
elif args.queue.count('-') == 2:
_, username, queue = args.queue.split('-', 2)
args.username = username
args.queue = queue
else:
raise errors.UserError("Build queue must be of the form build-USERNAME-QUEUENAME or USERNAME/QUEUENAME")
log.info('Starting worker:')
log.info('User: %s' % args.username)
log.info('Queue: %s' % args.queue)
log.info('Platform: %s' % args.platform)
worker = Worker(bs, args)
worker.write_status(True, "Starting")
try:
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(
WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
args.conda_build_dir = args.conda_build_dir.format(
platform=worker_config.platform)
setup_logging(logging.getLogger('binstar_build_client'),
args.log_level,
args.color,
show_tb=args.show_traceback)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
worker.write_stats()
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def main(args):
bs = get_binstar(args)
spec = args.spec
owner = spec.user
package = spec.package
if args.add_collaborator:
collaborator = args.add_collaborator
bs.package_add_collaborator(owner, package, collaborator)
args.add_collaborator
elif args.list_collaborators:
log.info(":Collaborators:")
for collab in binstar.package_collaborators(owner, package):
log.info(" +", collab["login"])
elif args.create:
public = args.access != "private"
bs.add_package(
args.spec.user,
args.spec.package,
args.summary,
public=public,
license=args.license,
license_url=args.license_url,
)
log.info("Package created!")
def main(args):
bs = get_binstar()
auth = bs.authentication()
bs.remove_authentication(auth['id'])
remove_token()
log.info("logout successful")
def submit_build(args):
binstar = get_binstar(args)
path = abspath(args.path)
log.info("Getting build product: %s" % abspath(args.path))
with open(join(path, ".binstar.yml")) as cfg:
build_matrix = list(yaml.load_all(cfg))
builds = list(serialize_builds(build_matrix))
log.info("Submitting %i sub builds" % len(builds))
for i, build in enumerate(builds):
log.info(" %i)" % i + " %(platform)-10s %(engine)-15s %(env)-15s" % build)
if not args.dry_run:
with mktemp() as tmp:
with tarfile.open(tmp, mode="w|bz2") as tf:
tf.add(path, ".")
with open(tmp, mode="rb") as fd:
build_no = binstar.submit_for_build(args.package.user, args.package.name, fd, builds)
log.info("Build %s submitted" % build_no)
else:
log.info("Build not submitted (dry-run)")
def main(args):
bs = get_binstar(args)
spec = args.spec
owner = spec.user
package = spec.package
if args.add_collaborator:
collaborator = args.add_collaborator
bs.package_add_collaborator(owner, package, collaborator)
args.add_collaborator
elif args.list_collaborators:
log.info(':Collaborators:')
for collab in binstar.package_collaborators(owner, package):
log.info(' +', collab['login'])
elif args.create:
public = args.access != 'private'
bs.add_package(args.spec.user,
args.spec.package,
args.summary,
public=public,
license=args.license,
license_url=args.license_url)
log.info('Package created!')
def resubmit_build(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
binstar.resubmit_build(args.package.user, args.package.name, args.build_no)
log.info("Build %s resubmitted" % args.build_no)
def search(args):
binstar = get_binstar()
log.info("Run 'binstar show <USER/PACKAGE>' to get more details:")
packages = binstar.search(args.name, package_type=args.package_type)
pprint_packages(packages)
log.info("Found %i packages" % len(packages))
def main(args):
binstar = get_binstar()
if '/' in args.spec:
user, package = args.spec.split('/', 1)
else:
user = args.spec
package = None
if args.attrs:
with open(args.attrs) as fd:
attrs = json.load(fd)
else:
attrs = {}
if args.action == 'add':
binstar.add_package(user, package, args.type,
args.summary, args.license, args.license_url,
args.public, attrs)
elif args.action == 'show':
print binstar.package(user, package)
elif args.action == 'list':
package_list(binstar.user_packages(user), args.verbose)
else:
raise NotImplementedError(args.action)
def interactive_get_token():
bs = get_binstar()
config = get_config()
url = config.get('url', 'https://api.binstar.org')
token = None
username = input('Username: '******'s " % username)
password = getpass.getpass(stream=sys.stderr)
token = bs.authenticate(username,
password,
'binstar_client:%s' %
(socket.gethostname()),
url,
created_with=' '.join(sys.argv))
break
except Unauthorized:
log.error(
'Invalid Username password combination, please try again')
continue
if token is None:
raise BinstarError(
'Sorry. Please try again (go to https://binstar.org/account/forgot_password to reset your password)'
)
return token
def tail(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
log_items = binstar.tail_build(args.package.user,
args.package.name,
args.build_no,
limit=args.n)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry']
while args.f and not log_items.get('finished'):
time.sleep(4)
log_items = binstar.tail_build(args.package.user,
args.package.name,
args.build_no,
after=last_entry)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry'] or last_entry
if log_items.get('finished'):
if log_items['failed']:
log.error('Build Failed')
else:
log.info('Build Succeeded')
else:
log.info('... Build still running ...')
def publish(args):
binstar = get_binstar()
spec = args.spec[0]
if args.test:
print binstar.published(spec.user, spec.package)
return
binstar.publish(spec.user, spec.package)
if not args.quiet:
package = binstar.package(spec.user, spec.package)
if 'pypi' in package['package_types']:
if sys.platform.startswith('win'):
pypirc = '%HOME%\pip\pip.ini'
else:
pypirc = '$HOME/.pip/pip.conf'
print
print 'Package is added to public pypi repository:'
print 'Please add the following lines to your pip config file (%r)' % (pypirc,)
print '''[install]
find-links =
https://pypi.binstar.org/simple
'''
if 'conda' in package['package_types']:
print
print 'Package is added to public conda repository:'
print 'Please run the following command to add the public chanel to your conda search path'
print 'conda config --add channel https://conda.binstar.org/public'
def main(args):
binstar = get_binstar(args)
spec = args.spec
if args.action == 'upload':
for file in args.files:
attrs = detect(binstar, spec.user, spec.package, file)
with open(file) as fd:
print 'Uploading %s ... ' % file
binstar.upload(spec.user, spec.package, spec.version, basename(file), fd, args.description, **attrs)
print '... done'
elif args.action == 'download':
requests_handle = binstar.download(spec.user, spec.package, spec.version, spec.basename)
if args.files:
fname = args.files[0]
with open(fname, 'w') as fdout:
for chunk in requests.handle.iter_content(4096):
fdout.write(chunk)
elif args.action == 'list':
release = binstar.release(spec.user, spec.package, spec.version)
for dist in release.get('distributions',[]):
print '%(basename)s id: %(_id)s' % dist
for key_value in dist['attrs'].items():
print ' + %s: %r' % key_value
elif args.action == 'remove':
print spec.user, spec.package, spec.version, spec.basename, spec.attrs
print binstar.remove_dist(spec.user, spec.package, spec.version, spec.basename, spec.attrs)
else:
raise NotImplementedError(args.action)
def main(args):
binstar = get_binstar(args)
for spec in args.specs:
try:
if spec._basename:
msg = 'Are you sure you want to remove file %s ?' % (spec, )
if not args.force and bool_input(msg, False):
binstar.remove_dist(spec.user, spec.package, spec.version,
spec.basename)
else:
log.warn('Not removing file %s' % (spec))
elif spec._version:
msg = 'Are you sure you want to remove the package release %s ? (and all files under it?)' % (
spec, )
if not args.force and bool_input(msg, False):
binstar.remove_release(spec.user, spec.package,
spec.version)
else:
log.warn('Not removing release %s' % (spec))
elif spec._package:
msg = 'Are you sure you want to remove the package %s ? (and all data with it?)' % (
spec, )
if not args.force and bool_input(msg, False):
binstar.remove_package(spec.user, spec.package)
else:
log.warn('Not removing release %s' % (spec))
except NotFound:
if args.force:
continue
else:
raise
def main(args):
if docker is None:
raise errors.UserError(
"binstar-build docker-worker requires docker and dockerpy to be installed\n"
"Run:\n\tpip install dockerpy")
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.queue.count('/') == 1:
username, queue = args.queue.split('/', 1)
args.username = username
args.queue = queue
elif args.queue.count('-') == 2:
_, username, queue = args.queue.split('-', 2)
args.username = username
args.queue = queue
else:
raise errors.UserError(
"Build queue must be of the form build-USERNAME-QUEUENAME or USERNAME/QUEUENAME"
)
log.info('Starting worker:')
log.info('User: %s' % args.username)
log.info('Queue: %s' % args.queue)
log.info('Platform: %s' % args.platform)
woker = DockerWorker(bs, args)
woker.work_forever()
def main():
token = os.environ.get('BINSTAR_TOKEN')
description = ('Upload or check consistency of a built version of a '
'conda recipe with binstar. Note: The existence of the '
'BINSTAR_TOKEN environment variable determines '
'whether the upload should actually take place.')
parser = argparse.ArgumentParser(description=description)
parser.add_argument('recipe_dir', help='the conda recipe directory')
parser.add_argument('owner', help='the binstar owner/user')
parser.add_argument('--channel', help='the binstar channel', default='main')
args = parser.parse_args()
recipe_dir, owner, channel = args.recipe_dir, args.owner, args.channel
cli = get_binstar(argparse.Namespace(token=token, site=None))
meta_main = MetaData(recipe_dir)
for _, meta in meta_main.get_output_metadata_set(files=None):
print("Processing {}".format(meta.name()))
if meta.skip():
print("No upload to take place - this configuration was skipped in build/skip.")
return
exists = built_distribution_already_exists(cli, meta, owner)
if token:
if not exists:
upload(cli, meta, owner, channel)
print('Uploaded {}'.format(bldpkg_path(meta)))
else:
print('Distribution {} already \nexists for {}.'
''.format(bldpkg_path(meta), owner))
else:
print("No BINSTAR_TOKEN present, so no upload is taking place. "
"The distribution just built {} already available for {}."
"".format('is' if exists else 'is not', owner))
def __init__(self, conda_recipes_root, upload_owner, upload_channel):
"""
Build a directory of conda recipes sequentially, if they don't already exist on the owner's binstar account.
If the build does exist on the binstar account, but isn't in the targeted channel, it will be added to upload_channel,
All built distributions will be uploaded to the owner's channel.
"""
self.conda_recipes_root = conda_recipes_root
self.upload_owner = upload_owner
self.upload_channel = upload_channel
self.binstar_token = os.environ.get('BINSTAR_TOKEN', None)
self.can_upload = self.binstar_token is not None
if not self.can_upload:
print('**Build will continue, but no uploads will take place.**')
print(
'To automatically upload from this script, define the BINSTAR_TOKEN env variable.'
)
print(
'This is done automatically on the travis-ci system once the PR has been merged.'
)
self.binstar_cli = get_binstar(
Namespace(token=self.binstar_token, site=None))
def main(args):
aserver_api = get_binstar(args)
if args.organization:
owner = args.organization
else:
current_user = aserver_api.user()
owner = current_user['login']
if args.copy:
aserver_api.copy_channel(args.copy[0], owner, args.copy[1])
log.info("Copied channel %s to %s" % tuple(args.copy))
elif args.remove:
aserver_api.remove_channel(args.remove, owner)
log.info("Removed channel %s" % args.remove)
elif args.list:
log.info('Channels')
for channel, info in aserver_api.list_channels(owner).items():
if isinstance(info, int): # OLD API
log.info((' + %s ' % channel))
else:
log.info((' + %s ' % channel) + ('[locked]' if info['is_locked'] else ''))
elif args.show:
info = aserver_api.show_channel(args.show, owner)
log.info(('Channel %s ' % args.show) + ('[locked]' if info['is_locked'] else ''))
for f in info['files']:
log.info(' + %(full_name)s' % f)
elif args.lock:
aserver_api.lock_channel(args.lock, owner)
log.info("Channel %s is now locked" % args.lock)
elif args.unlock:
aserver_api.unlock_channel(args.unlock, owner)
log.info("Channel %s is now unlocked" % args.unlock)
else:
raise NotImplementedError()
def main(args):
binstar = get_binstar(args)
for spec in args.specs:
try:
if spec._basename:
msg = 'Are you sure you want to remove file %s ?' % (spec,)
if args.force or bool_input(msg, False):
binstar.remove_dist(spec.user, spec.package, spec.version, spec.basename)
else:
log.warn('Not removing file %s' % (spec))
elif spec._version:
msg = 'Are you sure you want to remove the package release %s ? (and all files under it?)' % (spec,)
if args.force or bool_input(msg, False):
binstar.remove_release(spec.user, spec.package, spec.version)
else:
log.warn('Not removing release %s' % (spec))
elif spec._package:
msg = 'Are you sure you want to remove the package %s ? (and all data with it?)' % (spec,)
if args.force or bool_input(msg, False):
binstar.remove_package(spec.user, spec.package)
else:
log.warn('Not removing release %s' % (spec))
except NotFound:
if args.force:
continue
else:
raise
def main(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
build_no = binstar.trigger_build(
args.package.user,
args.package.name,
channels=args.channels,
queue_name=args.queue,
branch=args.branch,
test_only=args.test_only,
filter_platform=args.platform,
)
log.info("")
log.info(
"To view this build go to http://alpha.binstar.org/%s/%s/builds/matrix/%s"
% (args.package.user, args.package.name, build_no)
)
log.info("")
log.info(
"You may also run\n\n binstar-build tail -f %s/%s %s\n" % (args.package.user, args.package.name, build_no)
)
log.info("")
log.info("Build %s submitted" % build_no)
def main(args):
binstar = get_binstar(args)
if args.list:
show_auths(binstar.authentications())
return
elif args.remove:
for auth_id in args.remove:
binstar.remove_authentication(auth_id)
return
elif args.list_scopes:
scopes = binstar.list_scopes()
for key in sorted(scopes):
print key
print ' ', scopes[key]
print
elif args.create:
sys.stderr.write('Username: '******'')
password = getpass.getpass()
scopes = [scope for scopes in args.scopes for scope in scopes.split()]
print binstar.authenticate(username, password,
args.name, application_url=args.url,
scopes=scopes,
for_user=args.user,
max_age=args.max_age,
created_with=' '.join(sys.argv))
def construct_build_list(packages, conda_channel=None):
channel = conda_channel or BINSTAR_CHANNEL
conda_py = _conda_python_build_string()
for package in packages:
print("Checking status of {}...".format(package.conda_name), end="")
binstar = get_binstar()
# Decide whether the package needs to be built by checking to see if
# it exists on binstar.
try:
binstar_info = binstar.release(channel, package.conda_name, package.required_version)
except NotFound:
# No builds for this version on any platform, so need to build.
package.build = True
if not package.build:
# We have binstar_info, need to check whether we have this
# platform and python version.
for d in binstar_info["distributions"]:
if d["attrs"]["subdir"] == config.subdir and conda_py in d["attrs"]["build"]:
break
else:
package.build = True
build_message = "BUILD"
if not package.build:
build_message = ("DO NOT " + build_message).lower()
elif package.is_dev:
build_message = "skip because package version is dev"
print(build_message)
return [p for p in packages if p.build and not p.is_dev and p.url]
def main(args):
binstar = get_binstar(args)
if args.organization:
owner = args.organization
else:
current_user = binstar.user()
owner = current_user["login"]
if args.copy:
binstar.copy_channel(args.copy[0], owner, args.copy[1])
log.info("Copied channel %s to %s" % tuple(args.copy))
elif args.remove:
binstar.remove_channel(args.remove, owner)
log.info("Removed channel %s" % args.remove)
elif args.list:
log.info("Channels")
for channel, info in binstar.list_channels(owner).items():
if isinstance(info, int): # OLD API
log.info((" + %s " % channel))
else:
log.info((" + %s " % channel) + ("[locked]" if info["is_locked"] else ""))
elif args.show:
info = binstar.show_channel(args.show, owner)
log.info(("Channel %s " % args.show) + ("[locked]" if info["is_locked"] else ""))
for f in info["files"]:
log.info(" + %(full_name)s" % f)
elif args.lock:
binstar.lock_channel(args.lock, owner)
log.info("Channel %s is now locked" % args.lock)
elif args.unlock:
binstar.unlock_channel(args.unlock, owner)
log.info("Channel %s is now unlocked" % args.unlock)
else:
raise NotImplementedError()
def main(args):
binstar = get_binstar()
spec = args.spec
if spec._basename:
print 'file'
elif args.spec._version:
print 'version', spec.version
release = binstar.release(spec.user, spec.package, spec.version)
for dist in release['distributions']:
print ' + %(basename)s' % dist, ' platform:%(platform)-10s arch:%(arch)-10s' % dist['attrs']
print
print '%(description)s' % release['public_attrs']
elif args.spec._package:
package = binstar.package(spec.user, spec.package)
package['permission'] = 'public' if package['public'] else 'private'
print '- %(name)s: [%(permission)s] %(summary)s' % package
for release in package['releases']:
print ' + %(version)s' % release
elif args.spec._user:
print binstar.user(spec.user)
for package in binstar.user_packages(spec.user):
package['permission'] = 'public' if package['public'] else 'private'
print ' + %(name)25s: [%(permission)s] %(summary)s' % package
else:
print args.spec
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.queue.count('/') == 1:
username, queue = args.queue.split('/', 1)
args.username = username
args.queue = queue
elif args.queue.count('-') == 2:
_, username, queue = args.queue.split('-', 2)
args.username = username
args.queue = queue
else:
raise errors.UserError(
"Build queue must be of the form build-USERNAME-QUEUENAME or USERNAME/QUEUENAME"
)
backlog = bs.build_backlog(args.username, args.queue)
print("Backlog for queue %s/%s" % (args.username, args.queue))
header = {
'name': 'Package',
'build_no': 'Build',
'tags': 'Platform',
'enqueued': 'Enqueued'
}
print('%(name)-30s | %(build_no)-10s | %(tags)-10s | %(enqueued)-30s' %
header)
print('-' * 89)
for job in backlog:
job['tags'] = ', '.join(job['tags'])
print('%(name)-30s | %(build_no)10s | %(tags)10s | %(enqueued)30s' %
job)
def init_build(args):
binstar = get_binstar()
# Force user auth
user = binstar.user()
binstar_yml = join(args.path, '.binstar.yml')
if os.path.exists(binstar_yml):
result = bool_input("The file '%s' already exists. Would you like to overwrite it?" % binstar_yml,
default=False)
if not result:
log.error('goodbye')
sys.exit(1)
name = basename(abspath(args.path))
package_name = input('Please choose a name for this package: (default %s)\n> ' % name)
package_name = package_name or name
with open(binstar_yml, 'w') as fd:
fd.write(initial_build_config % dict(PACKAGE_NAME=package_name))
log.info("Wrote file '%s'" % binstar_yml)
try:
_ = binstar.package(user['login'], package_name)
except errors.NotFound:
log.warn('The package %(username)s/%(name)s does not exist\n'
'Please run:\n binstar package %(username)s/%(name)s --create' % dict(username=user['login'], name=package_name))
log.info("Run 'binstar build --submit' to submit your first build")
return
def main(args):
bs = get_binstar(args)
spec = args.spec
channels = bs.list_channels(spec.user)
if args.from_channel not in channels:
raise errors.UserError(
"Channel %s does not exist\n\tplease choose from: %s" %
(args.from_channel, ', '.join(channels)))
files = bs.copy(spec.user,
spec.package,
spec.version,
spec._basename,
to_owner=args.to_owner,
from_channel=args.from_channel,
to_channel=args.to_channel)
for binstar_file in files:
print("Copied file: %(basename)s" % binstar_file)
if files:
log.info("Copied %i files" % len(files))
else:
log.warning(
"Did not copy any files. Please check your inputs with\n\n\tanaconda show %s"
% spec)
def tail(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
log_items = binstar.tail_build(args.package.user, args.package.name, args.build_no, limit=args.n)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry']
while args.f and not log_items.get('finished'):
time.sleep(4)
log_items = binstar.tail_build(args.package.user, args.package.name, args.build_no,
after=last_entry)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry'] or last_entry
if log_items.get('finished'):
if log_items['failed']:
log.error('Build Failed')
else:
log.info('Build Succeeded')
else:
log.info('... Build still running ...')
def main(args):
bs = get_binstar()
auth = bs.authentication()
bs.remove_authentication(auth['id'])
remove_token()
log.info("logout successful")
def tail(package_user, package_name, build_no,
limit, follow=True, binstar=None, binstar_args=None):
if binstar is None:
binstar = get_binstar(binstar_args, cls=BinstarBuildAPI)
log_items = binstar.tail_build(package_user, package_name,
build_no, limit=limit)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry']
while follow and not log_items.get('finished'):
time.sleep(4)
log_items = binstar.tail_build(package_user, package_name,
build_no, after=last_entry)
for log_item in log_items['log']:
log.info(log_item.get('msg'))
last_entry = log_items['last_entry'] or last_entry
if log_items.get('finished'):
if log_items['failed']:
log.error('Build Failed')
else:
log.info('Build Succeeded')
else:
log.info('... Build still running ...')
def submit_build(args):
binstar = get_binstar(args)
path = abspath(args.path)
log.info('Getting build product: %s' % abspath(args.path))
with open(join(path, '.binstar.yml')) as cfg:
build_matrix = list(yaml.load_all(cfg))
builds = list(serialize_builds(build_matrix))
log.info('Submitting %i sub builds' % len(builds))
for i, build in enumerate(builds):
log.info(' %i)' % i +
' %(platform)-10s %(engine)-15s %(env)-15s' % build)
if not args.dry_run:
with mktemp() as tmp:
with tarfile.open(tmp, mode='w|bz2') as tf:
tf.add(path, '.')
with open(tmp, mode='rb') as fd:
build_no = binstar.submit_for_build(args.package.user,
args.package.name, fd,
builds)
log.info('Build %s submitted' % build_no)
else:
log.info('Build not submitted (dry-run)')
def init_binstar(quiet=False):
global binstar_client, binstar_domain, binstar_domain_tok
global binstar_regex, BINSTAR_TOKEN_PAT
if binstar_domain is not None:
return
elif binstar_client is None:
try:
from binstar_client.utils import get_binstar
# Turn off output in offline mode so people don't think we're going online
args = namedtuple("args", "log_level")(0) if quiet or offline else None
binstar_client = get_binstar(args)
except ImportError:
log.debug("Could not import binstar")
binstar_client = ()
except Exception as e:
stderrlog.info("Warning: could not import binstar_client (%s)" % e)
if binstar_client == ():
binstar_domain = DEFAULT_CHANNEL_ALIAS
binstar_domain_tok = None
else:
binstar_domain = binstar_client.domain.replace("api", "conda").rstrip("/") + "/"
if add_anaconda_token and binstar_client.token:
binstar_domain_tok = binstar_domain + "t/%s/" % (binstar_client.token,)
binstar_regex = r"((:?%s|binstar\.org|anaconda\.org)/?)(t/[0-9a-zA-Z\-<>]{4,})/" % re.escape(binstar_domain[:-1])
BINSTAR_TOKEN_PAT = re.compile(binstar_regex)
def __init__(self, name=None, **kwargs):
self.name = name
self.quiet = False
if get_binstar is not None:
self.binstar = get_binstar()
else:
self.binstar = None
def resubmit_build(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
binstar.resubmit_build(args.package.user, args.package.name, args.build_no)
log.info("Build %s resubmitted" % args.build_no)
def __init__(self, conda_recipes_root, upload_owner, upload_channel):
"""
Build a directory of conda recipes sequentially, if they don't already exist on the owner's binstar account.
If the build does exist on the binstar account, but isn't in the targeted channel, it will be added to upload_channel,
All built distributions will be uploaded to the owner's channel.
Note: Recipes may not compute their version/build# at build time.
"""
self.conda_recipes_root = conda_recipes_root
self.upload_owner = upload_owner
self.upload_channel = upload_channel
self.binstar_token = os.environ.get('BINSTAR_TOKEN', None)
self.can_upload = self.binstar_token is not None
if not self.can_upload:
print('**Build will continue, but no uploads will take place.**')
print('To automatically upload from this script, define the BINSTAR_TOKEN env variable.')
print('This is done automatically on the travis-ci system once the PR has been merged.')
else:
print('conda build currently leaks all environment variables on windows, therefore the BINSTAR_TOKEN '
'is being reset. See https://github.com/conda/conda-build/pull/274 for progress.')
os.environ['BINSTAR_TOKEN'] = 'Hidden by Obvious-CI'
self.binstar_cli = get_binstar(Namespace(token=self.binstar_token, site=None))
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs, warn=True)
WorkerConfiguration.validate_worker_name(bs, args.worker_id)
if worker_config.hostname != WorkerConfiguration.HOSTNAME:
log.warn(WRONG_HOSTNAME_MSG.format(worker_config.hostname,
WorkerConfiguration.HOSTNAME))
args.conda_build_dir = args.conda_build_dir.format(platform=worker_config.platform)
setup_logging(logging.getLogger('binstar_build_client'), args.log_level,
args.color, show_tb=args.show_traceback)
log.info("Using conda build directory: {}".format(args.conda_build_dir))
log.info(str(worker_config))
worker = Worker(bs, worker_config, args)
worker.write_status(True, "Starting")
worker.write_stats()
try:
with worker_config.running():
worker.work_forever()
finally:
worker.write_status(False, "Exited")
def __init__(self, name=None, **kwargs):
self.name = name
self.quiet = False
if get_binstar is not None:
self.binstar = get_binstar()
else:
self.binstar = None
def main(args):
binstar = get_binstar(args)
spec = args.spec
if args.action == 'upload':
for file in args.files:
attrs = detect(binstar, spec.user, spec.package, file)
with open(file) as fd:
print 'Uploading %s ... ' % file
binstar.upload(spec.user, spec.package, spec.version,
basename(file), fd, args.description, **attrs)
print '... done'
elif args.action == 'download':
requests_handle = binstar.download(spec.user, spec.package,
spec.version, spec.basename)
if args.files:
fname = args.files[0]
with open(fname, 'w') as fdout:
for chunk in requests.handle.iter_content(4096):
fdout.write(chunk)
elif args.action == 'list':
release = binstar.release(spec.user, spec.package, spec.version)
for dist in release.get('distributions', []):
print '%(basename)s id: %(_id)s' % dist
for key_value in dist['attrs'].items():
print ' + %s: %r' % key_value
elif args.action == 'remove':
print spec.user, spec.package, spec.version, spec.basename, spec.attrs
print binstar.remove_dist(spec.user, spec.package, spec.version,
spec.basename, spec.attrs)
else:
raise NotImplementedError(args.action)
def list_builds(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
log.info('Getting builds:')
fmt = '%(build_no)15s | %(status)15s | %(platform)15s | %(engine)15s | %(env)15s'
header = {
'build_no': 'Build #',
'status': 'Status',
'platform': 'Platform',
'engine': 'Engine',
'env': 'Env',
}
log.info(fmt % header)
log.info(fmt.replace('|', '+') % dict.fromkeys(header, '-' * 15))
for build in binstar.builds(args.package.user, args.package.name,
args.build_no):
for item in build['items']:
item.setdefault('status', build.get('status', '?'))
item['build_no'] = '%s.%s' % (build['build_no'],
item['sub_build_no'])
log.info(fmt % item)
log.info('')
return
def main(args):
bs = get_binstar()
auth = bs.authentication()
bs.remove_authentication(auth['id'])
kr = get_keyring()
kr.delete_password('binstar-token', getpass.getuser())
log.info("logout successful")
def main(args):
bs = get_binstar()
auth = bs.authentication()
bs.remove_authentication(auth['id'])
kr = get_keyring()
kr.delete_password('binstar-token', getpass.getuser())
log.info("logout successful")
def search(args):
aserver_api = get_binstar(args)
log.info("Run 'anaconda show <USER/PACKAGE>' to get more details:")
packages = aserver_api.search(args.name, package_type=args.package_type)
pprint_packages(packages, access=False)
log.info("Found %i packages" % len(packages))
def main(args):
bs = get_binstar(args)
spec = args.spec
result = bs.copy(spec.user, spec.package, spec.version, spec._basename,
to_owner=args.to_owner, from_channel=args.from_channel, to_channel=args.to_channel)
print result
def search(args):
aserver_api = get_binstar(args)
log.info("Run 'anaconda show <USER/PACKAGE>' to get more details:")
packages = aserver_api.search(args.name, package_type=args.package_type)
pprint_packages(packages, access=False)
log.info("Found %i packages" % len(packages))
def main(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
package_name = None
user_name = None
if args.git_url:
args.path = args.git_url
if args.git_url or is_url(args.path):
args.git_url = args.path
args.git_url_path = get_urlpath(args.path)
args.dont_git_ignore = True
submit_git_build(binstar, args)
# not a github repo (must check for valid .binstar.yml file
else:
binstar_yml = join(args.path, '.binstar.yml')
if not isfile(binstar_yml):
raise UserError("file %s does not exist\n perhaps you should run\n\n binstar-build init\n" % binstar_yml)
package_name = None
user_name = None
with open(binstar_yml) as cfg:
for build in yaml.load_all(cfg):
if build.get('package'):
package_name = build.get('package')
if build.get('user'):
user_name = build.get('user')
# Force package to exist
if args.package:
if user_name and not args.package.user == user_name:
log.warn('User name does not match the user specified in the .binstar.yml file (%s != %s)', args.package.user, user_name)
user_name = args.package.user
if package_name and not args.package.name == package_name:
log.warn('Package name does not match the user specified in the .binstar.yml file (%s != %s)', args.package.name, package_name)
package_name = args.package.name
else:
if user_name is None:
user = binstar.user()
user_name = user['login']
if not package_name:
raise UserError("You must specify the package name in the .binstar.yml file or the command line")
try:
_ = binstar.package(user_name, package_name)
except errors.NotFound:
log.error("The package %s/%s does not exist." % (user_name, package_name))
log.error("Run: \n\n binstar package --create %s/%s\n\n to create this package" % (user_name, package_name))
raise errors.NotFound('Package %s/%s' % (user_name, package_name))
args.package = PackageSpec(user_name, package_name)
submit_build(binstar, args)
def pull(source, target_type="filename", callback=print_callback):
args = Namespace(site=None, token=None)
binstar = get_binstar(args)
if "/" in source:
username = source.split("/", 1)[0]
else:
user = binstar.user()
username = user["login"]
package_name = source.rsplit("/", 1)[-1]
pkg = binstar.package(username, package_name)
file_data = max(pkg["files"], key=semantic_version_key)
req = binstar.download(username, package_name, file_data["version"], file_data["basename"])
filename = "+".join([username, package_name, file_data["version"], file_data["basename"]])
filename = filename.lower()
cache_file = os.path.join(dirs.user_data_dir, "conda-data", filename)
callback(0, 0)
info_file = os.path.join(dirs.user_data_dir, "conda-data", "info", filename.rsplit(".", 1)[0] + ".json")
if os.path.isfile(cache_file):
md5 = md5_file(cache_file)
if file_data["md5"] == md5:
if target_type == "filename":
return cache_file
return odo(cache_file.encode(), pd.DataFrame)
if not os.path.exists(os.path.dirname(cache_file)):
os.makedirs(os.path.dirname(cache_file))
n = 0
with open(cache_file, "wb") as fd:
data = req.raw.read(CHUNK_SIZE)
n += len(data)
while data:
fd.write(data)
data = req.raw.read(CHUNK_SIZE)
n += len(data)
callback(n, file_data["size"])
callback(file_data["size"], file_data["size"])
if not os.path.exists(os.path.dirname(info_file)):
os.makedirs(os.path.dirname(info_file))
with open(info_file, "w") as fd:
file_data["summary"] = pkg["summary"]
file_data["package"] = pkg["name"]
file_data["cache_file"] = cache_file
json.dump(file_data, fd)
if target_type == "filename":
return cache_file
return odo(cache_file.encode(), target_type)
def main(args):
binstar = get_binstar(args, cls=BinstarBuildAPI)
package_name = None
user_name = None
if args.git_url:
args.path = args.git_url
if args.git_url or is_url(args.path):
args.git_url = args.path
args.git_url_path = get_urlpath(args.path)
args.dont_git_ignore = True
submit_git_build(binstar, args)
# not a github repo (must check for valid .binstar.yml file
else:
binstar_yml = join(args.path, '.binstar.yml')
if not isfile(binstar_yml):
raise UserError("file %s does not exist\n perhaps you should run\n\n binstar-build init\n" % binstar_yml)
package_name = None
user_name = None
with open(binstar_yml) as cfg:
for build in yaml.load_all(cfg):
if build.get('package'):
package_name = build.get('package')
if build.get('user'):
user_name = build.get('user')
# Force package to exist
if args.package:
if user_name and not args.package.user == user_name:
log.warn('User name does not match the user specified in the .binstar.yml file (%s != %s)', args.package.user, user_name)
user_name = args.package.user
if package_name and not args.package.name == package_name:
log.warn('Package name does not match the user specified in the .binstar.yml file (%s != %s)', args.package.name, package_name)
package_name = args.package.name
else:
if user_name is None:
user = binstar.user()
user_name = user['login']
if not package_name:
raise UserError("You must specify the package name in the .binstar.yml file or the command line")
try:
_ = binstar.package(user_name, package_name)
except errors.NotFound:
log.error("The package %s/%s does not exist." % (user_name, package_name))
log.error("Run: \n\n binstar package --create %s/%s\n\n to create this package" % (user_name, package_name))
raise errors.NotFound('Package %s/%s' % (user_name, package_name))
args.package = PackageSpec(user_name, package_name)
submit_build(binstar, args)
def main(args):
binstar = get_binstar(args)
spec = args.spec
if spec._basename:
dist = binstar.distribution(spec.user, spec.package, spec.version,
spec.basename)
log.info(dist.pop('basename'))
log.info(dist.pop('description') or 'no description')
log.info('')
metadata = dist.pop('attrs', {})
for key_value in dist.items():
log.info('%-25s: %r' % key_value)
log.info('Metadata:')
for key_value in metadata.items():
log.info(' + %-25s: %r' % key_value)
elif args.spec._version:
log.info('version %s' % spec.version)
release = binstar.release(spec.user, spec.package, spec.version)
for dist in release['distributions']:
log.info(' + %(basename)s' % dist)
log.info('%s' % release.get('public_attrs', {}).get('description'))
elif args.spec._package:
package = binstar.package(spec.user, spec.package)
package['access'] = 'published' if package.get(
'published') else 'public' if package['public'] else 'private'
log.info('Name: %(name)s' % package)
log.info('Summary: %(summary)s' % package)
log.info('Access: %(access)s' % package)
log.info('Package Types: %s' %
', '.join(package.get('package_types')))
log.info('Versions:' % package)
for release in package['releases']:
log.info(' + %(version)s' % release)
log.info('')
for package_type in package.get('package_types'):
install_info(package, package_type)
if not package['public']:
log.info('To generate a $TOKEN run:')
log.info(' TOKEN=$(binstar auth --create --name <TOKEN-NAME>)')
elif args.spec._user:
user_info = binstar.user(spec.user)
pprint_user(user_info)
pprint_packages(binstar.user_packages(spec.user))
if user_info['user_type'] == 'user':
pprint_orgs(binstar.user_orgs(spec.user))
else:
pprint_collections(binstar.collections(spec.user))
else:
log.info(args.spec)
def main(args):
binstar = get_binstar(args)
if args.info:
data = binstar.authentication()
log.info('Name: %s' % data['application'])
log.info('Id: %s' % data['id'])
if args.list:
show_auths(binstar.authentications())
return
elif args.remove:
for auth_name in args.remove:
binstar.remove_authentication(auth_name)
log.info("Removed token %s" % auth_name)
return
elif args.list_scopes:
scopes = binstar.list_scopes()
for key in sorted(scopes):
log.info(key)
log.info(' ' + scopes[key])
log.info('')
log.info(SCOPE_EXAMPLES)
elif args.create:
try:
current_user = binstar.user()
username = current_user['login']
except:
current_user = None
sys.stderr.write('Username: '******'')
scopes = [scope for scopes in args.scopes for scope in scopes.split()]
if not scopes:
log.warn("You have not specified the scope of this token with the '--scopes' argument.")
log.warn("This token will grant full access to %s's account" % username)
log.warn("Use the --list-scopes option to see a listing of your options")
for _ in range(3):
try:
sys.stderr.write("Please re-enter %s's " % username)
password = getpass.getpass()
token = binstar.authenticate(username, password,
args.name, application_url=args.url,
scopes=scopes,
for_user=args.organization,
max_age=args.max_age,
created_with=' '.join(sys.argv),
strength=args.strength,
fail_if_already_exists=True)
args.out.write(token)
break
except errors.Unauthorized:
log.error('Invalid Username password combination, please try again')
continue
def main(args):
binstar = get_binstar(args)
if args.info:
data = binstar.authentication()
log.info('Name: %s' % data['application'])
log.info('Id: %s' % data['id'])
if args.list:
show_auths(binstar.authentications())
return
elif args.remove:
for auth_name in args.remove:
binstar.remove_authentication(auth_name)
log.info("Removed token %s" % auth_name)
return
elif args.list_scopes:
scopes = binstar.list_scopes()
for key in sorted(scopes):
log.info(key)
log.info(' ' + scopes[key])
log.info('')
log.info(SCOPE_EXAMPLES)
elif args.create:
try:
current_user = binstar.user()
username = current_user['login']
except:
current_user = None
sys.stderr.write('Username: '******'')
scopes = [scope for scopes in args.scopes for scope in scopes.split()]
if not scopes:
log.warn("You have not specified the scope of this token with the '--scopes' argument.")
log.warn("This token will grant full access to %s's account" % (args.organization or username))
log.warn("Use the --list-scopes option to see a listing of your options")
for _ in range(3):
try:
sys.stderr.write("Please re-enter %s's " % username)
password = getpass.getpass()
token = binstar.authenticate(username, password,
args.name, application_url=args.url,
scopes=scopes,
for_user=args.organization,
max_age=args.max_age,
created_with=' '.join(sys.argv),
strength=args.strength,
fail_if_already_exists=True)
args.out.write(token)
break
except errors.Unauthorized:
log.error('Invalid Username password combination, please try again')
continue
def main(args):
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.queue is None:
username = queue_name = None
elif args.queue.count('/') == 1:
username, queue_name = args.queue.split('/', 1)
elif args.queue.count('/') == 2:
_, username, queue_name = args.queue.split('/', 2)
elif args.queue.count('-') == 2:
_, username, queue_name = args.queue.split('-', 2)
else:
raise errors.UserError(
"Build queue must be of the form build-USERNAME-QUEUENAME or USERNAME/QUEUENAME"
)
if args.create:
if queue_name is None:
raise errors.BinstarError("Must specify a queue name to create")
if not is_valid_name(queue_name):
raise errors.BinstarError(
'Invalid name for '
'queue: {}. Must start'
' with a letter and contain'
' only numbers, letters, -, and _'.format(queue_name))
bs.add_build_queue(username, queue_name)
print("Created queue %s" % queue_name)
return
if queue_name:
queue = bs.build_queue(username, queue_name)
if args.remove:
if queue.get('workers'):
prompt = ('This build queue still has workers attached. '
'Are you sure you want to remove it')
if not bool_input(prompt, False):
print("Not removing queue")
return
bs.remove_build_queue(username, queue_name)
print("Removed queue %s" % queue_name)
return
if args.remove_worker:
bs.remove_worker(username, queue_name, args.remove_worker)
print("Removed worker %s from queue %s" %
(args.remove_worker, queue_name))
return
if queue_name:
print()
show_queue(queue)
else:
show_queues(bs, username)
def __init__(self, name, content, public=True, user=None, env_name=None):
self.aserver_api = get_binstar()
self.name = parameterize(name)
self.content = content
self.summary = "IPython notebook"
self.public = public
self.username = user
self.env_name = env_name
if self.username is None:
self.username = self.aserver_api.user()['login']
def main(args, context="worker"):
bs = get_binstar(args, cls=BinstarBuildAPI)
if args.all:
WorkerConfiguration.deregister_all(bs)
elif args.worker_id:
wconfig = WorkerConfiguration.load(args.worker_id, bs)
wconfig.deregister(bs)
else:
log.info(context_info)
def main(args):
binstar = get_binstar(args)
try:
user = binstar.user()
except Unauthorized:
log.info('Anonymous User')
return 1
pprint_user(user)
def main(args):
binstar = get_binstar(args)
try:
user = binstar.user()
except Unauthorized:
print 'Anonymous User'
return -1
pprint_user(user)
def main(args):
if docker is None:
raise errors.UserError(
"anaconda worker docker_run requires docker and docker-py to be installed\n"
"Run:\n\tpip install docker-py")
bs = get_binstar(args, cls=BinstarBuildAPI)
worker_config = WorkerConfiguration.load(args.worker_id, bs)
worker = DockerWorker(bs, worker_config, args)
worker.work_forever()
