def build_all():
logger.info('Running build script')
parser = argparse.ArgumentParser()
parser.add_argument('packages', nargs='*')
parser.add_argument('--channel', action='store', required=True)
parser.add_argument('--no-build', action='store_true', required=False)
parser.add_argument('--no-upload', action='store_true', required=False)
parser.add_argument('--del-old-builds',
action='store_true',
required=False)
parser.add_argument('--token', action='store')
args = parser.parse_args()
channel = args.channel
# Grab token from environment variable if not specified
if not args.token and not args.no_upload:
token = os.getenv('ANACONDA_TOKEN')
if not token:
raise ValueError("Token must be provided using `--token` or in "
"environment variable 'ANACONDA_TOKEN', or you "
"can specify --no-upload.")
else:
token = args.token
if token:
client = binstar_client.Binstar(token=token)
files = get_uploaded_files(client, channel)
else:
files = set()
if args.del_old_builds:
try:
shutil.rmtree(BUILD_DIR)
except Exception:
pass
build_path = Path(BUILD_DIR)
if not build_path.exists():
build_path.mkdir()
to_build = check_all(files, channel, packages=args.packages)
built = []
num = 0
for _, (package, channel, py, np, full_path) in sorted(to_build.items()):
if full_path not in built:
num += 1
built.append(full_path)
if not args.no_build:
build(package, channel, py=py, np=np)
if not args.no_upload:
upload(client, channel, full_path)
if num == 0:
logger.info('Done. Built 0 packages.')
else:
logger.info('Done. Built {} packages:'.format(num))
for pkg in built:
logger.info(pkg)
def build_all():
allowed_host = 'psbuild-rhel6'
if gethostname() != allowed_host:
print('You should be running this on {}!'.format(allowed_host))
return
print('Running build script')
parser = argparse.ArgumentParser()
parser.add_argument('--channel', action='store', required=True)
parser.add_argument('--token', action='store', required=True)
parser.add_argument('--no-build', action='store_true', required=False)
args = parser.parse_args()
channel = args.channel
token = args.token
client = binstar_client.Binstar(token=token)
files = get_uploaded_files(client, channel)
try:
shutil.rmtree(BUILD_DIR)
except Exception:
pass
build_path = Path(BUILD_DIR)
build_path.mkdir()
to_build = check_all(files, channel)
built = []
num = 0
for _, (package, channel, py, np, full_path) in sorted(to_build.items()):
if full_path not in built:
num += 1
built.append(full_path)
if not args.no_build:
build(package, channel, py=py, np=np)
upload(client, channel, full_path)
print('')
if num == 0:
print('Done. Built 0 packages.')
else:
print('Done. Built {} packages:'.format(num))
for pkg in built:
print(pkg)
def get_anaconda_cli(token=None, site=None):
if token is None:
token = get_binstar_token()
return binstar_client.Binstar(token=token, domain=site)
def cli():
p = ArgumentParser("Mirror packages from one channel to a different "
"channel for a given anaconda.org site with an "
"anaconda token. Note: will also work with "
"the BINSTAR_TOKEN environmental variable set or if "
"you have logged in to anaconda via the `anaconda "
"login` command built in to anaconda-client")
p.add_argument(
'packages',
nargs='*',
help="List of package names to mirror from one channel to another")
p.add_argument(
'--all',
action='store_true',
help=("Supercedes `packages` argument. Mirror *all* of the packages "
"from `from-owner` to `to-owner`."),
default=False)
p.add_argument('--list',
action='store_true',
help='List all the packages on --from-user and then exit')
p.add_argument(
'--from-owner',
nargs='?',
help=("anaconda user to mirror packages from. Also acceptable to "
"pass in user/channel. channel will default to main unless "
"explicitly provided"))
p.add_argument(
'--from-domain',
nargs='?',
help="anaconda api domain to mirror from. Only relevant if you are "
"not using anaconda.org",
default="https://api.anaconda.org")
p.add_argument(
'--from-token',
nargs='?',
help=("anaconda token used to authenticate you to the given anaconda "
"site. Required for uploading unless you are logged in (via "
"`anaconda login`)"),
)
p.add_argument('--from-disable-verify',
action='store_false',
help=('ssl verify connection to the `from_site`'),
default=True)
p.add_argument(
'--to-owner',
nargs='?',
help=("anaconda user to mirror packages to. Also acceptable to "
"pass in user/channel. channel will default to main unless "
"explicitly provided"))
p.add_argument(
'--to-domain',
nargs='?',
help="anaconda api domain to mirror to. Only relevant if you are "
"not using anaconda.org",
default="https://api.anaconda.org")
p.add_argument(
'--to-token',
nargs='?',
help=("anaconda token used to authenticate you to the given anaconda "
"site. Required for uploading unless you are logged in (via "
"`anaconda login`)"),
)
p.add_argument('--to-disable-verify',
action='store_false',
help=('ssl verify connection to the `from_site`'),
default=True)
p.add_argument(
'--dry-run',
action='store_true',
help=("Figure out which packages would be copied, print it out and "
"then exit"))
p.add_argument(
'--platform',
nargs="*",
action="store",
help=("Only copy packages for the listed platforms. Options are "
"'osx-32', 'osx-64', 'linux-32', 'linux-64', 'win-32' and "
"'win-64'. Defaults to 'linux-64'"),
default=["linux-64"])
p.add_argument(
'--log',
nargs="?",
action="store",
help="File to log to",
)
p.add_argument(
'--slack-token',
action='store',
nargs='?',
help=("Slack authentication token"),
)
p.add_argument(
'--slack-channel',
action='store',
nargs='?',
help=("Slack channel to post to"),
default="bob-the-builder",
)
args = p.parse_args()
args.to_channel = 'main'
args.from_channel = 'main'
# init some logging
if args.log:
stream = logging.StreamHandler()
filehandler = logging.FileHandler(args.log, mode='a')
stream.setLevel(logging.INFO)
filehandler.setLevel(logging.INFO)
logger.addHandler(stream)
logger.addHandler(filehandler)
logger.setLevel(logging.INFO)
logger.info("Logging to {}".format(args.log))
# set up slack integration
slack_token = args.slack_token
slack_channel = args.slack_channel
slack_api = slacker.Slacker(slack_token)
try:
ret = slack_api.auth.test()
except slacker.Error as e:
slack_api = None
if slack_token is None:
logger.info('No slack token provided. Not sending messages to '
'slack')
else:
logger.error('slack_token {} does grant access to the {} channel'
''.format(slack_token, slack_channel))
logger.error(traceback.format_exc())
logger.error(e)
else:
logger.info("Slack authentication successful.")
logger.info("Authenticating as the %s user", ret.body['user'])
logger.info("Authenticating to the %s team", ret.body['team'])
logger.info("\nSummary")
logger.info("-------")
logger.info("Mirroring from {} at {}".format(args.from_owner,
args.from_domain))
logger.info("Mirroring to {} at {}".format(args.to_owner, args.to_domain))
logger.info("\nPlatforms")
logger.info("---------")
logger.info(pformat(args.platform))
logger.info("\nPackages list")
logger.info("-------------")
if args.all:
logger.info("**all packages**")
else:
logger.info(pformat(args.packages))
try:
args.from_owner, args.from_channel = args.from_owner.split('/')
except ValueError:
# no extra channel information was passed
pass
try:
args.to_owner, args.to_channel = args.to_owner.split('/')
except ValueError:
# no extra channel information was passed
pass
from_cli = binstar_client.Binstar(token=args.from_token,
domain=args.from_domain,
verify=args.from_disable_verify)
# Get the package metadata from the specified anaconda channel
from_packages = from_cli.show_channel(args.from_channel, args.from_owner)
if ',' in args.platform[0]:
pt_list = args.platform[0].split(',')
else:
pt_list = args.platform
# only focus on one platform
pt = pt_list[0]
from_files = {
f['basename']: f
for f in from_packages['files'] if pt in f['full_name']
}
# f['attrs'] may not have key 'subdir'
#from_files = {f['basename']: f for f in from_packages['files']
# if f['attrs']['subdir'] in pt}
if args.list:
# print out the list of all files on the source channel and exit
logger.info("\nComplete files list on {} at {}:".format(
args.from_owner, args.from_domain))
logger.info("-------------------")
logger.info(pformat(sorted(list(from_files.keys()))))
sys.exit(0)
# Find the packages on the source channel that match the packages specified
# on the command line
if args.all:
matched = list(from_files.keys())
else:
matched = [
f for f in from_files.keys() for p in args.packages if p in f
]
# and print them out
logger.info("\nFiles that exist on {} at {}:".format(
args.from_owner, args.from_domain))
logger.info(pformat(sorted(matched)))
# get the package metadata on the target channel
#to_packages = to_cli.show_channel(args.to_channel, args.to_owner)
# folder is at /www/conda/channel_name/
download_dir = os.path.join(os.sep, 'www', 'conda', args.to_owner, pt)
files_existed = glob.glob(os.path.join(download_dir, '*'))
to_files = [f.split(os.sep)[-1] for f in files_existed]
# figure out which packages already exist
already_exist = [f for f in matched if f.split(os.sep)[-1] in to_files]
logger.info("\nFiles that already exist on {} at {}:".format(
args.to_owner, args.to_domain))
logger.info(pformat(sorted(already_exist)))
# figure out which of these packages actually need to be copied
to_copy = [f for f in matched if f.split(os.sep)[-1] not in to_files]
# print out the packages that need to be copied
logger.info(pformat(sorted(to_copy)))
logger.info('number of files to copy: {}'.format(len(to_copy)))
if args.dry_run:
# don't upload anything. Print out why we are quitting and then quit
logger.info("\nExiting because --dry-run flag is set")
sys.exit(0)
for copy_filename in to_copy:
# get the full metadata
md = from_files[copy_filename]
(login, package_name, version, platform,
filename) = md['full_name'].split('/')
destination = os.path.join(download_dir, filename)
logger.info("Downloading {} to {}".format(md['basename'], destination))
ret = from_cli.download(login, package_name, md['version'],
md['basename'])
with open(destination, 'wb+') as f:
f.write(ret.content)
assert os.stat(destination).st_size == md['size']
message = '{} to {} at {}\n'.format(filename, args.to_owner,
destination)
logger.info('Uploading {}'.format(message))
#stdout, stderr, returncode = Popen(upload_cmd + [destination])
if slack_api:
slack_api.chat.post_message(slack_channel,
"Mirrored {}".format(message))
logger.info("Script complete.")