def update_me():
"""
Update the webservice on Heroku by pushing a commit to this repo.
"""
pkgs = ["conda-build", "conda-smithy", "conda-forge-pinning"]
installed_vers = get_installed_version(root_dir, pkgs)
index = get_index(channel_urls=['conda-forge'])
r = Resolve(index)
to_install = {}
for pkg in pkgs:
available_versions = [p.version for p in r.get_pkgs(MatchSpec(pkg))]
available_versions = sorted(available_versions, key=VersionOrder)
latest_version = available_versions[-1]
print(latest_version, installed_vers[pkg])
if VersionOrder(latest_version) > VersionOrder(installed_vers[pkg]):
to_install[pkg] = latest_version
if not to_install:
return
with tmp_directory() as tmp_dir:
repo_name = "conda-forge-webservices"
clone_dir = os.path.join(tmp_dir, repo_name)
url = "https://{}@github.com/conda-forge/{}.git".format(
os.environ['GH_TOKEN'], repo_name)
repo = Repo.clone_from(url, clone_dir)
msg_vers = ", ".join(
["{}={}".format(k, v) for k, v in to_install.items()])
author = Actor("conda-forge-admin", "*****@*****.**")
repo.index.commit("Empty commit to rebuild for {}".format(msg_vers))
repo.git.push("origin", "master")
def distribution_exists_on_channel(binstar_cli,
meta,
fname,
owner,
channel='main'):
"""
Determine whether a distribution exists on a specific channel.
Note from @pelson: As far as I can see, there is no easy way to do this on binstar.
"""
channel_url = '/'.join([owner, 'label', channel])
fname = os.path.basename(fname)
distributions_on_channel = get_index([channel_url],
prepend=False,
use_cache=False)
try:
on_channel = (
distributions_on_channel[fname]['subdir'] == conda_subdir)
except KeyError:
on_channel = False
return on_channel
def main():
"""Get current versions from the heroku app and update if they are old.
Note this script runs on CircleCI, not on the heroku app.
"""
# keep these imports here to protect the webservice from memory errors
# due to conda
from conda_build.conda_interface import (
VersionOrder, MatchSpec, get_index, Resolve)
r = requests.get(
"https://conda-forge.herokuapp.com/conda-webservice-update/versions")
r.raise_for_status()
installed_vers = r.json()
index = get_index(channel_urls=['conda-forge'])
r = Resolve(index)
to_install = {}
final_install = {}
for pkg in PKGS:
available_versions = [p.version for p in r.get_pkgs(MatchSpec(pkg))]
available_versions = sorted(available_versions, key=VersionOrder)
latest_version = available_versions[-1]
LOGGER.info("%s|latest|installed:" % pkg, latest_version, installed_vers[pkg])
if VersionOrder(latest_version) != VersionOrder(installed_vers[pkg]):
to_install[pkg] = latest_version
final_install[pkg] = latest_version
else:
final_install[pkg] = installed_vers[pkg]
if to_install:
tmpdir = None
try:
tmpdir = tempfile.mkdtemp('_recipe')
repo_name = "conda-forge-webservices"
clone_dir = os.path.join(tmpdir, repo_name)
url = "https://{}@github.com/conda-forge/{}.git".format(
os.environ['GH_TOKEN'], repo_name)
repo = Repo.clone_from(url, clone_dir, depth=1)
# keep a record around
pth = os.path.join(clone_dir, "pkg_versions.json")
with open(pth, "w") as fp:
json.dump(final_install, fp)
repo.index.add(pth)
msg_vers = ", ".join(["{}={}".format(k, v) for k, v in to_install.items()])
repo.index.commit("redeploy for '%s'" % msg_vers)
repo.git.push("origin", "master")
finally:
if tmpdir is not None:
shutil.rmtree(tmpdir)
def test_installable(channel='defaults'):
success = True
log = get_logger(__name__)
has_py = re.compile(r'py(\d)(\d)')
for platform in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']:
log.info("######## Testing platform %s ########", platform)
channels = [channel]
index = get_index(channel_urls=channels,
prepend=False,
platform=platform)
for _, rec in iteritems(index):
# If we give channels at the command line, only look at
# packages from those channels (not defaults).
if channel != 'defaults' and rec.get('schannel',
'defaults') == 'defaults':
continue
name = rec['name']
if name in {'conda', 'conda-build'}:
# conda can only be installed in the root environment
continue
if name.endswith('@'):
# this is a 'virtual' feature record that conda adds to the index for the solver
# and should be ignored here
continue
# Don't fail just because the package is a different version of Python
# than the default. We should probably check depends rather than the
# build string.
build = rec['build']
match = has_py.search(build)
assert match if 'py' in build else True, build
if match:
additional_packages = [
'python=%s.%s' % (match.group(1), match.group(2))
]
else:
additional_packages = []
version = rec['version']
log.info('Testing %s=%s', name, version)
try:
install_steps = check_install([name + '=' + version] +
additional_packages,
channel_urls=channels,
prepend=False,
platform=platform)
success &= bool(install_steps)
except KeyboardInterrupt:
raise
# sys.exit raises an exception that doesn't subclass from Exception
except BaseException as e:
success = False
log.error("FAIL: %s %s on %s with %s (%s)", name, version,
platform, additional_packages, e)
return success
def check_install(packages, platform=None, channel_urls=(), prepend=True,
minimal_hint=False):
prefix = tempfile.mkdtemp('conda')
try:
specs = specs_from_args(packages)
index = get_index(channel_urls=channel_urls, prepend=prepend,
platform=platform, prefix=prefix)
actions = install_actions(prefix, index, specs, pinned=False,
minimal_hint=minimal_hint)
display_actions(actions, index)
return actions
finally:
rm_rf(prefix)
return None
def latest_pkg_version(pkg):
'''
:returns: the latest version of the specified conda package available
'''
r = Resolve(get_index())
try:
pkg_list = sorted(r.get_pkgs(MatchSpec(pkg)))
except:
pkg_list = None
if pkg_list:
pkg_version = parse_version(pkg_list[-1].version)
else:
pkg_version = None
return pkg_version
def latest_pkg_version(pkg):
'''
:returns: the latest version of the specified conda package available
'''
r = Resolve(get_index())
try:
pkg_list = sorted(r.get_pkgs(MatchSpec(pkg)))
except:
pkg_list = None
if pkg_list:
pkg_version = parse_version(pkg_list[-1].version)
else:
pkg_version = None
return pkg_version
def check_install(packages, platform=None, channel_urls=(), prepend=True,
minimal_hint=False):
prefix = tempfile.mkdtemp('conda')
try:
specs = specs_from_args(packages)
index = get_index(channel_urls=channel_urls, prepend=prepend,
platform=platform, prefix=prefix)
actions = install_actions(prefix, index, specs, pinned=False,
minimal_hint=minimal_hint)
display_actions(actions, index)
return actions
finally:
rm_rf(prefix)
return None
def test_installable(channel='defaults'):
success = True
log = get_logger(__name__)
has_py = re.compile(r'py(\d)(\d)')
for platform in ['osx-64', 'linux-32', 'linux-64', 'win-32', 'win-64']:
log.info("######## Testing platform %s ########", platform)
channels = [channel]
index = get_index(channel_urls=channels, prepend=False, platform=platform)
for _, rec in iteritems(index):
# If we give channels at the command line, only look at
# packages from those channels (not defaults).
if channel != 'defaults' and rec.get('schannel', 'defaults') == 'defaults':
continue
name = rec['name']
if name in {'conda', 'conda-build'}:
# conda can only be installed in the root environment
continue
if name.endswith('@'):
# this is a 'virtual' feature record that conda adds to the index for the solver
# and should be ignored here
continue
# Don't fail just because the package is a different version of Python
# than the default. We should probably check depends rather than the
# build string.
build = rec['build']
match = has_py.search(build)
assert match if 'py' in build else True, build
if match:
additional_packages = ['python=%s.%s' % (match.group(1), match.group(2))]
else:
additional_packages = []
version = rec['version']
log.info('Testing %s=%s', name, version)
try:
install_steps = check_install([name + '=' + version] + additional_packages,
channel_urls=channels, prepend=False,
platform=platform)
success &= bool(install_steps)
except KeyboardInterrupt:
raise
# sys.exit raises an exception that doesn't subclass from Exception
except BaseException as e:
success = False
log.error("FAIL: %s %s on %s with %s (%s)", name, version,
platform, additional_packages, e)
return success
def _build_cdt_meta(recipes, dist_arch_slug):
print("getting conda-forge/label/main channel index...", flush=True)
channel_url = '/'.join(['conda-forge', 'label', 'main'])
dist_index = get_index([channel_url], prepend=False, use_cache=False)
channel_index = {
c.to_filename(): a
for c, a in dist_index.items() if a['subdir'] == 'noarch'
}
cdt_meta = {}
for recipe in tqdm.tqdm(recipes, desc='building CDT metadata', ncols=80):
if dist_arch_slug not in os.path.basename(recipe):
continue
node, attrs = _get_recipe_attrs(recipe, channel_index)
cdt_meta[node] = attrs
return cdt_meta
def get_dask_outputs(path,
packages=(),
filter_dirty=True,
git_rev='HEAD',
stop_rev=None,
steps=0,
visualize="",
test=False,
max_downstream=5,
**kwargs):
checkout_rev = stop_rev or git_rev
results = {}
conda_build_test = '--{}test'.format("" if test else "no-")
runs = ['test']
# not testing means build and test
if not test:
runs.insert(0, 'build')
output = []
indexes = {}
with checkout_git_rev(checkout_rev, path):
for run in runs:
platform_folder = '{}_platforms.d'.format(run)
# loop over platforms here because each platform may have different dependencies
# each platform will be submitted with a different label
for platform in load_platforms(os.path.join(path,
platform_folder)):
index_key = '-'.join(
[platform['platform'],
str(platform['arch'])])
if index_key not in indexes:
indexes[index_key] = Resolve(get_index(platform=index_key))
g = construct_graph(path,
platform=platform['platform'],
bits=platform['arch'],
folders=packages,
git_rev=git_rev,
stop_rev=stop_rev,
deps_type=run)
# note that the graph is changed in place here.
expand_run(g,
conda_resolve=indexes[index_key],
run=run,
steps=steps,
max_downstream=max_downstream)
# sort build order, and also filter so that we have solely dirty nodes in subgraph
subgraph, order = order_build(g, filter_dirty=filter_dirty)
for node in order:
for configuration in expand_build_matrix(
node, path, label=platform['worker_label']):
configuration['variables'][
'TEST_MODE'] = conda_build_test
commit_sha = stop_rev or git_rev
dependencies = [
results[_platform_package_key(run, n, platform)]
for n in subgraph[node].keys() if n in subgraph
]
key_name = _platform_package_key(run, node, platform)
# make the test run depend on the build run's completion
build_key_name = _platform_package_key(
"build", node, platform)
if build_key_name in results:
dependencies.append(results[build_key_name])
results[key_name] = delayed(_job, pure=True)(
configuration=configuration,
dependencies=dependencies,
commit_sha=commit_sha,
dask_key_name=key_name,
passthrough=visualize,
**kwargs)
output.append(results[key_name])
return output
