def prepare_patches(self, product, dir): patch_dir = os.path.join(self.config.patch_dir, product) if not os.path.isdir(patch_dir): return '' patch_files = glob.glob(os.path.join(patch_dir, '*.patch')) for patchfn in patch_files: shutil.copy2(patchfn, dir) # convert to meta.yaml string patchlist = [ os.path.basename(p) for p in patch_files ] patches = ' patches:' + create_yaml_list(patchlist) return patches
def prepare_patches(self, product, dir):
patch_dir = os.path.join(self.config.patch_dir, product)
if not os.path.isdir(patch_dir):
return ''
patch_files = glob.glob(os.path.join(patch_dir, '*.patch'))
for patchfn in patch_files:
shutil.copy2(patchfn, dir)
# convert to meta.yaml string
patchlist = [os.path.basename(p) for p in patch_files]
patches = ' patches:' + create_yaml_list(patchlist)
return patches
def gen_conda_package(self, product, sha, eups_version, giturl, eups_deps):
# What do we call this product in conda?
conda_name = self.config.conda_name_for(product)
# convert to conda version
version, build_string_prefix, buildnum, compliant = eups_to_conda_version(product, eups_version, giturl)
# warn if the version is not compliant
problem = "" if compliant else " [WARNING: version format incompatible with conda]"
# write out a progress message
self.report_progress(conda_name, "%s%s" % (version, problem))
#
# process dependencies
#
eups_deps = set(eups_deps)
eups_deps -= self.config.skip_products # skip unwanted dependencies
# Now start tracking runtime vs build depencendies separately
# FIXME: We should do this from the start, but EUPS still isn't tracking the two separately
bdeps, rdeps = [], []
for prod in eups_deps:
# Add the dependency
dep_conda_name = self.config.conda_name_for(prod)
bdeps.append(dep_conda_name)
rdeps.append(dep_conda_name)
# If the dependency is an internal product, add the internal product
# to the build dependencies as well. This works around the problem where
# e.g., lsst-afw depends on lsst-numpy-eups-config which depends on
# numpy ==1.9 in the build section, but numpy >=1.9 in the run. If lsst-afw
# didn't depend on numpy ==1.9 directly in its build section, then numpy
# would've been pulled in via lsst-numpy-eups-config's *run* section (and
# thus may be built against a newer numpy).
if prod in self.config.internal_products:
bdeps.append(self.config.internal_products[prod]['build'])
# If this is an internal product, also add the conda package as a dependency
try:
internals = self.config.internal_products[product]
except KeyError:
pass
else:
bdeps.append(internals['build'])
rdeps.append(internals['run'])
bplus, rplus = self.add_missing_deps(conda_name) # manually add any missing dependencies
bdeps += bplus; rdeps += rplus;
bdeps, rdeps = sorted(bdeps), sorted(rdeps) # sort, so the ordering is predicatble in meta.yaml
#
# Create the Conda packaging spec files
#
dir = os.path.join(self.config.output_dir, conda_name)
os.makedirs(dir)
# Copy any patches into the recipe dir
patches = self.prepare_patches(product, dir)
# build.sh (TBD: use exact eups versions, instead of -r .)
setups = []
SEP = 'setup '
setups = SEP + ('\n'+SEP).join(setups) if setups else ''
template_dir = self.config.template_dir
build_template = 'build.sh.template' if product not in self.config.internal_products else 'build-internal.sh.template'
fill_out_template(os.path.join(dir, 'build.sh'), os.path.join(template_dir, build_template),
setups = setups,
eups_version = eups_version,
eups_tags = ' '.join(self.config.global_eups_tags)
)
# pre-link.sh (to add the global tags)
fill_out_template(os.path.join(dir, 'pre-link.sh'), os.path.join(template_dir, 'pre-link.sh.template'),
product = product,
)
# meta.yaml
rdeps = [ self.conda_version_spec(p) if p in self.products else p for p in rdeps ]
bdeps = [ self.conda_version_spec(p) if p in self.products else p for p in bdeps ]
reqstr_r = create_yaml_list(rdeps)
reqstr_b = create_yaml_list(bdeps)
meta_yaml = os.path.join(dir, 'meta.yaml')
fill_out_template(meta_yaml, os.path.join(template_dir, 'meta.yaml.template'),
productNameLowercase = conda_name.lower(),
version = version,
gitrev = sha,
giturl = giturl,
build_req = reqstr_b,
run_req = reqstr_r,
patches = patches,
)
# The recipe is now (almost) complete.
# Find our build number. If this package already exists in the release DB,
# re-use the build number and mark it as '.done' so it doesn't get rebuilt.
# Otherwise, increment the max build number by one and use that.
buildnum, build_string, is_built = self.get_build_info(conda_name.lower(), version, dir, build_string_prefix)
# Fill in the build number and string
fill_out_template(meta_yaml, meta_yaml,
buildnum = buildnum,
build_string = build_string
)
# record we've seen this product
self.products[conda_name] = ProductInfo(conda_name, version, build_string, buildnum, product, eups_version, is_built, True)
def gen_conda_package(self, product, sha, eups_version, giturl, eups_deps, eups_tags):
# What do we call this product in conda?
conda_name = self.config.conda_name_for(product)
# convert to conda version
version, build_string_prefix, buildnum, compliant = eups_to_conda_version(product, eups_version, giturl)
# warn if the version is not compliant
problem = "" if compliant else " [WARNING: version format incompatible with conda]"
# write out a progress message
self.report_progress(conda_name, "%s%s" % (version, problem))
#
# process dependencies
#
eups_deps = set(eups_deps)
eups_deps -= self.config.skip_products # skip unwanted dependencies
# Now start tracking runtime vs build depencendies separately
# FIXME: We should do this from the start, but EUPS still isn't tracking the two separately
bdeps, rdeps = [], []
depends_on_internal = False
for prod in eups_deps: # transform to Anaconda product names
dep_conda_name = self.config.conda_name_for(prod)
internals = self.config.internal_products.get(prod, None)
depends_on_internal |= prod in self.config.internal_products
if internals is not None:
bdeps.append(internals.get('build', dep_conda_name))
rdeps.append(internals.get('run', dep_conda_name))
else:
bdeps.append(dep_conda_name)
rdeps.append(dep_conda_name)
# If we depend on one of the internal packages, make sure we depend on
# the lsst-product-configs package as well, as that's where the .table
# and .cfg files are.
if depends_on_internal:
# add lsst-product-configs to the build queue then fish out the
# version string
self.copy_additional_recipe('lsst-product-configs')
lpc_version = self.products['lsst-product-configs'].version
# inject a dep on the specific version
product_configs = 'lsst-product-configs ==%s' % lpc_version
bdeps.append(product_configs)
rdeps.append(product_configs)
bplus, rplus = self.add_missing_deps(conda_name) # manually add any missing dependencies
bdeps += bplus; rdeps += rplus;
bdeps, rdeps = sorted(bdeps), sorted(rdeps) # sort, so the ordering is predicatble in meta.yaml
#
# Create the Conda packaging spec files
#
dir = os.path.join(self.config.output_dir, conda_name)
os.makedirs(dir)
# Copy any patches into the recipe dir
patches = self.prepare_patches(product, dir)
# build.sh (TBD: use exact eups versions, instead of -r .)
setups = []
SEP = 'setup '
setups = SEP + ('\n'+SEP).join(setups) if setups else ''
template_dir = self.config.template_dir
fill_out_template(os.path.join(dir, 'build.sh'), os.path.join(template_dir, 'build.sh.template'),
setups = setups,
eups_version = eups_version,
eups_tags = ' '.join(eups_tags + self.config.global_eups_tags)
)
# pre-link.sh (to add the global tags)
fill_out_template(os.path.join(dir, 'pre-link.sh'), os.path.join(template_dir, 'pre-link.sh.template'),
product = product,
)
# meta.yaml
rdeps = [ self.conda_version_spec(p) if p in self.products else p for p in rdeps ]
bdeps = [ self.conda_version_spec(p) if p in self.products else p for p in bdeps ]
reqstr_r = create_yaml_list(rdeps)
reqstr_b = create_yaml_list(bdeps)
meta_yaml = os.path.join(dir, 'meta.yaml')
fill_out_template(meta_yaml, os.path.join(template_dir, 'meta.yaml.template'),
productNameLowercase = conda_name.lower(),
version = version,
gitrev = sha,
giturl = giturl,
build_req = reqstr_b,
run_req = reqstr_r,
patches = patches,
)
# The recipe is now (almost) complete.
# Find our build number. If this package already exists in the release DB,
# re-use the build number and mark it as '.done' so it doesn't get rebuilt.
# Otherwise, increment the max build number by one and use that.
buildnum, build_string, is_built = self.get_build_info(conda_name.lower(), version, dir, build_string_prefix)
# Fill in the build number and string
fill_out_template(meta_yaml, meta_yaml,
buildnum = buildnum,
build_string = build_string
)
# record we've seen this product
self.products[conda_name] = ProductInfo(conda_name, version, build_string, buildnum, product, eups_version, is_built, True)
def gen_conda_package(self, product, sha, eups_version, giturl, eups_deps,
eups_tags):
# What do we call this product in conda?
conda_name = self.config.conda_name_for(product)
# convert to conda version
version, build_string_prefix, buildnum, compliant = eups_to_conda_version(
product, eups_version, giturl)
# warn if the version is not compliant
problem = "" if compliant else " [WARNING: version format incompatible with conda]"
# write out a progress message
self.report_progress(conda_name, "%s%s" % (version, problem))
#
# process dependencies
#
eups_deps = set(eups_deps)
eups_deps -= self.config.skip_products # skip unwanted dependencies
# Now start tracking runtime vs build depencendies separately
# FIXME: We should do this from the start, but EUPS still isn't tracking the two separately
bdeps, rdeps = [], []
depends_on_internal = False
for prod in eups_deps: # transform to Anaconda product names
dep_conda_name = self.config.conda_name_for(prod)
internals = self.config.internal_products.get(prod, None)
depends_on_internal |= prod in self.config.internal_products
if internals is not None:
bdeps.append(internals.get('build', dep_conda_name))
rdeps.append(internals.get('run', dep_conda_name))
else:
bdeps.append(dep_conda_name)
rdeps.append(dep_conda_name)
# If we depend on one of the internal packages, make sure we depend on
# the lsst-product-configs package as well, as that's where the .table
# and .cfg files are.
if depends_on_internal:
# add lsst-product-configs to the build queue then fish out the
# version string
self.copy_additional_recipe('lsst-product-configs')
lpc_version = self.products['lsst-product-configs'].version
# inject a dep on the specific version
product_configs = 'lsst-product-configs ==%s' % lpc_version
bdeps.append(product_configs)
rdeps.append(product_configs)
bplus, rplus = self.add_missing_deps(
conda_name) # manually add any missing dependencies
bdeps += bplus
rdeps += rplus
bdeps, rdeps = sorted(bdeps), sorted(
rdeps) # sort, so the ordering is predicatble in meta.yaml
#
# Create the Conda packaging spec files
#
dir = os.path.join(self.config.output_dir, conda_name)
os.makedirs(dir)
# Copy any patches into the recipe dir
patches = self.prepare_patches(product, dir)
# build.sh (TBD: use exact eups versions, instead of -r .)
setups = []
SEP = 'setup '
setups = SEP + ('\n' + SEP).join(setups) if setups else ''
template_dir = self.config.template_dir
fill_out_template(os.path.join(dir, 'build.sh'),
os.path.join(template_dir, 'build.sh.template'),
setups=setups,
eups_version=eups_version,
eups_tags=' '.join(eups_tags +
self.config.global_eups_tags))
# pre-link.sh (to add the global tags)
fill_out_template(
os.path.join(dir, 'pre-link.sh'),
os.path.join(template_dir, 'pre-link.sh.template'),
product=product,
)
# meta.yaml
rdeps = [
self.conda_version_spec(p) if p in self.products else p
for p in rdeps
]
bdeps = [
self.conda_version_spec(p) if p in self.products else p
for p in bdeps
]
reqstr_r = create_yaml_list(rdeps)
reqstr_b = create_yaml_list(bdeps)
meta_yaml = os.path.join(dir, 'meta.yaml')
fill_out_template(
meta_yaml,
os.path.join(template_dir, 'meta.yaml.template'),
productNameLowercase=conda_name.lower(),
version=version,
gitrev=sha,
giturl=giturl,
build_req=reqstr_b,
run_req=reqstr_r,
patches=patches,
)
# The recipe is now (almost) complete.
# Find our build number. If this package already exists in the release DB,
# re-use the build number and mark it as '.done' so it doesn't get rebuilt.
# Otherwise, increment the max build number by one and use that.
buildnum, build_string, is_built = self.get_build_info(
conda_name.lower(), version, dir, build_string_prefix)
# Fill in the build number and string
fill_out_template(meta_yaml,
meta_yaml,
buildnum=buildnum,
build_string=build_string)
# record we've seen this product
self.products[conda_name] = ProductInfo(conda_name, version,
build_string, buildnum,
product, eups_version,
is_built, True)