def test_relative_imports():
for rel in relative_imports:
test_object = Initter(rel)
imports = depfinder.get_imported_libs(test_object.code)
assert imports.describe() == test_object.targets
def test_imports():
for simple_import in complex_imports + simple_imports:
test_object = Initter(simple_import)
imports = depfinder.get_imported_libs(test_object.code)
assert imports.describe() == test_object.targets
def execute_programmatically(skeletor_config_path, source_path, output_dir):
"""Execute conda skeletor
Parameters
----------
skeletor_config_path : str
Path to the skeletor config yaml. Typically named 'conda-skeletor.yml'
source_path : str
Path to the source code for which a full conda recipe is to be
generated.
output_dir : str
Directory in which the generated conda recipe is to be placed. Will be
made if it does not already exist
"""
if source_path == ".":
source_path = os.path.abspath(source_path)
# load the default configuration info
skeletor_config = defaultdict(list)
with open(DEFAULT_YAML_PATH) as f:
skeletor_config.update(yaml.load(f.read()))
logger.info("\nDefault configuration" "\n---------------------")
logger.info(pprint.pformat(skeletor_config))
# make sure the skeletor_config_path is an absolute path
skeletor_config_path = os.path.expanduser(skeletor_config_path)
skeletor_config_path = os.path.abspath(skeletor_config_path)
skeletor_config = defaultdict(list)
with open(skeletor_config_path) as f:
skeletor_config.update(yaml.load(f.read()))
logger.info("\nskeletor-config file" "\n--------------------")
logger.info("path = %s" % skeletor_config_path)
logger.info(pprint.pformat(skeletor_config))
# find the dependencies for all modules in the source directory
repo_deps = list(depfinder.iterate_over_library(source_path))
logger.info("\nFound %s python files" "\n---------------------" % len(repo_deps))
logger.info("source directory = %s" % source_path)
paths = [p[1] for p in repo_deps]
logger.info(pprint.pformat(paths))
# Compile the regexers listed in the conda-skeleton.yml
test_regexers = skeletor_config.get("test_regex", [])
ignore_path_regexers = skeletor_config.get("ignore_path_regex", [])
include_path_regexers = skeletor_config.get("include_path_regex", [])
extra_setup_regexers = skeletor_config.get("extra_setup_files_regex", [])
logger.info("\nComparison strings" "\n------------------")
logger.info("test_regexers = %s" % test_regexers)
logger.info("ignore_path_regexers = %s" % ignore_path_regexers)
logger.info("include_path_regexers = %s" % include_path_regexers)
logger.info("extra_setup_regexers = %s" % extra_setup_regexers)
ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers)
setup_files, non_setup_files = split_deps(without_ignored, extra_setup_regexers)
if include_path_regexers:
included, without_included = split_deps(non_setup_files, include_path_regexers)
else:
included = non_setup_files
without_included = None
tests, without_tests = split_deps(included, test_regexers)
logger.info("\nSplitting up the modules" "\n------------------------")
logger.info("\nIgnored modules" "\n---------------")
ignored_paths = [p[1] for p in ignored]
logger.info(pprint.pformat(ignored_paths))
logger.info("\nModules to scrape for build dependencies" "\n----------------------------------------")
setup_file_paths = [p[1] for p in setup_files]
logger.info(pprint.pformat(setup_file_paths))
logger.info("\nModules to split into test/run time" "\n-----------------------------------")
included_paths = [p[1] for p in included]
logger.info(pprint.pformat(included_paths))
if without_included:
logger.info("\nExcluded" "\n--------")
excluded_paths = [p[1] for p in without_included]
logger.info(pprint.pformat(excluded_paths))
logger.info("\nModules to scrape for testing dependencies" "\n------------------------------------------")
test_paths = [p[1] for p in tests]
logger.info(pprint.pformat(test_paths))
logger.info("\nModules to scrape for runtime dependencies" "\n------------------------------------------")
without_test_paths = [p[1] for p in without_tests]
logger.info(pprint.pformat(without_test_paths))
def find_lib_name(without_tests):
"""Helper function to find the library name from all non-test modules
Parameters
----------
without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples
Pass in the second return argument from `split_deps`
Returns
-------
str
The name of the library (hopefully!)
"""
logger.info("\nEntering find_lib_name")
non_test_paths = [path for name, path, catcher in without_tests]
common_path = os.path.commonprefix(non_test_paths)
logger.info("common_path = %s", common_path)
lib_name = common_path.strip(os.sep).split(os.sep)[-1]
is_single_module_package = False
if lib_name.endswith(".py"):
logger.info(
"Found that lib_name ends with '.py'. This must be a" "single-module package. Stripping the .py"
)
lib_name = lib_name[:-3]
is_single_module_package = True
logger.info("Leaving find_lib_name and returning %s", lib_name)
ret = namedtuple("ReturnVals", ["library_name", "is_single_module_package"])
return ret(lib_name, is_single_module_package)
ret = find_lib_name(without_tests)
importable_lib_name, is_single_module_package = ret
skeletor_config["blacklist_packages"].append(importable_lib_name)
# find the runtime deps
run_requires = get_run_requires(without_tests, blacklisted_packages=skeletor_config.get("blacklist_packages"))
logger.info("\nExtracted dependencies" "\n----------------------")
logger.info("Runtime")
logger.info(pprint.pformat(run_requires))
# find the testing time deps
test_requires = get_run_requires(tests, blacklisted_packages=skeletor_config.get("blacklist_packages"))
logger.info("Test")
logger.info(pprint.pformat(test_requires))
logger.info("\nSetup info" "\n----------")
setup_info = None
for mod_name, full_path, mod_deps in setup_files:
if "setup.py" in full_path:
setup_info = (mod_name, full_path, mod_deps)
logger.info("setup path = %s" % full_path)
logger.info("setup info = {}".format(setup_info))
if setup_info:
# try:
setup_info_dict = setup_parser.parse(setup_info[1])
logger.info("\nInformation scraped from setup.py" "\n---------------------------------")
logger.info(pprint.pformat(setup_info_dict))
# except IndexError:
# # Occurs when setup.py has a call like setup(**kwargs). Looking
# # at you pims...
# logger.info("No information gained from setup.py")
# setup_info_dict = {}
else:
logger.info("No setup.py file found. Is this a python project?")
setup_info_dict = {}
# grab the code out of the setup.py file to find its deps so that I can
# parse the code for imports
setup_deps = None
code = ""
for mod_name, full_path, catcher in setup_files:
with open(full_path, "r") as f:
code += f.read()
setup_deps = depfinder.get_imported_libs(code).describe()
# add any setup deps that are included in the setup.py under
# 'install_requires'
install_requires = setup_info_dict.get("install_requires", [])
# make sure that we are not iterating over a string...
if isinstance(install_requires, str):
install_requires = [install_requires]
# add all extra deps in 'install_requires' to build requirements of the
# conda recipe
for dep in install_requires:
setup_deps["required"].add(dep)
logger.info("\nSetup dependencies" "\n------------------")
logger.info(pprint.pformat(setup_deps))
# create the git repo path so that I can determine the version string from
# the source code
git_repo_path = os.path.dirname(setup_info[1])
template_info = construct_template_info(
git_repo_path, setup_info_dict, setup_deps=setup_deps, user_config=skeletor_config
)
# remove self-references
try:
run_requires.remove(template_info["packagename"])
except ValueError:
pass
try:
test_requires.remove(template_info["packagename"])
except ValueError:
pass
build_requirements = template_info["build_requirements"]
# remap deps
for k, v in _PACKAGE_MAPPING.items():
if k in run_requires:
logger.info("remapping %s -> %s in run_requires" % (k, v))
run_requires[run_requires.index(k)] = v
if k in test_requires:
logger.info("remapping %s -> %s in test_requires" % (k, v))
test_requires[test_requires.index(k)] = v
if k in build_requirements:
logger.info("remapping %s -> %s in build_requires" % (k, v))
build_requirements[build_requirements.index(k)] = v
template_info["build_requirements"] = build_requirements
extra_requirements = skeletor_config.get("requirements", {})
extra_test = extra_requirements.get("test", {})
for lib in extra_test:
test_requires.append(lib)
extra_run = extra_requirements.get("run", {})
for lib in extra_run:
run_requires.append(lib)
run_requires = sorted(run_requires)
# conda build treats the test_requires sections as a superset of
# run_requires. No need to explicitly have a long list of test_requirements
test_requires = sorted(set(test_requires) - set(run_requires))
template_info["run_requirements"] = run_requires
template_info["test_requires"] = test_requires
template_info["lib_descriptions"] = {}
# grab the package description from pypi if it exists
for lib_name in set(
template_info["run_requirements"] + template_info["test_requires"] + template_info["build_requirements"]
):
try:
description = pypi.description(lib_name)
except ValueError:
description = "A description for %s is not available on pypi" % lib_name
template_info["lib_descriptions"][lib_name] = description
if "test_imports" not in template_info:
if is_single_module_package:
test_imports = [importable_lib_name]
else:
test_imports = find_test_imports(importable_lib_name, without_tests)
template_info["test_imports"] = test_imports
logger.info("\nTemplate Information" "\n--------------------")
logger.info(pprint.pformat(template_info))
# load and render the template
logger.info("\nTemplate directory" "\n------------------" "\n%s" % (TEMPLATE_DIR))
# create the jinja environment
jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR))
# template.render(**setup_info)
meta_fname = os.path.join(output_dir, "meta.yaml")
try:
os.makedirs(output_dir)
except FileExistsError:
# the file, uh, already exists
pass
template = jinja_env.get_template("meta.tmpl")
meta_yml = template.render(**template_info)
with open(meta_fname, "w") as f:
f.write(meta_yml)
if skeletor_config["generate_build_script"]:
build_bash_fname = os.path.join(output_dir, "build.sh")
with open(build_bash_fname, "w") as f:
f.write(DEFAULT_BUILD_BASH)
logger.info("\nmeta.yaml" "\n---------")
logger.info("\nwritten to: %s" % meta_fname)
logger.info(meta_yml)
def test_relative_imports():
for rel in relative_imports:
test_object = Initter(rel)
imports = depfinder.get_imported_libs(test_object.code)
assert imports.describe() == test_object.targets
def test_imports():
for simple_import in complex_imports + simple_imports:
test_object = Initter(simple_import)
imports = depfinder.get_imported_libs(test_object.code)
assert imports.describe() == test_object.targets
def execute_programmatically(skeletor_config_path, source_path, output_dir):
with open(skeletor_config_path) as f:
skeletor_config = yaml.load(f.read())
print('loaded config = %s' % skeletor_config)
# find the dependencies for all modules in the source directory
repo_deps = list(depfinder.iterate_over_library(source_path))
# Compile the regexers listed in the conda-skeleton.yml
test_regexers = [
re.compile(reg) for reg in skeletor_config.get('test_regex', [])
]
ignore_path_regexers = [
re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', [])
]
include_path_regexers = [
re.compile(reg)
for reg in skeletor_config.get('include_path_regex', [])
]
extra_setup_regexers = [
re.compile(reg)
for reg in skeletor_config.get('extra_setup_files_regex', [])
]
print('extra_setup_regexers')
print(extra_setup_regexers)
ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers)
setup_files, _ = split_deps(repo_deps, extra_setup_regexers)
included, without_included = split_deps(without_ignored,
include_path_regexers)
tests, without_tests = split_deps(included, test_regexers)
def find_lib_name(without_tests):
"""Helper function to find the library name from all non-test modules
Parameters
----------
without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples
Pass in the second return argument from `split_deps`
Returns
-------
str
The name of the library (hopefully!)
"""
non_test_paths = [
full_path for mod_name, full_path, catcher in without_tests
]
common_path = os.path.commonprefix(non_test_paths)
return common_path.strip(os.sep).split(os.sep)[-1]
importable_lib_name = find_lib_name(without_tests)
skeletor_config['blacklist_packages'].append(importable_lib_name)
# find the runtime deps
runtime_deps = get_runtime_deps(
without_tests,
blacklisted_packages=skeletor_config.get('blacklist_packages'))
print('runtime_deps')
print(runtime_deps)
# find the testing time deps
test_requires = get_runtime_deps(
tests, blacklisted_packages=skeletor_config.get('blacklist_packages'))
print('test time deps')
print(test_requires)
setup_info = None
for mod_name, full_path, mod_deps in without_ignored:
if ('setup.py' in full_path):
setup_info = (mod_name, full_path, mod_deps)
if setup_info:
try:
setup_info_dict = setup_parser.parse(setup_info[1])
print('setup_info\n')
print(setup_info_dict)
except IndexError:
# Occurs when setup.py has a call like setup(**kwargs). Looking at
# you pims...
setup_info_dict = {}
else:
print("No setup.py file found. Is this a python project?")
raise
# grab the code out of the setup.py file to find its deps so that I can
# parse the code for imports
setup_deps = None
code = ''
for mod_name, full_path, catcher in setup_files:
with open(full_path, 'r') as f:
code += f.read()
setup_deps = depfinder.get_imported_libs(code).describe()
print('setup_deps')
print(setup_deps)
# create the git repo path so that I can determine the version string from
# the source code
git_repo_path = os.path.dirname(setup_info[1])
template_info = construct_template_info(git_repo_path,
setup_info_dict,
setup_deps=setup_deps,
user_config=skeletor_config)
# remove self-references
try:
runtime_deps.remove(template_info['packagename'])
except ValueError:
pass
try:
test_requires.remove(template_info['packagename'])
except ValueError:
pass
# remap deps
for k, v in package_mapping.items():
if k in runtime_deps:
runtime_deps.remove(k)
runtime_deps.append(v)
if k in test_requires:
test_requires.remove(k)
test_requires.append(v)
template_info['run_requirements'] = runtime_deps
template_info['test_requires'] = test_requires
if 'test_imports' not in template_info:
template_info['test_imports'] = find_test_imports(
importable_lib_name, without_tests)
print('template_info')
print(template_info)
# load and render the template
template_dir = os.path.join(conda_skeletor_content, 'templates')
print('template directory = %s' % template_dir)
# create the jinja environment
jinja_env = Environment(loader=FileSystemLoader(template_dir))
if skeletor_config.get('generate_test_suite', 'False'):
if 'nose' in test_requires:
template_info['test_commands'] = [
'nosetests -v %s' % importable_lib_name
]
elif 'pytest' in test_requires:
template_info['test_commands'] = [
'py.test -v %s' % importable_lib_name
]
template = jinja_env.get_template('meta.tmpl')
# template.render(**setup_info)
meta_fname = os.path.join(output_dir, 'meta.yaml')
with open(meta_fname, 'w') as f:
f.write(template.render(**template_info))
build_bash_fname = os.path.join(output_dir, 'build.sh')
with open(build_bash_fname, 'w') as f:
f.write(DEFAULT_BUILD_BASH)
def execute_programmatically(skeletor_config_path, source_path, output_dir):
with open(skeletor_config_path) as f:
skeletor_config = yaml.load(f.read())
print('loaded config = %s' % skeletor_config)
# find the dependencies for all modules in the source directory
repo_deps = list(depfinder.iterate_over_library(source_path))
# Compile the regexers listed in the conda-skeleton.yml
test_regexers = [re.compile(reg) for reg in skeletor_config.get('test_regex', [])]
ignore_path_regexers = [re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', [])]
include_path_regexers = [re.compile(reg) for reg in skeletor_config.get('include_path_regex', [])]
extra_setup_regexers = [re.compile(reg) for reg in skeletor_config.get('extra_setup_files_regex', [])]
print('extra_setup_regexers')
print(extra_setup_regexers)
ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers)
setup_files, _ = split_deps(repo_deps, extra_setup_regexers)
included, without_included = split_deps(without_ignored, include_path_regexers)
tests, without_tests = split_deps(included, test_regexers)
def find_lib_name(without_tests):
"""Helper function to find the library name from all non-test modules
Parameters
----------
without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples
Pass in the second return argument from `split_deps`
Returns
-------
str
The name of the library (hopefully!)
"""
non_test_paths = [full_path for mod_name, full_path, catcher in without_tests]
common_path = os.path.commonprefix(non_test_paths)
return common_path.strip(os.sep).split(os.sep)[-1]
importable_lib_name = find_lib_name(without_tests)
skeletor_config['blacklist_packages'].append(importable_lib_name)
# find the runtime deps
runtime_deps = get_runtime_deps(
without_tests,
blacklisted_packages=skeletor_config.get('blacklist_packages')
)
print('runtime_deps')
print(runtime_deps)
# find the testing time deps
test_requires = get_runtime_deps(
tests,
blacklisted_packages=skeletor_config.get('blacklist_packages')
)
print('test time deps')
print(test_requires)
setup_info = None
for mod_name, full_path, mod_deps in without_ignored:
if('setup.py' in full_path):
setup_info = (mod_name, full_path, mod_deps)
if setup_info:
try:
setup_info_dict = setup_parser.parse(setup_info[1])
print('setup_info\n')
print(setup_info_dict)
except IndexError:
# Occurs when setup.py has a call like setup(**kwargs). Looking at
# you pims...
setup_info_dict = {}
else:
print("No setup.py file found. Is this a python project?")
raise
# grab the code out of the setup.py file to find its deps so that I can
# parse the code for imports
setup_deps = None
code = ''
for mod_name, full_path, catcher in setup_files:
with open(full_path, 'r') as f:
code += f.read()
setup_deps = depfinder.get_imported_libs(code).describe()
print('setup_deps')
print(setup_deps)
# create the git repo path so that I can determine the version string from
# the source code
git_repo_path = os.path.dirname(setup_info[1])
template_info = construct_template_info(git_repo_path, setup_info_dict,
setup_deps=setup_deps,
user_config=skeletor_config)
# remove self-references
try:
runtime_deps.remove(template_info['packagename'])
except ValueError:
pass
try:
test_requires.remove(template_info['packagename'])
except ValueError:
pass
# remap deps
for k, v in package_mapping.items():
if k in runtime_deps:
runtime_deps.remove(k)
runtime_deps.append(v)
if k in test_requires:
test_requires.remove(k)
test_requires.append(v)
template_info['run_requirements'] = runtime_deps
template_info['test_requires'] = test_requires
if 'test_imports' not in template_info:
template_info['test_imports'] = find_test_imports(importable_lib_name,
without_tests)
print('template_info')
print(template_info)
# load and render the template
template_dir = os.path.join(conda_skeletor_content, 'templates')
print('template directory = %s' % template_dir)
# create the jinja environment
jinja_env = Environment(loader=FileSystemLoader(template_dir))
if skeletor_config.get('generate_test_suite', 'False'):
if 'nose' in test_requires:
template_info['test_commands'] = ['nosetests -v %s' % importable_lib_name]
elif 'pytest' in test_requires:
template_info['test_commands'] = ['py.test -v %s' % importable_lib_name]
template = jinja_env.get_template('meta.tmpl')
# template.render(**setup_info)
meta_fname = os.path.join(output_dir, 'meta.yaml')
with open(meta_fname, 'w') as f:
f.write(template.render(**template_info))
build_bash_fname = os.path.join(output_dir, 'build.sh')
with open(build_bash_fname, 'w') as f:
f.write(DEFAULT_BUILD_BASH)
