def test_for_smoke(): """Do not validate the output of the functions, just make sure that calling them does not make depfinder blow up """ deps = list(depfinder.iterate_over_library('.')) assert deps is not None assert str(deps) is not None assert repr(deps) is not None # hit the simple api assert depfinder.simple_import_search('.') is not None
def execute_programmatically(skeletor_config_path, source_path, output_dir): """Execute conda skeletor Parameters ---------- skeletor_config_path : str Path to the skeletor config yaml. Typically named 'conda-skeletor.yml' source_path : str Path to the source code for which a full conda recipe is to be generated. output_dir : str Directory in which the generated conda recipe is to be placed. Will be made if it does not already exist """ if source_path == ".": source_path = os.path.abspath(source_path) # load the default configuration info skeletor_config = defaultdict(list) with open(DEFAULT_YAML_PATH) as f: skeletor_config.update(yaml.load(f.read())) logger.info("\nDefault configuration" "\n---------------------") logger.info(pprint.pformat(skeletor_config)) # make sure the skeletor_config_path is an absolute path skeletor_config_path = os.path.expanduser(skeletor_config_path) skeletor_config_path = os.path.abspath(skeletor_config_path) skeletor_config = defaultdict(list) with open(skeletor_config_path) as f: skeletor_config.update(yaml.load(f.read())) logger.info("\nskeletor-config file" "\n--------------------") logger.info("path = %s" % skeletor_config_path) logger.info(pprint.pformat(skeletor_config)) # find the dependencies for all modules in the source directory repo_deps = list(depfinder.iterate_over_library(source_path)) logger.info("\nFound %s python files" "\n---------------------" % len(repo_deps)) logger.info("source directory = %s" % source_path) paths = [p[1] for p in repo_deps] logger.info(pprint.pformat(paths)) # Compile the regexers listed in the conda-skeleton.yml test_regexers = skeletor_config.get("test_regex", []) ignore_path_regexers = skeletor_config.get("ignore_path_regex", []) include_path_regexers = skeletor_config.get("include_path_regex", []) extra_setup_regexers = skeletor_config.get("extra_setup_files_regex", []) logger.info("\nComparison strings" "\n------------------") logger.info("test_regexers = %s" % test_regexers) logger.info("ignore_path_regexers = %s" % ignore_path_regexers) logger.info("include_path_regexers = %s" % include_path_regexers) logger.info("extra_setup_regexers = %s" % extra_setup_regexers) ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers) setup_files, non_setup_files = split_deps(without_ignored, extra_setup_regexers) if include_path_regexers: included, without_included = split_deps(non_setup_files, include_path_regexers) else: included = non_setup_files without_included = None tests, without_tests = split_deps(included, test_regexers) logger.info("\nSplitting up the modules" "\n------------------------") logger.info("\nIgnored modules" "\n---------------") ignored_paths = [p[1] for p in ignored] logger.info(pprint.pformat(ignored_paths)) logger.info("\nModules to scrape for build dependencies" "\n----------------------------------------") setup_file_paths = [p[1] for p in setup_files] logger.info(pprint.pformat(setup_file_paths)) logger.info("\nModules to split into test/run time" "\n-----------------------------------") included_paths = [p[1] for p in included] logger.info(pprint.pformat(included_paths)) if without_included: logger.info("\nExcluded" "\n--------") excluded_paths = [p[1] for p in without_included] logger.info(pprint.pformat(excluded_paths)) logger.info("\nModules to scrape for testing dependencies" "\n------------------------------------------") test_paths = [p[1] for p in tests] logger.info(pprint.pformat(test_paths)) logger.info("\nModules to scrape for runtime dependencies" "\n------------------------------------------") without_test_paths = [p[1] for p in without_tests] logger.info(pprint.pformat(without_test_paths)) def find_lib_name(without_tests): """Helper function to find the library name from all non-test modules Parameters ---------- without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples Pass in the second return argument from `split_deps` Returns ------- str The name of the library (hopefully!) """ logger.info("\nEntering find_lib_name") non_test_paths = [path for name, path, catcher in without_tests] common_path = os.path.commonprefix(non_test_paths) logger.info("common_path = %s", common_path) lib_name = common_path.strip(os.sep).split(os.sep)[-1] is_single_module_package = False if lib_name.endswith(".py"): logger.info( "Found that lib_name ends with '.py'. This must be a" "single-module package. Stripping the .py" ) lib_name = lib_name[:-3] is_single_module_package = True logger.info("Leaving find_lib_name and returning %s", lib_name) ret = namedtuple("ReturnVals", ["library_name", "is_single_module_package"]) return ret(lib_name, is_single_module_package) ret = find_lib_name(without_tests) importable_lib_name, is_single_module_package = ret skeletor_config["blacklist_packages"].append(importable_lib_name) # find the runtime deps run_requires = get_run_requires(without_tests, blacklisted_packages=skeletor_config.get("blacklist_packages")) logger.info("\nExtracted dependencies" "\n----------------------") logger.info("Runtime") logger.info(pprint.pformat(run_requires)) # find the testing time deps test_requires = get_run_requires(tests, blacklisted_packages=skeletor_config.get("blacklist_packages")) logger.info("Test") logger.info(pprint.pformat(test_requires)) logger.info("\nSetup info" "\n----------") setup_info = None for mod_name, full_path, mod_deps in setup_files: if "setup.py" in full_path: setup_info = (mod_name, full_path, mod_deps) logger.info("setup path = %s" % full_path) logger.info("setup info = {}".format(setup_info)) if setup_info: # try: setup_info_dict = setup_parser.parse(setup_info[1]) logger.info("\nInformation scraped from setup.py" "\n---------------------------------") logger.info(pprint.pformat(setup_info_dict)) # except IndexError: # # Occurs when setup.py has a call like setup(**kwargs). Looking # # at you pims... # logger.info("No information gained from setup.py") # setup_info_dict = {} else: logger.info("No setup.py file found. Is this a python project?") setup_info_dict = {} # grab the code out of the setup.py file to find its deps so that I can # parse the code for imports setup_deps = None code = "" for mod_name, full_path, catcher in setup_files: with open(full_path, "r") as f: code += f.read() setup_deps = depfinder.get_imported_libs(code).describe() # add any setup deps that are included in the setup.py under # 'install_requires' install_requires = setup_info_dict.get("install_requires", []) # make sure that we are not iterating over a string... if isinstance(install_requires, str): install_requires = [install_requires] # add all extra deps in 'install_requires' to build requirements of the # conda recipe for dep in install_requires: setup_deps["required"].add(dep) logger.info("\nSetup dependencies" "\n------------------") logger.info(pprint.pformat(setup_deps)) # create the git repo path so that I can determine the version string from # the source code git_repo_path = os.path.dirname(setup_info[1]) template_info = construct_template_info( git_repo_path, setup_info_dict, setup_deps=setup_deps, user_config=skeletor_config ) # remove self-references try: run_requires.remove(template_info["packagename"]) except ValueError: pass try: test_requires.remove(template_info["packagename"]) except ValueError: pass build_requirements = template_info["build_requirements"] # remap deps for k, v in _PACKAGE_MAPPING.items(): if k in run_requires: logger.info("remapping %s -> %s in run_requires" % (k, v)) run_requires[run_requires.index(k)] = v if k in test_requires: logger.info("remapping %s -> %s in test_requires" % (k, v)) test_requires[test_requires.index(k)] = v if k in build_requirements: logger.info("remapping %s -> %s in build_requires" % (k, v)) build_requirements[build_requirements.index(k)] = v template_info["build_requirements"] = build_requirements extra_requirements = skeletor_config.get("requirements", {}) extra_test = extra_requirements.get("test", {}) for lib in extra_test: test_requires.append(lib) extra_run = extra_requirements.get("run", {}) for lib in extra_run: run_requires.append(lib) run_requires = sorted(run_requires) # conda build treats the test_requires sections as a superset of # run_requires. No need to explicitly have a long list of test_requirements test_requires = sorted(set(test_requires) - set(run_requires)) template_info["run_requirements"] = run_requires template_info["test_requires"] = test_requires template_info["lib_descriptions"] = {} # grab the package description from pypi if it exists for lib_name in set( template_info["run_requirements"] + template_info["test_requires"] + template_info["build_requirements"] ): try: description = pypi.description(lib_name) except ValueError: description = "A description for %s is not available on pypi" % lib_name template_info["lib_descriptions"][lib_name] = description if "test_imports" not in template_info: if is_single_module_package: test_imports = [importable_lib_name] else: test_imports = find_test_imports(importable_lib_name, without_tests) template_info["test_imports"] = test_imports logger.info("\nTemplate Information" "\n--------------------") logger.info(pprint.pformat(template_info)) # load and render the template logger.info("\nTemplate directory" "\n------------------" "\n%s" % (TEMPLATE_DIR)) # create the jinja environment jinja_env = Environment(loader=FileSystemLoader(TEMPLATE_DIR)) # template.render(**setup_info) meta_fname = os.path.join(output_dir, "meta.yaml") try: os.makedirs(output_dir) except FileExistsError: # the file, uh, already exists pass template = jinja_env.get_template("meta.tmpl") meta_yml = template.render(**template_info) with open(meta_fname, "w") as f: f.write(meta_yml) if skeletor_config["generate_build_script"]: build_bash_fname = os.path.join(output_dir, "build.sh") with open(build_bash_fname, "w") as f: f.write(DEFAULT_BUILD_BASH) logger.info("\nmeta.yaml" "\n---------") logger.info("\nwritten to: %s" % meta_fname) logger.info(meta_yml)
def execute_programmatically(skeletor_config_path, source_path, output_dir): with open(skeletor_config_path) as f: skeletor_config = yaml.load(f.read()) print('loaded config = %s' % skeletor_config) # find the dependencies for all modules in the source directory repo_deps = list(depfinder.iterate_over_library(source_path)) # Compile the regexers listed in the conda-skeleton.yml test_regexers = [ re.compile(reg) for reg in skeletor_config.get('test_regex', []) ] ignore_path_regexers = [ re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', []) ] include_path_regexers = [ re.compile(reg) for reg in skeletor_config.get('include_path_regex', []) ] extra_setup_regexers = [ re.compile(reg) for reg in skeletor_config.get('extra_setup_files_regex', []) ] print('extra_setup_regexers') print(extra_setup_regexers) ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers) setup_files, _ = split_deps(repo_deps, extra_setup_regexers) included, without_included = split_deps(without_ignored, include_path_regexers) tests, without_tests = split_deps(included, test_regexers) def find_lib_name(without_tests): """Helper function to find the library name from all non-test modules Parameters ---------- without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples Pass in the second return argument from `split_deps` Returns ------- str The name of the library (hopefully!) """ non_test_paths = [ full_path for mod_name, full_path, catcher in without_tests ] common_path = os.path.commonprefix(non_test_paths) return common_path.strip(os.sep).split(os.sep)[-1] importable_lib_name = find_lib_name(without_tests) skeletor_config['blacklist_packages'].append(importable_lib_name) # find the runtime deps runtime_deps = get_runtime_deps( without_tests, blacklisted_packages=skeletor_config.get('blacklist_packages')) print('runtime_deps') print(runtime_deps) # find the testing time deps test_requires = get_runtime_deps( tests, blacklisted_packages=skeletor_config.get('blacklist_packages')) print('test time deps') print(test_requires) setup_info = None for mod_name, full_path, mod_deps in without_ignored: if ('setup.py' in full_path): setup_info = (mod_name, full_path, mod_deps) if setup_info: try: setup_info_dict = setup_parser.parse(setup_info[1]) print('setup_info\n') print(setup_info_dict) except IndexError: # Occurs when setup.py has a call like setup(**kwargs). Looking at # you pims... setup_info_dict = {} else: print("No setup.py file found. Is this a python project?") raise # grab the code out of the setup.py file to find its deps so that I can # parse the code for imports setup_deps = None code = '' for mod_name, full_path, catcher in setup_files: with open(full_path, 'r') as f: code += f.read() setup_deps = depfinder.get_imported_libs(code).describe() print('setup_deps') print(setup_deps) # create the git repo path so that I can determine the version string from # the source code git_repo_path = os.path.dirname(setup_info[1]) template_info = construct_template_info(git_repo_path, setup_info_dict, setup_deps=setup_deps, user_config=skeletor_config) # remove self-references try: runtime_deps.remove(template_info['packagename']) except ValueError: pass try: test_requires.remove(template_info['packagename']) except ValueError: pass # remap deps for k, v in package_mapping.items(): if k in runtime_deps: runtime_deps.remove(k) runtime_deps.append(v) if k in test_requires: test_requires.remove(k) test_requires.append(v) template_info['run_requirements'] = runtime_deps template_info['test_requires'] = test_requires if 'test_imports' not in template_info: template_info['test_imports'] = find_test_imports( importable_lib_name, without_tests) print('template_info') print(template_info) # load and render the template template_dir = os.path.join(conda_skeletor_content, 'templates') print('template directory = %s' % template_dir) # create the jinja environment jinja_env = Environment(loader=FileSystemLoader(template_dir)) if skeletor_config.get('generate_test_suite', 'False'): if 'nose' in test_requires: template_info['test_commands'] = [ 'nosetests -v %s' % importable_lib_name ] elif 'pytest' in test_requires: template_info['test_commands'] = [ 'py.test -v %s' % importable_lib_name ] template = jinja_env.get_template('meta.tmpl') # template.render(**setup_info) meta_fname = os.path.join(output_dir, 'meta.yaml') with open(meta_fname, 'w') as f: f.write(template.render(**template_info)) build_bash_fname = os.path.join(output_dir, 'build.sh') with open(build_bash_fname, 'w') as f: f.write(DEFAULT_BUILD_BASH)
def execute_programmatically(skeletor_config_path, source_path, output_dir): with open(skeletor_config_path) as f: skeletor_config = yaml.load(f.read()) print('loaded config = %s' % skeletor_config) # find the dependencies for all modules in the source directory repo_deps = list(depfinder.iterate_over_library(source_path)) # Compile the regexers listed in the conda-skeleton.yml test_regexers = [re.compile(reg) for reg in skeletor_config.get('test_regex', [])] ignore_path_regexers = [re.compile(reg) for reg in skeletor_config.get('ignore_path_regex', [])] include_path_regexers = [re.compile(reg) for reg in skeletor_config.get('include_path_regex', [])] extra_setup_regexers = [re.compile(reg) for reg in skeletor_config.get('extra_setup_files_regex', [])] print('extra_setup_regexers') print(extra_setup_regexers) ignored, without_ignored = split_deps(repo_deps, ignore_path_regexers) setup_files, _ = split_deps(repo_deps, extra_setup_regexers) included, without_included = split_deps(without_ignored, include_path_regexers) tests, without_tests = split_deps(included, test_regexers) def find_lib_name(without_tests): """Helper function to find the library name from all non-test modules Parameters ---------- without_tests : iterable of (mod_name, full_path, ImportCatcher) tuples Pass in the second return argument from `split_deps` Returns ------- str The name of the library (hopefully!) """ non_test_paths = [full_path for mod_name, full_path, catcher in without_tests] common_path = os.path.commonprefix(non_test_paths) return common_path.strip(os.sep).split(os.sep)[-1] importable_lib_name = find_lib_name(without_tests) skeletor_config['blacklist_packages'].append(importable_lib_name) # find the runtime deps runtime_deps = get_runtime_deps( without_tests, blacklisted_packages=skeletor_config.get('blacklist_packages') ) print('runtime_deps') print(runtime_deps) # find the testing time deps test_requires = get_runtime_deps( tests, blacklisted_packages=skeletor_config.get('blacklist_packages') ) print('test time deps') print(test_requires) setup_info = None for mod_name, full_path, mod_deps in without_ignored: if('setup.py' in full_path): setup_info = (mod_name, full_path, mod_deps) if setup_info: try: setup_info_dict = setup_parser.parse(setup_info[1]) print('setup_info\n') print(setup_info_dict) except IndexError: # Occurs when setup.py has a call like setup(**kwargs). Looking at # you pims... setup_info_dict = {} else: print("No setup.py file found. Is this a python project?") raise # grab the code out of the setup.py file to find its deps so that I can # parse the code for imports setup_deps = None code = '' for mod_name, full_path, catcher in setup_files: with open(full_path, 'r') as f: code += f.read() setup_deps = depfinder.get_imported_libs(code).describe() print('setup_deps') print(setup_deps) # create the git repo path so that I can determine the version string from # the source code git_repo_path = os.path.dirname(setup_info[1]) template_info = construct_template_info(git_repo_path, setup_info_dict, setup_deps=setup_deps, user_config=skeletor_config) # remove self-references try: runtime_deps.remove(template_info['packagename']) except ValueError: pass try: test_requires.remove(template_info['packagename']) except ValueError: pass # remap deps for k, v in package_mapping.items(): if k in runtime_deps: runtime_deps.remove(k) runtime_deps.append(v) if k in test_requires: test_requires.remove(k) test_requires.append(v) template_info['run_requirements'] = runtime_deps template_info['test_requires'] = test_requires if 'test_imports' not in template_info: template_info['test_imports'] = find_test_imports(importable_lib_name, without_tests) print('template_info') print(template_info) # load and render the template template_dir = os.path.join(conda_skeletor_content, 'templates') print('template directory = %s' % template_dir) # create the jinja environment jinja_env = Environment(loader=FileSystemLoader(template_dir)) if skeletor_config.get('generate_test_suite', 'False'): if 'nose' in test_requires: template_info['test_commands'] = ['nosetests -v %s' % importable_lib_name] elif 'pytest' in test_requires: template_info['test_commands'] = ['py.test -v %s' % importable_lib_name] template = jinja_env.get_template('meta.tmpl') # template.render(**setup_info) meta_fname = os.path.join(output_dir, 'meta.yaml') with open(meta_fname, 'w') as f: f.write(template.render(**template_info)) build_bash_fname = os.path.join(output_dir, 'build.sh') with open(build_bash_fname, 'w') as f: f.write(DEFAULT_BUILD_BASH)