def merge(self, other): """ Merge the environment markers. Assumes package and qualified_package are the same with other. :param other: other requirement :type other: Requirement """ new_markers = {} for marker in self.markers: key = "{name} {operator}".format(name=marker.name, operator=marker.operator) if key not in new_markers: new_markers[key] = [] new_markers[key].extend(marker.value) for marker in other.markers: key = "{name} {operator}".format(name=marker.name, operator=marker.operator) if key not in new_markers: new_markers[key] = [] new_markers[key].extend(marker.value) self.markers = [] for key in new_markers: name, operator = key.split(' ') value = sorted(list(set(new_markers[key]))) debug("merge => {name} {operator} {value}".format(name=name, operator=operator, value=value)) self.markers.append(EnvironmentMarker("{name} {operator} {value}".format(name=name, operator=operator, value=' '.join(value))))
def get_project_version(project_package=None): r""" Get the version from __init__.py with a line: /^__version__\s*=\s*(\S+)/ If it doesn't exist try to load it from the VERSION.txt file. If still no joy, then return '0.0.0' :param project_package: the root package :type project_package: str :returns: the version string :rtype: str """ # trying __init__.py first try: file_name = _file_spec('__init__.py', project_package) debug("version_file => %s" % file_name) # noinspection PyBroadException try: # python3 with open(file_name, 'r', encoding='utf-8') as inFile: for line in inFile.readlines(): match = re.match(VERSION_REGEX, line) if match: return match.group(1) except: # python2 with open(file_name, 'r') as inFile: for line in inFile.readlines(): match = re.match(VERSION_REGEX, line) if match: return match.group(1) except IOError: pass # no joy, so try getting the version from a VERSION.txt file. try: file_name = _file_spec('VERSION.txt', project_package) info("version_file => %s" % file_name) # noinspection PyBroadException try: # python3 with open(file_name, 'r', encoding='utf-8') as in_file: return in_file.read().strip() except: # python2 with open(file_name, 'r') as in_file: return in_file.read().strip() except IOError: try: file_name = _file_spec('VERSION.txt', Project.herringfile_dir) info("version_file => %s" % file_name) with open(file_name) as in_file: return in_file.read().strip() except IOError: pass # no joy again, so set to initial version and try again set_project_version('0.0.1', project_package) return get_project_version(project_package)
def infos(self, exists=True): """ get VenvInfo instances generator. Usage ----- :: for venv_info in venvs.infos(): pass :param exists: the virtualenv must exist to be included in the generator :type exists: bool """ if not self.in_virtualenv and not self.defined: raise NoAvailableVirtualenv() value = self._ver_attr if not is_sequence(value): value = [value] try: for ver in value: debug("ver: {ver}".format(ver=ver)) if re.match(r'\d+', ver): venv_info = VenvInfo(ver) else: venv_info = VenvInfo(venv=ver) if exists: if venv_info.exists(): yield venv_info else: yield venv_info except Exception as ex: error(str(ex)) error(traceback.format_exc())
def _create_module_diagrams(path): """ create module UML diagrams :param path: the module path :type path: str """ info("_create_module_diagrams") if not executables_available(['pyreverse']): warning('pyreverse not available') return with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter: for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']: debug("module_path: {path}".format(path=module_path)) init_filename = os.path.join(module_path, '__init__.py') if os.path.exists(init_filename): info(init_filename) name = os.path.basename(module_path).split(".")[0] output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path), verbose=True, ignore_errors=True) outputter.write(output) errors = [line for line in output.splitlines() if not line.startswith('parsing')] if errors: info(errors)
def required_files(self): """ Add required packages (specified in module docstrings) to the appropriate requirements text file(s). """ debug("requiredFiles") needed_dict = Requirements(self._project).find_missing_requirements() for filename in needed_dict.keys(): needed = needed_dict[filename] debug("needed: %s" % repr(needed)) try: requirements_filename = os.path.join(self._project.herringfile_dir, filename) if not os.path.isfile(requirements_filename): with open(requirements_filename, 'w') as req_file: req_file.write('-e .\n\n') with open(requirements_filename, 'a') as req_file: for need in sorted(unique_list(list(needed))): out_line = need.qualified(qualifiers=True) if out_line: req_file.write(out_line + "\n") for need in sorted(unique_list(list(needed))): out_line = need.qualified(qualifiers=False) if out_line: req_file.write(out_line + "\n") except IOError as ex: warning("Can not add the following to the {filename} file: {needed}\n{err}".format( filename=filename, needed=repr(needed), err=str(ex)))
def find_missing_requirements(self): """ Find the required packages that are not in the requirements.txt file. :return: set of missing packages. :rtype: set[Requirement] """ requirements = self._reduce_by_version(self._get_requirements_dict_from_py_files()) debug('requirements:') debug(pformat(requirements)) needed = [] needed.extend(sorted(compress_list(unique_list(requirements)))) filename = 'requirements.txt' if not os.path.exists(filename): debug("Missing: " + filename) diff = sorted(set(needed)) else: with open(filename) as in_file: existing_requirements = [] for line in [line.strip() for line in in_file.readlines()]: if line and not line.startswith('#'): existing_requirements.append(Requirement(line)) existing = sorted(compress_list(unique_list(existing_requirements))) difference = [req for req in needed if req not in existing] diff = sorted(set([req for req in difference if not req.markers or Requirement(req.package) not in needed])) debug("find_missing_requirements.needed: {pkgs}".format(pkgs=pformat(needed))) debug("find_missing_requirements.diff: {pkgs}".format(pkgs=pformat(diff))) return diff
def _find_item_groups(self, lines): item_indexes = [i for i, item in enumerate(lines) if re.match(self.ITEM_REGEX, item)] debug("item_indexes: %s" % repr(item_indexes)) item_groups = [] for k, g in groupby(enumerate(item_indexes), lambda x: x[0] - x[1]): # lambda (i, x): i - x): item_groups.append(list(map(itemgetter(1), g))) return item_groups
def _hack_package(self, line): match = re.match(r"(.+)\s+package\s*", line) if match: debug("matched package") line = "|package| " + match.group(1) self.line_length = len(line) line += "\n" return line
def _hack_package(self, line): match = re.match(r'(.+)\s+package\s*', line) if match: debug("matched package") line = '|package| ' + match.group(1) self.line_length = len(line) line += '\n' return line
def check_requirements(): """ Checks that herringfile and herringlib/* required packages are in requirements.txt file """ debug("check_requirements") needed = Requirements(Project).find_missing_requirements() if needed: info("Please add the following to your %s file:\n" % 'requirements.txt') info("\n".join(str(needed))) else: info("Your %s includes all known herringlib task requirements" % 'requirements.txt')
def _get_herringlib_py_files(self): """find all the .py files in the herringlib directory""" lib_files = [] debug("HerringFile.herringlib_paths: %s" % repr(HerringFile.herringlib_paths)) for herringlib_path in [os.path.join(path_, 'herringlib') for path_ in HerringFile.herringlib_paths]: for dir_path, dir_names, files in os.walk(herringlib_path): for f in fnmatch.filter(files, '*.py'): lib_files.append(os.path.join(dir_path, f)) return lib_files
def _hack_module(self, line): match = re.match(r"(.+)\s+module\s*", line) if match: debug("matched module") self.package = False self.class_name = match.group(1).split(".")[-1] line = "|module| " + match.group(1) self.line_length = len(line) line += "\n" return line
def _hack_module(self, line): match = re.match(r'(.+)\s+module\s*', line) if match: debug("matched module") self.package = False self.class_name = match.group(1).split('.')[-1] line = '|module| ' + match.group(1) self.line_length = len(line) line += '\n' return line
def _hack_underline(self, line): if re.match(r"[=\-\.][=\-\.][=\-\.]+", line): debug("matched [=\-\.][=\-\.][=\-\.]+") if self.line_length > 0: line = "%s\n" % (line[0] * self.line_length) if self.package: line += _package_line(self.module_name) if self.class_name: line += _class_line(self.module_name, self.class_name) return line
def __init__(self, *attr_names): self._ver_attr = None self._raise_when_in_venv = False debug(repr(attr_names)) for name in attr_names: debug(name) self._ver_attr = getattr(Project, name, None) if self._ver_attr is not None: info("_ver_attr: %s" % repr(self._ver_attr)) break
def _reduce_by_version(self, requirements): requirement_dict = {} debug("requirements:\n" + pformat(requirements)) for requirement in requirements: if requirement.package not in requirement_dict: requirement_dict[requirement.package] = requirement else: debug("merge {src} into {dest}".format(src=str(requirement), dest=str(requirement_dict[requirement.package]))) requirement_dict[requirement.package].merge(requirement) return requirement_dict.values()
def clean(): """ remove build artifacts """ recursively_remove(Project.herringfile_dir, '*.pyc') recursively_remove(Project.herringfile_dir, '*~') debug(repr(Project.__dict__)) dirs = [Project.dist_dir, Project.egg_dir] # print("dirs => %s" % repr(dirs)) for dir_name in dirs: if os.path.exists(dir_name): shutil.rmtree(dir_name)
def _hack_mod(self, line): match = re.match(r":mod:`(.+)`(.*)", line) if match: debug("matched :mod:") key = match.group(1) if key in self.name_dict: value = self.name_dict[key] line = "".join(":mod:`%s`%s\n" % (value, match.group(2))) self.line_length = len(line) self.package = re.search(r":mod:.+Package", line) self.class_name = key return line
def __init__(self, marker): self.marker = marker self.name = None self.operator = None self.value = None if self.marker is not None: match = re.match(r'''(\S+)\s*((?:[!=<>]+)|(?:not in)|(?<!not )in)\s*[\"\']?([\d\.\s]+)[\"\']?''', self.marker) if match: self.name = match.group(1) self.operator = match.group(2) self.value = match.group(3).split(' ') debug("EnvironmentMarker:\n {marker}\n raw='{raw}'".format(marker=self, raw=match.group(3)))
def _get_module_docstring(self, file_path): """ Get module-level docstring of Python module at filepath, e.g. 'path/to/file.py'. :param file_path: The filepath to a module file. :type: str :returns: the module docstring :rtype: str """ debug("_get_module_docstring('{file}')".format(file=file_path)) tree = ast.parse(''.join(open(file_path))) # noinspection PyArgumentEqualDefault docstring = (ast.get_docstring(tree, clean=True) or '').strip() debug("docstring: %s" % docstring) return docstring
def recursively_remove(path, pattern): """ recursively remove files that match a given pattern :param path: The directory to start removing files from. :type path: str :param pattern: The file glob pattern to match for removal. :type pattern: str """ files = [os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, pattern)] for file_ in files: debug("removing: %s" % file_) os.remove(file_)
def __init__(self, line): self.line = line.strip() # strip double quotes off of line if self.line.startswith('"') and self.line.endswith('"'): self.line = self.line[1:-1] debug("Requirement: {line}".format(line=self.line)) match = re.match(r'(.*?#egg=[^\s;]+)', self.line) if match: self.package = match.group(1).strip().strip(';') else: self.package = re.split(r'[^a-zA-Z0-9_\-]', self.line)[0].strip().strip(';') self.qualified_package = re.split(r';', self.line)[0].strip() try: self.markers = [EnvironmentMarker(re.split(r';', self.line)[1].strip().replace('"', "'"))] except IndexError: self.markers = []
def _parse_docstring(self, doc_string): """ Extract the required packages from the docstring. This makes the following assumptions: 1) there is a line in the docstring that contains "requirements.txt". 2) after that line, ignoring blank lines, there are bullet list items starting with a '*' 3) these bullet list items are the names of the required third party packages followed by any optional conditions :param doc_string: a module docstring :type: str :return: requirements by requirement file :rtype: list(Requirement) """ requirements = [] debug("_parse_docstring") if doc_string is None or not doc_string: return requirements raw_lines = list(filter(str.strip, doc_string.splitlines())) lines = self._variable_substitution(raw_lines) # lines should now contain: # ['blah', 'blah', '...requirements.txt...','* pkg 1', '* pkg 2', 'blah'] debug(lines) requirement_indexes = [i for i, item in enumerate(lines) if re.search(self.REQUIREMENT_REGEX, item)] debug("requirement_indexes: %s" % repr(requirement_indexes)) item_groups = self._find_item_groups(lines) # print("item_groups: %s" % repr(item_groups)) # example using doc_string: # # item_indexes: [4, 5, 6, 7, 8, 10, 13, 14] # item_groups: [[4, 5, 6, 7, 8], [10], [13, 14]] # # we want: # requirements = [ # [lines[4], lines[5], lines[6], lines[7], lines[8]], # [lines[10]] # [lines[13], lines[14]], # ] for index in requirement_indexes: for item_group in item_groups: if item_group[0] == index + 1: # yes we have items for the requirement file requirements.extend([Requirement(re.match(self.ITEM_REGEX, lines[item_index]).group(1)) for item_index in item_group]) debug("requirements:\n%s" % pformat(requirements)) return requirements
def supported_python(self): """ Is this requirement intended for the currently running version of python? If this requirement has a marker (ex: 'foo; python_version == "2.7"') check if the current python qualifies. If this requirement does not have a marker then return True. """ for marker in [m for m in self.markers if m.name == 'python_version']: if marker.operator is not None and marker.value is not None: # noinspection PyUnresolvedReferences import sys code = "sys.version_info {operator} {version}".format(operator=marker.operator, version=str(marker.value)) result = eval(code) debug("{code} returned: {result}".format(code=code, result=str(result))) return result return True
def _package_line(self, module_name): """create the package figure lines for the given module""" info("_package_line(%s)" % module_name) line = '' package_image = "uml/packages_{name}.svg".format(name=module_name.split('.')[-1]) classes_image = "uml/classes_{name}.svg".format(name=module_name.split('.')[-1]) image_path = os.path.join(Project.docs_dir, '_src', package_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format( image=package_image, name=module_name) line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format( image=classes_image, name=module_name) else: debug("%s does not exist!" % image_path) return line
def _class_line(self, module_name, class_name): """create the class figure lines for the given module and class""" info("_class_line(%s, %s)" % (module_name, class_name)) line = '' classes_images = [ "uml/classes_{module}.{name}.png".format(module=module_name, name=class_name), "uml/classes_{module}.png".format(module=module_name), ] for classes_image in classes_images: image_path = os.path.join(Project.docs_dir, '_src', classes_image) if os.path.exists(image_path): info("adding figure %s" % image_path) line += "\n.. figure:: {image}\n\n {name} Class\n\n".format( image=classes_image, name=class_name) break else: debug("%s does not exist!" % image_path) return line
def mkvirtualenv(self): """Make a virtualenv""" new_env = Project.env_without_virtualenvwrapper() debug("os.environ['PATH']: \"{path}\"".format(path=os.environ['PATH'])) debug("new_env: {env}".format(env=pformat(new_env))) with LocalShell() as local: venv_script = Project.virtualenvwrapper_script # noinspection PyArgumentEqualDefault venvs = local.run('/bin/bash -c "source {venv_script} ;' 'lsvirtualenv -b"'.format(venv_script=venv_script), verbose=False, env=new_env).strip().split("\n") if self.venv not in venvs: python_path = local.system('which {python}'.format(python=self.python)).strip() local.run('/bin/bash -c "source {venv_script} ; ' 'mkvirtualenv -p {python} {venv}"'.format(venv_script=venv_script, python=python_path, venv=self.venv), verbose=True, env=new_env)
def _create_class_diagrams(path): """ Create class UML diagram :param path: path to the module file. :type path: str """ info("_create_class_diagrams") if not executables_available(["pynsource"]): return files = [ os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, "*.py") ] debug("files: {files}".format(files=repr(files))) for src_file in files: debug(src_file) name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".") output = "classes_{name}".format(name=name) debug(output) if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)): run_python( "pynsource -y {output} {source}".format(output=output, source=src_file), verbose=False, ignore_errors=True, )
def _get_requirements_dict_from_py_files(self, lib_files=None): """ Scan the herringlib py file docstrings extracting the 3rd party requirements. :param lib_files: list of filenames to scan for requirement comments. Set to None to scan all py files in the herringlib directory and the herringfile. :type lib_files: list[str] :return: requirements dict where key is the requirements file name (ex: "requirements.txt") and the value is a list of package names (ex: ['argparse', 'wheel']). :rtype: dict[str,list[Requirement]] """ if lib_files is None: lib_files = self._get_herringlib_py_files() lib_files.append(os.path.join(self._project.herringfile_dir, 'herringfile')) debug("files: %s" % repr(lib_files)) requirements = {} for file_ in lib_files: debug('file: %s' % file_) required_files_dict = self._parse_docstring(self._get_module_docstring(file_)) debug('required_files: %s' % pformat(required_files_dict)) for requirement_filename in required_files_dict.keys(): if requirement_filename not in requirements.keys(): requirements[requirement_filename] = [] for req in required_files_dict[requirement_filename]: if req not in requirements[requirement_filename]: requirements[requirement_filename].append(req) return requirements
def _create_class_diagrams(path): """ Create class UML diagram :param path: path to the module file. :type path: str """ info("_create_class_diagrams") if not executables_available(['pynsource']): warning('pynsource not available') return files = [os.path.join(dir_path, f) for dir_path, dir_names, files in os.walk(path) for f in fnmatch.filter(files, '*.py')] debug("files: {files}".format(files=repr(files))) with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter: for src_file in files: debug(src_file) name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.') output = "classes_{name}".format(name=name) debug(output) if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)): output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file), verbose=False, ignore_errors=True) outputter.write(output)