def merge(self, other):
"""
Merge the environment markers. Assumes package and qualified_package are the same with other.
:param other: other requirement
:type other: Requirement
"""
new_markers = {}
for marker in self.markers:
key = "{name} {operator}".format(name=marker.name, operator=marker.operator)
if key not in new_markers:
new_markers[key] = []
new_markers[key].extend(marker.value)
for marker in other.markers:
key = "{name} {operator}".format(name=marker.name, operator=marker.operator)
if key not in new_markers:
new_markers[key] = []
new_markers[key].extend(marker.value)
self.markers = []
for key in new_markers:
name, operator = key.split(' ')
value = sorted(list(set(new_markers[key])))
debug("merge => {name} {operator} {value}".format(name=name,
operator=operator,
value=value))
self.markers.append(EnvironmentMarker("{name} {operator} {value}".format(name=name,
operator=operator,
value=' '.join(value))))
def get_project_version(project_package=None):
r"""
Get the version from __init__.py with a line: /^__version__\s*=\s*(\S+)/
If it doesn't exist try to load it from the VERSION.txt file.
If still no joy, then return '0.0.0'
:param project_package: the root package
:type project_package: str
:returns: the version string
:rtype: str
"""
# trying __init__.py first
try:
file_name = _file_spec('__init__.py', project_package)
debug("version_file => %s" % file_name)
# noinspection PyBroadException
try:
# python3
with open(file_name, 'r', encoding='utf-8') as inFile:
for line in inFile.readlines():
match = re.match(VERSION_REGEX, line)
if match:
return match.group(1)
except:
# python2
with open(file_name, 'r') as inFile:
for line in inFile.readlines():
match = re.match(VERSION_REGEX, line)
if match:
return match.group(1)
except IOError:
pass
# no joy, so try getting the version from a VERSION.txt file.
try:
file_name = _file_spec('VERSION.txt', project_package)
info("version_file => %s" % file_name)
# noinspection PyBroadException
try:
# python3
with open(file_name, 'r', encoding='utf-8') as in_file:
return in_file.read().strip()
except:
# python2
with open(file_name, 'r') as in_file:
return in_file.read().strip()
except IOError:
try:
file_name = _file_spec('VERSION.txt', Project.herringfile_dir)
info("version_file => %s" % file_name)
with open(file_name) as in_file:
return in_file.read().strip()
except IOError:
pass
# no joy again, so set to initial version and try again
set_project_version('0.0.1', project_package)
return get_project_version(project_package)
def infos(self, exists=True):
"""
get VenvInfo instances generator.
Usage
-----
::
for venv_info in venvs.infos():
pass
:param exists: the virtualenv must exist to be included in the generator
:type exists: bool
"""
if not self.in_virtualenv and not self.defined:
raise NoAvailableVirtualenv()
value = self._ver_attr
if not is_sequence(value):
value = [value]
try:
for ver in value:
debug("ver: {ver}".format(ver=ver))
if re.match(r'\d+', ver):
venv_info = VenvInfo(ver)
else:
venv_info = VenvInfo(venv=ver)
if exists:
if venv_info.exists():
yield venv_info
else:
yield venv_info
except Exception as ex:
error(str(ex))
error(traceback.format_exc())
def _create_module_diagrams(path):
"""
create module UML diagrams
:param path: the module path
:type path: str
"""
info("_create_module_diagrams")
if not executables_available(['pyreverse']):
warning('pyreverse not available')
return
with open(os.path.join(Project.docs_dir, "pyreverse.log"), "w") as outputter:
for module_path in [root for root, dirs, files in os.walk(path) if os.path.basename(root) != '__pycache__']:
debug("module_path: {path}".format(path=module_path))
init_filename = os.path.join(module_path, '__init__.py')
if os.path.exists(init_filename):
info(init_filename)
name = os.path.basename(module_path).split(".")[0]
output = run_python('pyreverse -o svg -p {name} {module} '.format(name=name, module=module_path),
verbose=True, ignore_errors=True)
outputter.write(output)
errors = [line for line in output.splitlines() if not line.startswith('parsing')]
if errors:
info(errors)
def required_files(self):
"""
Add required packages (specified in module docstrings) to the appropriate requirements text file(s).
"""
debug("requiredFiles")
needed_dict = Requirements(self._project).find_missing_requirements()
for filename in needed_dict.keys():
needed = needed_dict[filename]
debug("needed: %s" % repr(needed))
try:
requirements_filename = os.path.join(self._project.herringfile_dir, filename)
if not os.path.isfile(requirements_filename):
with open(requirements_filename, 'w') as req_file:
req_file.write('-e .\n\n')
with open(requirements_filename, 'a') as req_file:
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=True)
if out_line:
req_file.write(out_line + "\n")
for need in sorted(unique_list(list(needed))):
out_line = need.qualified(qualifiers=False)
if out_line:
req_file.write(out_line + "\n")
except IOError as ex:
warning("Can not add the following to the {filename} file: {needed}\n{err}".format(
filename=filename, needed=repr(needed), err=str(ex)))
def find_missing_requirements(self):
"""
Find the required packages that are not in the requirements.txt file.
:return: set of missing packages.
:rtype: set[Requirement]
"""
requirements = self._reduce_by_version(self._get_requirements_dict_from_py_files())
debug('requirements:')
debug(pformat(requirements))
needed = []
needed.extend(sorted(compress_list(unique_list(requirements))))
filename = 'requirements.txt'
if not os.path.exists(filename):
debug("Missing: " + filename)
diff = sorted(set(needed))
else:
with open(filename) as in_file:
existing_requirements = []
for line in [line.strip() for line in in_file.readlines()]:
if line and not line.startswith('#'):
existing_requirements.append(Requirement(line))
existing = sorted(compress_list(unique_list(existing_requirements)))
difference = [req for req in needed if req not in existing]
diff = sorted(set([req for req in difference
if not req.markers or Requirement(req.package) not in needed]))
debug("find_missing_requirements.needed: {pkgs}".format(pkgs=pformat(needed)))
debug("find_missing_requirements.diff: {pkgs}".format(pkgs=pformat(diff)))
return diff
def _find_item_groups(self, lines):
item_indexes = [i for i, item in enumerate(lines) if re.match(self.ITEM_REGEX, item)]
debug("item_indexes: %s" % repr(item_indexes))
item_groups = []
for k, g in groupby(enumerate(item_indexes), lambda x: x[0] - x[1]): # lambda (i, x): i - x):
item_groups.append(list(map(itemgetter(1), g)))
return item_groups
def _hack_package(self, line):
match = re.match(r"(.+)\s+package\s*", line)
if match:
debug("matched package")
line = "|package| " + match.group(1)
self.line_length = len(line)
line += "\n"
return line
def _hack_package(self, line):
match = re.match(r'(.+)\s+package\s*', line)
if match:
debug("matched package")
line = '|package| ' + match.group(1)
self.line_length = len(line)
line += '\n'
return line
def check_requirements():
""" Checks that herringfile and herringlib/* required packages are in requirements.txt file """
debug("check_requirements")
needed = Requirements(Project).find_missing_requirements()
if needed:
info("Please add the following to your %s file:\n" % 'requirements.txt')
info("\n".join(str(needed)))
else:
info("Your %s includes all known herringlib task requirements" % 'requirements.txt')
def _get_herringlib_py_files(self):
"""find all the .py files in the herringlib directory"""
lib_files = []
debug("HerringFile.herringlib_paths: %s" % repr(HerringFile.herringlib_paths))
for herringlib_path in [os.path.join(path_, 'herringlib') for path_ in HerringFile.herringlib_paths]:
for dir_path, dir_names, files in os.walk(herringlib_path):
for f in fnmatch.filter(files, '*.py'):
lib_files.append(os.path.join(dir_path, f))
return lib_files
def _hack_module(self, line):
match = re.match(r"(.+)\s+module\s*", line)
if match:
debug("matched module")
self.package = False
self.class_name = match.group(1).split(".")[-1]
line = "|module| " + match.group(1)
self.line_length = len(line)
line += "\n"
return line
def _hack_module(self, line):
match = re.match(r'(.+)\s+module\s*', line)
if match:
debug("matched module")
self.package = False
self.class_name = match.group(1).split('.')[-1]
line = '|module| ' + match.group(1)
self.line_length = len(line)
line += '\n'
return line
def _hack_underline(self, line):
if re.match(r"[=\-\.][=\-\.][=\-\.]+", line):
debug("matched [=\-\.][=\-\.][=\-\.]+")
if self.line_length > 0:
line = "%s\n" % (line[0] * self.line_length)
if self.package:
line += _package_line(self.module_name)
if self.class_name:
line += _class_line(self.module_name, self.class_name)
return line
def __init__(self, *attr_names):
self._ver_attr = None
self._raise_when_in_venv = False
debug(repr(attr_names))
for name in attr_names:
debug(name)
self._ver_attr = getattr(Project, name, None)
if self._ver_attr is not None:
info("_ver_attr: %s" % repr(self._ver_attr))
break
def _reduce_by_version(self, requirements):
requirement_dict = {}
debug("requirements:\n" + pformat(requirements))
for requirement in requirements:
if requirement.package not in requirement_dict:
requirement_dict[requirement.package] = requirement
else:
debug("merge {src} into {dest}".format(src=str(requirement),
dest=str(requirement_dict[requirement.package])))
requirement_dict[requirement.package].merge(requirement)
return requirement_dict.values()
def clean():
""" remove build artifacts """
recursively_remove(Project.herringfile_dir, '*.pyc')
recursively_remove(Project.herringfile_dir, '*~')
debug(repr(Project.__dict__))
dirs = [Project.dist_dir, Project.egg_dir]
# print("dirs => %s" % repr(dirs))
for dir_name in dirs:
if os.path.exists(dir_name):
shutil.rmtree(dir_name)
def _hack_mod(self, line):
match = re.match(r":mod:`(.+)`(.*)", line)
if match:
debug("matched :mod:")
key = match.group(1)
if key in self.name_dict:
value = self.name_dict[key]
line = "".join(":mod:`%s`%s\n" % (value, match.group(2)))
self.line_length = len(line)
self.package = re.search(r":mod:.+Package", line)
self.class_name = key
return line
def __init__(self, marker):
self.marker = marker
self.name = None
self.operator = None
self.value = None
if self.marker is not None:
match = re.match(r'''(\S+)\s*((?:[!=<>]+)|(?:not in)|(?<!not )in)\s*[\"\']?([\d\.\s]+)[\"\']?''',
self.marker)
if match:
self.name = match.group(1)
self.operator = match.group(2)
self.value = match.group(3).split(' ')
debug("EnvironmentMarker:\n {marker}\n raw='{raw}'".format(marker=self, raw=match.group(3)))
def _get_module_docstring(self, file_path):
"""
Get module-level docstring of Python module at filepath, e.g. 'path/to/file.py'.
:param file_path: The filepath to a module file.
:type: str
:returns: the module docstring
:rtype: str
"""
debug("_get_module_docstring('{file}')".format(file=file_path))
tree = ast.parse(''.join(open(file_path)))
# noinspection PyArgumentEqualDefault
docstring = (ast.get_docstring(tree, clean=True) or '').strip()
debug("docstring: %s" % docstring)
return docstring
def recursively_remove(path, pattern):
"""
recursively remove files that match a given pattern
:param path: The directory to start removing files from.
:type path: str
:param pattern: The file glob pattern to match for removal.
:type pattern: str
"""
files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, pattern)]
for file_ in files:
debug("removing: %s" % file_)
os.remove(file_)
def __init__(self, line):
self.line = line.strip()
# strip double quotes off of line
if self.line.startswith('"') and self.line.endswith('"'):
self.line = self.line[1:-1]
debug("Requirement: {line}".format(line=self.line))
match = re.match(r'(.*?#egg=[^\s;]+)', self.line)
if match:
self.package = match.group(1).strip().strip(';')
else:
self.package = re.split(r'[^a-zA-Z0-9_\-]', self.line)[0].strip().strip(';')
self.qualified_package = re.split(r';', self.line)[0].strip()
try:
self.markers = [EnvironmentMarker(re.split(r';', self.line)[1].strip().replace('"', "'"))]
except IndexError:
self.markers = []
def _parse_docstring(self, doc_string):
"""
Extract the required packages from the docstring.
This makes the following assumptions:
1) there is a line in the docstring that contains "requirements.txt".
2) after that line, ignoring blank lines, there are bullet list items starting with a '*'
3) these bullet list items are the names of the required third party packages followed by any optional
conditions
:param doc_string: a module docstring
:type: str
:return: requirements by requirement file
:rtype: list(Requirement)
"""
requirements = []
debug("_parse_docstring")
if doc_string is None or not doc_string:
return requirements
raw_lines = list(filter(str.strip, doc_string.splitlines()))
lines = self._variable_substitution(raw_lines)
# lines should now contain:
# ['blah', 'blah', '...requirements.txt...','* pkg 1', '* pkg 2', 'blah']
debug(lines)
requirement_indexes = [i for i, item in enumerate(lines) if re.search(self.REQUIREMENT_REGEX, item)]
debug("requirement_indexes: %s" % repr(requirement_indexes))
item_groups = self._find_item_groups(lines)
# print("item_groups: %s" % repr(item_groups))
# example using doc_string:
#
# item_indexes: [4, 5, 6, 7, 8, 10, 13, 14]
# item_groups: [[4, 5, 6, 7, 8], [10], [13, 14]]
#
# we want:
# requirements = [
# [lines[4], lines[5], lines[6], lines[7], lines[8]],
# [lines[10]]
# [lines[13], lines[14]],
# ]
for index in requirement_indexes:
for item_group in item_groups:
if item_group[0] == index + 1:
# yes we have items for the requirement file
requirements.extend([Requirement(re.match(self.ITEM_REGEX,
lines[item_index]).group(1))
for item_index in item_group])
debug("requirements:\n%s" % pformat(requirements))
return requirements
def supported_python(self):
"""
Is this requirement intended for the currently running version of python?
If this requirement has a marker (ex: 'foo; python_version == "2.7"') check if the
current python qualifies. If this requirement does not have a marker then return True.
"""
for marker in [m for m in self.markers if m.name == 'python_version']:
if marker.operator is not None and marker.value is not None:
# noinspection PyUnresolvedReferences
import sys
code = "sys.version_info {operator} {version}".format(operator=marker.operator,
version=str(marker.value))
result = eval(code)
debug("{code} returned: {result}".format(code=code, result=str(result)))
return result
return True
def _package_line(self, module_name):
"""create the package figure lines for the given module"""
info("_package_line(%s)" % module_name)
line = ''
package_image = "uml/packages_{name}.svg".format(name=module_name.split('.')[-1])
classes_image = "uml/classes_{name}.svg".format(name=module_name.split('.')[-1])
image_path = os.path.join(Project.docs_dir, '_src', package_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n :width: 1100 px\n\n {name} Packages\n\n".format(
image=package_image,
name=module_name)
line += "\n.. figure:: {image}\n\n {name} Classes\n\n".format(
image=classes_image,
name=module_name)
else:
debug("%s does not exist!" % image_path)
return line
def _class_line(self, module_name, class_name):
"""create the class figure lines for the given module and class"""
info("_class_line(%s, %s)" % (module_name, class_name))
line = ''
classes_images = [
"uml/classes_{module}.{name}.png".format(module=module_name, name=class_name),
"uml/classes_{module}.png".format(module=module_name),
]
for classes_image in classes_images:
image_path = os.path.join(Project.docs_dir, '_src', classes_image)
if os.path.exists(image_path):
info("adding figure %s" % image_path)
line += "\n.. figure:: {image}\n\n {name} Class\n\n".format(
image=classes_image,
name=class_name)
break
else:
debug("%s does not exist!" % image_path)
return line
def mkvirtualenv(self):
"""Make a virtualenv"""
new_env = Project.env_without_virtualenvwrapper()
debug("os.environ['PATH']: \"{path}\"".format(path=os.environ['PATH']))
debug("new_env: {env}".format(env=pformat(new_env)))
with LocalShell() as local:
venv_script = Project.virtualenvwrapper_script
# noinspection PyArgumentEqualDefault
venvs = local.run('/bin/bash -c "source {venv_script} ;'
'lsvirtualenv -b"'.format(venv_script=venv_script),
verbose=False,
env=new_env).strip().split("\n")
if self.venv not in venvs:
python_path = local.system('which {python}'.format(python=self.python)).strip()
local.run('/bin/bash -c "source {venv_script} ; '
'mkvirtualenv -p {python} {venv}"'.format(venv_script=venv_script,
python=python_path,
venv=self.venv),
verbose=True,
env=new_env)
def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(["pynsource"]):
return
files = [
os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, "*.py")
]
debug("files: {files}".format(files=repr(files)))
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + "/", "").replace(".py", ".png").replace("/", ".")
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
run_python(
"pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False,
ignore_errors=True,
)
def _get_requirements_dict_from_py_files(self, lib_files=None):
"""
Scan the herringlib py file docstrings extracting the 3rd party requirements.
:param lib_files: list of filenames to scan for requirement comments. Set to None to scan all py files
in the herringlib directory and the herringfile.
:type lib_files: list[str]
:return: requirements dict where key is the requirements file name (ex: "requirements.txt") and
the value is a list of package names (ex: ['argparse', 'wheel']).
:rtype: dict[str,list[Requirement]]
"""
if lib_files is None:
lib_files = self._get_herringlib_py_files()
lib_files.append(os.path.join(self._project.herringfile_dir, 'herringfile'))
debug("files: %s" % repr(lib_files))
requirements = {}
for file_ in lib_files:
debug('file: %s' % file_)
required_files_dict = self._parse_docstring(self._get_module_docstring(file_))
debug('required_files: %s' % pformat(required_files_dict))
for requirement_filename in required_files_dict.keys():
if requirement_filename not in requirements.keys():
requirements[requirement_filename] = []
for req in required_files_dict[requirement_filename]:
if req not in requirements[requirement_filename]:
requirements[requirement_filename].append(req)
return requirements
def _create_class_diagrams(path):
"""
Create class UML diagram
:param path: path to the module file.
:type path: str
"""
info("_create_class_diagrams")
if not executables_available(['pynsource']):
warning('pynsource not available')
return
files = [os.path.join(dir_path, f)
for dir_path, dir_names, files in os.walk(path)
for f in fnmatch.filter(files, '*.py')]
debug("files: {files}".format(files=repr(files)))
with open(os.path.join(Project.docs_dir, "pynsource.log"), "w") as outputter:
for src_file in files:
debug(src_file)
name = src_file.replace(Project.herringfile_dir + '/', '').replace('.py', '.png').replace('/', '.')
output = "classes_{name}".format(name=name)
debug(output)
if not os.path.isfile(output) or (os.path.isfile(output) and is_newer(output, src_file)):
output = run_python("pynsource -y {output} {source}".format(output=output, source=src_file),
verbose=False, ignore_errors=True)
outputter.write(output)
