def check(): """Ensure the server exists.""" log.info("checking for a server...") if settings.SERVER_HOST is None: log.info("no server in use") return if not settings.SERVER_HOST: raise DoorstopError("no server specified") if not exists(): raise DoorstopError("unknown server: {}".format(settings.SERVER_HOST))
def new(tree, path, root, prefix, sep=None, digits=None, parent=None, auto=None): # pylint: disable=R0913,C0301 """Create a new document. :param tree: reference to tree that contains this document :param path: path to directory for the new document :param root: path to root of the project :param prefix: prefix for the new document :param sep: separator between prefix and numbers :param digits: number of digits for the new document :param parent: parent UID for the new document :param auto: automatically save the document :raises: :class:`~doorstop.common.DoorstopError` if the document already exists :return: new :class:`~doorstop.core.document.Document` """ # Check separator if sep and sep not in settings.SEP_CHARS: raise DoorstopError("invalid UID separator '{}'".format(sep)) config = os.path.join(path, Document.CONFIG) # Check for an existing document if os.path.exists(config): raise DoorstopError("document already exists: {}".format(path)) # Create the document directory Document._create(config, name='document') # Initialize the document document = Document(path, root=root, tree=tree, auto=False) document.prefix = ( # type: ignore prefix if prefix is not None else document.prefix) document.sep = sep if sep is not None else document.sep # type: ignore document.digits = ( # type: ignore digits if digits is not None else document.digits) document.parent = ( # type: ignore parent if parent is not None else document.parent) if auto or (auto is None and Document.auto): document.save() # Return the document return document
def _place(self, document): """Attempt to place the document in the current tree. :param document: :class:`doorstop.core.document.Document` to add :raises: :class:`~doorstop.common.DoorstopError` if the document cannot yet be placed """ log.debug("trying to add {}...".format(document)) if not self.document: # tree is empty if document.parent: msg = "unknown parent for {}: {}".format( document, document.parent) raise DoorstopError(msg) self.document = document elif document.parent: # tree has documents, document has parent if document.parent.lower() == self.document.prefix.lower(): # Current document is the parent node = Tree(document, self) self.children.append(node) else: # Search for the parent for child in self.children: try: child._place(document) # pylint: disable=W0212 except DoorstopError: pass # the error is raised later else: break else: msg = "unknown parent for {}: {}".format( document, document.parent) raise DoorstopError(msg) else: # tree has documents, but no parent specified for document msg = "no parent specified for {}".format(document) log.info(msg) prefixes = ', '.join(document.prefix for document in self) log.info("parent options: {}".format(prefixes)) raise DoorstopError(msg) for document2 in self: children = self._get_prefix_of_children(document2) document2.children = children
def from_list(documents, root=None): """Initialize a new tree from a list of documents. :param documents: list of :class:`~doorstop.core.document.Document` :param root: path to root of the project :raises: :class:`~doorstop.common.DoorstopError` when the tree cannot be built :return: new :class:`~doorstop.core.tree.Tree` """ if not documents: return Tree(document=None, root=root) unplaced = list(documents) for document in list(unplaced): if document.parent is None: log.info("root of the tree: {}".format(document)) tree = Tree(document) document.tree = tree unplaced.remove(document) break else: raise DoorstopError("no root document") while unplaced: count = len(unplaced) for document in list(unplaced): if document.parent is None: log.info("root of the tree: {}".format(document)) message = "multiple root documents:\n- {}: {}\n- {}: {}".format( tree.document.prefix, tree.document.path, document.prefix, document.path, ) raise DoorstopError(message) try: tree._place(document) # pylint: disable=W0212 except DoorstopError as error: log.debug(error) else: log.info("added to tree: {}".format(document)) document.tree = tree unplaced.remove(document) if len(unplaced) == count: # no more documents could be placed log.debug("unplaced documents: {}".format(unplaced)) msg = "unplaced document: {}".format(unplaced[0]) raise DoorstopError(msg) return tree
def find_document(self, value): """Get a document by its prefix. :param value: document or prefix :raises: :class:`~doorstop.common.DoorstopError` if the document cannot be found :return: matching :class:`~doorstop.core.document.Document` """ prefix = Prefix(value) log.debug("looking for document '{}'...".format(prefix)) try: document = self._document_cache[prefix] if document: log.trace("found cached document: {}".format(document)) return document else: log.trace("found cached unknown: {}".format(prefix)) except KeyError: for document in self: if document.prefix == prefix: log.trace("found document: {}".format(document)) if settings.CACHE_DOCUMENTS: self._document_cache[prefix] = document log.trace("cached document: {}".format(document)) return document log.debug("could not find document: {}".format(prefix)) if settings.CACHE_DOCUMENTS: self._document_cache[prefix] = None log.trace("cached unknown: {}".format(prefix)) raise DoorstopError(Prefix.UNKNOWN_MESSGE.format(prefix))
def _get_issues_document(self, document, skip): """Yield all the item's issues against its document.""" log.debug("getting issues against document...") if document in skip: log.debug("skipping issues against document %s...", document) return # Verify an item's UID matches its document's prefix if self.prefix != document.prefix: msg = "prefix differs from document ({})".format(document.prefix) yield DoorstopInfo(msg) # Verify an item has upward links if all((document.parent, self.normative, not self.derived)) and not self.links: msg = "no links to parent document: {}".format(document.parent) yield DoorstopWarning(msg) # Verify an item's links are to the correct parent for uid in self.links: try: prefix = uid.prefix except DoorstopError: msg = "invalid UID in links: {}".format(uid) yield DoorstopError(msg) else: if document.parent and prefix != document.parent: # this is only 'info' because a document is allowed # to contain items with a different prefix, but # Doorstop will not create items like this msg = "parent is '{}', but linked to: {}".format( document.parent, uid) yield DoorstopInfo(msg)
def split_uid(value): """Split an item's UID string into a prefix, number, name, and exception. >>> UID.split_uid('ABC00123') (Prefix('ABC'), 123, '', None) >>> UID.split_uid('ABC.HLR_01-00123') (Prefix('ABC.HLR_01'), 123, '', None) >>> UID.split_uid('REQ2-001') (Prefix('REQ2'), 1, '', None) >>> UID.split_uid('REQ2-NAME') (Prefix('REQ2'), -1, 'NAME', None) >>> UID.split_uid('REQ2-NAME007') (Prefix('REQ2'), -1, 'NAME007', None) >>> UID.split_uid('REQ2-123NAME') (Prefix('REQ2'), -1, '123NAME', None) """ m = re.match("([\\w.-]+)[" + settings.SEP_CHARS + "](\\w+)", value) if m: try: num = int(m.group(2)) return Prefix(m.group(1)), num, '', None except ValueError: return Prefix(m.group(1)), -1, m.group(2), None m = re.match(r"([\w.-]*\D)(\d+)", value) if m: num = m.group(2) return Prefix(m.group(1).rstrip( settings.SEP_CHARS)), int(num), '', None return None, None, None, DoorstopError("invalid UID: {}".format(value))
def _get_issues_tree(self, tree): """Yield all the item's issues against its tree.""" log.debug("getting issues against tree...") # Verify an item's links are valid identifiers = set() for uid in self.links: try: item = tree.find_item(uid) except DoorstopError: identifiers.add(uid) # keep the invalid UID msg = "linked to unknown item: {}".format(uid) yield DoorstopError(msg) else: # check the linked item if not item.active: msg = "linked to inactive item: {}".format(item) yield DoorstopInfo(msg) if not item.normative: msg = "linked to non-normative item: {}".format(item) yield DoorstopWarning(msg) # check the link status if uid.stamp == Stamp(True): uid.stamp = item.stamp() # convert True to a stamp elif uid.stamp != item.stamp(): if settings.CHECK_SUSPECT_LINKS: msg = "suspect link: {}".format(item) yield DoorstopWarning(msg) # reformat the item's UID identifier2 = UID(item.uid, stamp=uid.stamp) identifiers.add(identifier2) # Apply the reformatted item UIDs if settings.REFORMAT: self._data['links'] = identifiers
def __init__(self, path, root=os.getcwd(), **kwargs): """Initialize a document from an exiting directory. :param path: path to document directory :param root: path to root of project """ super().__init__() # Ensure the directory is valid if not os.path.isfile(os.path.join(path, Document.CONFIG)): relpath = os.path.relpath(path, root) msg = "no {} in {}".format(Document.CONFIG, relpath) raise DoorstopError(msg) # Initialize the document self.path = path self.root = root self.tree = kwargs.get('tree') self.auto = kwargs.get('auto', Document.auto) # Set default values self._data['prefix'] = Document.DEFAULT_PREFIX self._data['sep'] = Document.DEFAULT_SEP self._data['digits'] = Document.DEFAULT_DIGITS self._data['parent'] = None # the root document does not have a parent self._items = [] self._itered = False self.children = []
def find_ref(self, skip=None, root=None, ignored=None): """Get the external file reference and line number. :param skip: function to determine if a path is ignored :param root: override path to the working copy (for testing) :param ignored: override VCS ignore function (for testing) :raises: :class:`~doorstop.common.DoorstopError` when no reference is found :return: relative path to file or None (when no reference set), line number (when found in file) or None (when found as filename) or None (when no reference set) """ root = root or self.root ignored = ignored or \ self.tree.vcs.ignored if self.tree else (lambda _: False) # Return immediately if no external reference if not self.ref: log.debug("no external reference to search for") return None, None # Search for the external reference log.debug("seraching for ref '{}'...".format(self.ref)) pattern = r"(\b|\W){}(\b|\W)".format(re.escape(self.ref)) log.debug("regex: {}".format(pattern)) regex = re.compile(pattern) log.debug("search path: {}".format(root)) for root, _, filenames in os.walk(root): for filename in filenames: # pragma: no cover (integration test) path = os.path.join(root, filename) relpath = os.path.relpath(path, self.root) # Skip the item's file while searching if path == self.path: continue # Skip hidden directories if os.path.sep + '.' in path: continue # Skip ignored paths if ignored(path) or (skip and skip(path)): continue # Check for a matching filename if filename == self.ref: return relpath, None # Skip extensions that should not be considered text if os.path.splitext(filename)[-1] in settings.SKIP_EXTS: continue # Search for the reference in the file try: with open(path, 'r') as external: for index, line in enumerate(external): if regex.search(line): log.debug("found ref: {}".format(relpath)) return relpath, index + 1 except UnicodeDecodeError: pass msg = "external reference not found: {}".format(self.ref) raise DoorstopError(msg)
def __new__(cls, value=""): if isinstance(value, Prefix): return value else: if str(value).lower() in settings.RESERVED_WORDS: raise DoorstopError("cannot use reserved word: %s" % value) obj = super().__new__(cls, Prefix.load_prefix(value)) return obj
def launch(path, tool=None): # pragma: no cover (integration test) """Open a file using the default editor. :param path: path of file to open :param tool: path of alternate editor :raises: :class:`~doorstop.common.DoorstopError` no default editor or editor unavailable :return: launched process if long-running, else None """ # Determine how to launch the editor if tool: args = [tool, path] elif sys.platform.startswith('darwin'): args = ['open', path] elif os.name == 'nt': cygstart = find_executable('cygstart') if cygstart: args = [cygstart, path] else: args = ['start', path] elif os.name == 'posix': args = ['xdg-open', path] # Launch the editor try: log.info("opening '{}'...".format(path)) process = _call(args) except FileNotFoundError: raise DoorstopError("editor not found: {}".format(args[0])) # Wait for the editor to launch time.sleep(LAUNCH_DELAY) if process.poll() is None: log.debug("process is running...") else: log.debug("process exited: {}".format(process.returncode)) if process.returncode != 0: raise DoorstopError("no default editor for: {}".format(path)) # Return the process if it's still running if process.returncode is None: return process
def test_validate_invalid_ref(self): """Verify an invalid reference fails validity.""" with patch( 'doorstop.core.item.Item.find_ref', Mock(side_effect=DoorstopError("test invalid ref")), ): with ListLogHandler(core.base.log) as handler: self.assertFalse(self.item.validate()) self.assertIn("test invalid ref", handler.records)
def mock_find_item(uid): """Return a mock item and store it.""" if uid == 'bb': mock_item = self.document.add_item(level=Level('3.2'), uid=uid) elif uid == 'new': raise DoorstopError("unknown UID: bab") else: mock_item = self.document.add_item(uid=uid) return mock_item
def mock_find_item(uid): """Return a mock item and store it.""" mock_item = MagicMock() if uid == 'bb': mock_item.level = Level('3.2') elif uid == 'bab': raise DoorstopError("unknown UID: bab") mock_item.uid = uid mock_items.append(mock_item) return mock_item
def __init__(self, document, path, root=os.getcwd(), **kwargs): """Initialize an item from an existing file. :param path: path to Item file :param root: path to root of project """ super().__init__() # Ensure the path is valid if not os.path.isfile(path): raise DoorstopError("item does not exist: {}".format(path)) # Ensure the filename is valid filename = os.path.basename(path) name, ext = os.path.splitext(filename) try: UID(name).check() except DoorstopError: msg = "invalid item filename: {}".format(filename) raise DoorstopError(msg) from None # Ensure the file extension is valid if ext.lower() not in self.EXTENSIONS: msg = "'{0}' extension not in {1}".format(path, self.EXTENSIONS) raise DoorstopError(msg) # Initialize the item self.path = path self.root: str = root self.document = document self.tree = kwargs.get('tree') self.auto = kwargs.get('auto', Item.auto) self.reference_finder = ReferenceFinder() self.yaml_validator = YamlValidator() # Set default values self._data['level'] = Item.DEFAULT_LEVEL self._data['active'] = Item.DEFAULT_ACTIVE self._data['normative'] = Item.DEFAULT_NORMATIVE self._data['derived'] = Item.DEFAULT_DERIVED self._data['reviewed'] = Item.DEFAULT_REVIEWED self._data['text'] = Item.DEFAULT_TEXT self._data['ref'] = Item.DEFAULT_REF self._data['references'] = None self._data['links'] = set() if settings.ENABLE_HEADERS: self._data['header'] = Item.DEFAULT_HEADER
def _write(self, text, path): # pragma: no cover (integration test) """Write text to the object's file. :param text: text to write to a file :param path: path to the file """ if not self._exists: raise DoorstopError("cannot save to deleted: {}".format(self)) common.write_text(text, path)
def load(self, reload=False): """Load the document's properties from its file.""" if self._loaded and not reload: return log.debug("loading {}...".format(repr(self))) data = self._load_with_include(self.config) # Store parsed data sets = data.get('settings', {}) for key, value in sets.items(): try: if key == 'prefix': self._data[key] = Prefix(value) elif key == 'sep': self._data[key] = value.strip() elif key == 'parent': self._data[key] = value.strip() elif key == 'digits': self._data[key] = int(value) else: msg = "unexpected document setting '{}' in: {}".format( key, self.config ) raise DoorstopError(msg) except (AttributeError, TypeError, ValueError): msg = "invalid value for '{}' in: {}".format(key, self.config) raise DoorstopError(msg) # Store parsed attributes attributes = data.get('attributes', {}) for key, value in attributes.items(): if key == 'defaults': self._attribute_defaults = value elif key == 'reviewed': self._extended_reviewed = sorted(set(v for v in value)) else: msg = "unexpected attributes configuration '{}' in: {}".format( key, self.config ) raise DoorstopError(msg) # Set meta attributes self._loaded = True if reload: list(self._iter(reload=reload))
def check(ext, get_lines_gen=False, get_file_func=False): """Confirm an extension is supported for export. :param get_lines_func: return a lines generator if available :param get_file_func: return a file creator if available :raises: :class:`doorstop.common.DoorstopError` for unknown formats :return: function requested if available """ exts = ', '.join(ext for ext in FORMAT) lines_exts = ', '.join(ext for ext in FORMAT_LINES) file_exts = ', '.join(ext for ext in FORMAT_FILE) fmt = "unknown {{}} format: {} (options: {{}})".format(ext or None) if get_lines_gen: try: gen = FORMAT_LINES[ext] except KeyError: exc = DoorstopError(fmt.format("lines export", lines_exts)) raise exc from None else: log.debug("found lines generator for: {}".format(ext)) return gen if get_file_func: try: func = FORMAT_FILE[ext] except KeyError: exc = DoorstopError(fmt.format("file export", file_exts)) raise exc from None else: log.debug("found file creator for: {}".format(ext)) return func if ext not in FORMAT: exc = DoorstopError(fmt.format("export", exts)) raise exc return None
def _read(self, path): # pragma: no cover (integration test) """Read text from the object's file. :param path: path to a text file :return: contexts of text file """ if not self._exists: msg = "cannot read from deleted: {}".format(self.path) raise DoorstopError(msg) return common.read_text(path)
def __init__(self, *values, stamp=None): """Initialize an UID using a string, dictionary, or set of parts. Option 1: :param *values: UID + optional stamp ("UID:stamp") :param stamp: stamp of :class:`~doorstop.core.item.Item` (if known) Option 2: :param *values: {UID: stamp} :param stamp: stamp of :class:`~doorstop.core.item.Item` (if known) Option 3: :param *values: prefix, separator, number, digit count param stamp: stamp of :class:`~doorstop.core.item.Item` (if known) """ if values and isinstance(values[0], UID): self.stamp = stamp or values[0].stamp return self.stamp = stamp or Stamp() # Join values if len(values) == 0: self.value = '' elif len(values) == 1: value = values[0] if isinstance(value, str) and ':' in value: # split UID:stamp into a dictionary pair = value.rsplit(':', 1) value = {pair[0]: pair[1]} if isinstance(value, dict): pair = list(value.items())[0] self.value = str(pair[0]) self.stamp = self.stamp or Stamp(pair[1]) else: self.value = str(value) if values[0] else '' elif len(values) == 4: self.value = UID.join_uid(*values) else: raise TypeError("__init__() takes 1 or 4 positional arguments") # Split values try: parts = UID.split_uid(self.value) self._prefix = Prefix(parts[0]) self._number = parts[1] except ValueError: self._prefix = self._number = None self._exc = DoorstopError("invalid UID: {}".format(self.value)) else: self._exc = None
def include(self, node): container = IncludeLoader.filenames[0] dirname = os.path.dirname(container) filename = os.path.join(dirname, self.construct_scalar(node)) IncludeLoader.filenames.insert(0, filename) try: with open(filename, 'r') as f: data = yaml.load(f, IncludeLoader) except Exception as ex: msg = "include in '{}' failed: {}".format(container, ex) raise DoorstopError(msg) IncludeLoader.filenames.pop() return data
def _create(path, name): # pragma: no cover (integration test) """Create a new file for the object. :param path: path to new file :param name: humanized name for this file :raises: :class:`~doorstop.common.DoorstopError` if the file already exists """ if os.path.exists(path): raise DoorstopError("{} already exists: {}".format(name, path)) common.create_dirname(path) common.touch(path)
def find_item(self, value, _kind=''): """Get an item by its UID. :param value: item or UID :raises: :class:`~doorstop.common.DoorstopError` if the item cannot be found :return: matching :class:`~doorstop.core.item.Item` """ uid = UID(value) _kind = (' ' + _kind) if _kind else _kind # for logging messages log.debug("looking for{} item '{}'...".format(_kind, uid)) try: item = self._item_cache[uid] if item: log.trace("found cached item: {}".format(item)) # type: ignore if item.active: return item else: log.trace( "item is inactive: {}".format(item)) # type: ignore else: log.trace( "found cached unknown: {}".format(uid)) # type: ignore except KeyError: for document in self: try: item = document.find_item(uid, _kind=_kind) except DoorstopError: pass # item not found in that document else: log.trace("found item: {}".format(item)) # type: ignore if settings.CACHE_ITEMS: self._item_cache[uid] = item log.trace( "cached item: {}".format(item)) # type: ignore if item.active: return item else: log.trace("item is inactive: {}".format( item)) # type: ignore log.debug("could not find item: {}".format(uid)) if settings.CACHE_ITEMS: self._item_cache[uid] = None log.trace("cached unknown: {}".format(uid)) # type: ignore raise DoorstopError(UID.UNKNOWN_MESSAGE.format(k=_kind, u=uid))
def get_next_number(prefix): """Get the next number for the given document prefix.""" number = None url = utilities.build_url(path='/documents/{p}/numbers'.format(p=prefix)) if not url: log.info("no server to get the next number from") return None headers = {'content-type': 'application/json'} response = requests.post(url, headers=headers) if response.status_code == 200: data = response.json() number = data.get('next') if number is None: raise DoorstopError("bad response from: {}".format(url)) log.info("next number from the server: {}".format(number)) return number
def find_ref(self): """Get the external file reference and line number. :raises: :class:`~doorstop.common.DoorstopError` when no reference is found :return: relative path to file or None (when no reference set), line number (when found in file) or None (when found as filename) or None (when no reference set) """ # Return immediately if no external reference if not self.ref: log.debug("no external reference to search for") return None, None # Update the cache if not settings.CACHE_PATHS: pyficache.clear_file_cache() # Search for the external reference log.debug("seraching for ref '{}'...".format(self.ref)) pattern = r"(\b|\W){}(\b|\W)".format(re.escape(self.ref)) log.trace("regex: {}".format(pattern)) regex = re.compile(pattern) for path, filename, relpath in self.tree.vcs.paths: # Skip the item's file while searching if path == self.path: continue # Check for a matching filename if filename == self.ref: return relpath, None # Skip extensions that should not be considered text if os.path.splitext(filename)[-1] in settings.SKIP_EXTS: continue # Search for the reference in the file lines = pyficache.getlines(path) if lines is None: log.trace("unable to read lines from: {}".format(path)) continue for lineno, line in enumerate(lines, start=1): if regex.search(line): log.debug("found ref: {}".format(relpath)) return relpath, lineno msg = "external reference not found: {}".format(self.ref) raise DoorstopError(msg)
def find_item(self, value, _kind=''): """Return an item by its UID. :param value: item or UID :raises: :class:`~doorstop.common.DoorstopError` if the item cannot be found :return: matching :class:`~doorstop.core.item.Item` """ uid = UID(value) for item in self: if item.uid == uid: return item raise DoorstopError("no matching{} UID: {}".format(_kind, uid))
def new(tree, path, root, prefix, sep=None, digits=None, parent=None, auto=None): # pylint: disable=R0913,C0301 """Internal method to create a new document. :param tree: reference to tree that contains this document :param path: path to directory for the new document :param root: path to root of the project :param prefix: prefix for the new document :param sep: separator between prefix and numbers :param digits: number of digits for the new document :param parent: parent UID for the new document :param auto: automatically save the document :raises: :class:`~doorstop.common.DoorstopError` if the document already exists :return: new :class:`~doorstop.core.document.Document` """ # TODO: raise a specific exception for invalid separator characters? assert not sep or sep in settings.SEP_CHARS config = os.path.join(path, Document.CONFIG) # Check for an existing document if os.path.exists(config): raise DoorstopError("document already exists: {}".format(path)) # Create the document directory Document._create(config, name='document') # Initialize the document document = Document(path, root=root, tree=tree, auto=False) document.prefix = prefix if prefix is not None else document.prefix document.sep = sep if sep is not None else document.sep document.digits = digits if digits is not None else document.digits document.parent = parent if parent is not None else document.parent if auto or (auto is None and Document.auto): document.save() # Return the document return document
def check(ext): """Confirm an extension is supported for import. :raise DoorstopError: for unknown formats :return: file importer if available """ exts = ', '.join(ext for ext in FORMAT_FILE) msg = "unknown import format: {} (options: {})".format(ext or None, exts) exc = DoorstopError(msg) try: func = FORMAT_FILE[ext] except KeyError: raise exc from None else: log.debug("found file reader for: {}".format(ext)) return func
def check_for_cycle(self, item, cid, path): """Check if a cyclic dependency would be created. :param item: an item on the dependency path :param cid: the child item's UID :param path: the path of UIDs from the child item to the item :raises: :class:`~doorstop.common.DoorstopError` if the link would create a cyclic dependency """ for did in item.links: path2 = path + [did] if did in path: s = " -> ".join(list(map(str, path2))) msg = "link would create a cyclic dependency: {}".format(s) raise DoorstopError(msg) dep = self.find_item(did, _kind='dependency') self.check_for_cycle(dep, cid, path2)