def parse_email(email, entry, key): stripped = email.strip() if ' ' in stripped: terminal.warn( u"In entry {0}: possibly malformed email in field {1}: '{2}'". format(entry, key, email)) return stripped
def add_to_topic(self, topic, entry): if len(topic) > 0: if topic[0] not in self.subtopics: terminal.warn(u"In entry {0}: unknown (sub)topic {1}".format( entry, topic)) else: self.subtopics[topic[0]].add_to_topic(topic[1:], entry) else: self.entries.append(entry)
def validate(entry, attributes): sane_attributes = {} missing_keys = [] processed_keys = set() for key, (split, processor, default) in metadata.attribute_schema.items(): if processor is None: processor = lambda str, **kwargs: str if key.endswith("*"): shortkey = key[:len(key) - 1] result = OrderedDict() process = partial(processor, entry=entry, shortkey=shortkey) for appkey, str in attributes.items(): if appkey.startswith(shortkey + "-"): processed_keys.add(appkey) app = appkey[len(shortkey) + 1:] if not split: result[app] = process(str.strip(), appendix=app) else: result[app] = [ process(s.strip(), appendix=app) for s in str.split(',') ] sane_attributes[shortkey] = result else: process = partial(processor, entry=entry, key=key) if default is None and key not in attributes: missing_keys.append(key) sane_attributes[key] = process("") if not split else [] else: value = attributes[key] if key in attributes else default processed_keys.add(key) if not split: sane_attributes[key] = process(value) else: sane_attributes[key] = [ process(s.strip()) for s in value.split(',') ] if missing_keys: error(u"In entry {0}: missing key(s) {1!s}".format( entry, missing_keys), abort=True) extraneous_keys = set(attributes.keys()) - processed_keys if extraneous_keys: warn(u"In entry {0}: unknown key(s) {1!s}. Have you misspelled them?". format(entry, list(extraneous_keys))) return sane_attributes
def associate_releases(entries, versions, filename): for _, attributes in entries.items(): attributes['releases'] = OrderedDict() prog = re.compile(release_pattern) warnings = {} try: with open(filename) as input: lines = [] for line in input: line = line.strip() result = prog.match(line) try: entry, date = result.groups() except ValueError as ex: error(u"In file {0}: Malformed release {1}".format( filename, line.replace), exception=ex) else: if not entry in entries: if not entry in warnings: warnings[entry] = [line] else: warnings[entry].append(line) else: lines.append((entry, date)) for entry, releases in warnings.items(): warn(u"In file {0}: In release(s) {1!s}: Unknown entry {2}".format( filename, releases, entry)) lines.sort(reverse=True) for line in lines: found = False entry, date = line for version_number, version_date in versions: if version_date <= date: entry_releases = entries[entry]['releases'] if version_number not in entry_releases: entry_releases[version_number] = [] entry_releases[version_number].append(date) found = True break if not found: warn( u"In file {0}: In release {1}: Release date {2} has no matching version" .format(filename, line, date)) except Exception as ex: error(u"In file {0}: error".format(filename), exception=ex) error("Not processing releases")
def parse_name_url(name, entry, key): if name.find(" and ") != -1: terminal.warn( u"In entry {0}: {1} field contains 'and'. Use ',' to separate names." .format(entry, key)) url_start = name.find('<') url_end = name.find('>') if url_start != -1 and url_end != -1: url = name[url_start + 1:url_end].strip() if url.startswith("mailto:"): url = url.replace("@", " /at/ ").replace(".", " /dot/ ") elif "@" in url: terminal.warn( u"In entry {0}: Found mail address without 'mailto:': {1}". format(entry, url)) url = "mailto:" + url url = url.replace("@", " /at/ ").replace(".", " /dot/ ") return name[:url_start].strip(), url else: terminal.notice(u"In entry {0}: no URL specified for {1} {2} ".format( entry, key, name)) return name, None
def main(): usage = "sitegen.py [-h] [--templates TEMPLATES_DIR --dest DEST_DIR] [--status STATUS_FILE] [--deps DEPS_FILE] METADATA_DIR THYS_DIR" parser = argparse.ArgumentParser(usage=usage) parser.add_argument("metadata_dir", metavar="METADATA_DIR", action="store", help="metadata location") parser.add_argument("thys_dir", metavar="THYS_DIR", action="store", help="directory with afp entries") parser.add_argument("--templates", action="store", dest="templates_dir", help="directory with Jinja2 templates") parser.add_argument("--dest", action="store", dest="dest_dir", help="destination dir for generated html files") parser.add_argument("--status", action="store", dest="status_file", help="status file location (devel)") parser.add_argument("--deps", action="store", dest="deps_file", help="dependencies file location") parser.parse_args(namespace=options) options.is_devel = options.status_file is not None if options.dest_dir and not options.templates_dir: error("Please specify templates dir", abort=True) # parse metadata entries = parse(os.path.join(options.metadata_dir, "metadata")) versions = read_versions( os.path.join(options.metadata_dir, "release-dates")) associate_releases(entries, versions, os.path.join(options.metadata_dir, "releases")) if len(entries) == 0: warn("In metadata: No entries found") # generate depends-on, used-by entries, lines of code and number of lemmas # by using an afp_dict object # TODO: error instead of warn deps_dict = metadata.empty_deps(entries) if options.deps_file: with open(options.deps_file, 'r') as df: deps_dict = metadata.read_deps(df) else: warn("No dependencies file specified") afp_dict = afpstats.afp_dict(entries, options.thys_dir, deps_dict) afp_dict.build_stats() for e in entries: entries[e]['depends-on'] = list(map(str, afp_dict[e].imports)) entries[e]['used-by'] = list(map(str, afp_dict[e].used)) # perform check count = check_fs(entries, options.thys_dir) output = "Checked directory {0}. Found {1} warnings.".format( options.thys_dir, count) color = 'yellow' if count > 0 else 'green' print(colored(output, color, attrs=['bold'])) # perform generation if options.dest_dir: if options.status_file is not None: (build_data, status) = parse_status(options.status_file) for a in afp_dict: if a in status: afp_dict[a].status = status[a] else: afp_dict[a].status = "skipped" else: build_data = dict() builder = templates.Builder(options, entries, afp_dict) builder.generate_topics() builder.generate_index() builder.generate_entries() builder.generate_statistics() builder.generate_download() for s in [ "about", "citing", "updating", "search", "submitting", "using" ]: builder.generate_standard(s + ".html", s + ".tpl") builder.generate_rss(30) #TODO: look over it one more time if options.is_devel: builder.generate_status(build_data)