def parse_email(email, entry, key):
stripped = email.strip()
if ' ' in stripped:
terminal.warn(
u"In entry {0}: possibly malformed email in field {1}: '{2}'".
format(entry, key, email))
return stripped
def add_to_topic(self, topic, entry):
if len(topic) > 0:
if topic[0] not in self.subtopics:
terminal.warn(u"In entry {0}: unknown (sub)topic {1}".format(
entry, topic))
else:
self.subtopics[topic[0]].add_to_topic(topic[1:], entry)
else:
self.entries.append(entry)
def validate(entry, attributes):
sane_attributes = {}
missing_keys = []
processed_keys = set()
for key, (split, processor, default) in metadata.attribute_schema.items():
if processor is None:
processor = lambda str, **kwargs: str
if key.endswith("*"):
shortkey = key[:len(key) - 1]
result = OrderedDict()
process = partial(processor, entry=entry, shortkey=shortkey)
for appkey, str in attributes.items():
if appkey.startswith(shortkey + "-"):
processed_keys.add(appkey)
app = appkey[len(shortkey) + 1:]
if not split:
result[app] = process(str.strip(), appendix=app)
else:
result[app] = [
process(s.strip(), appendix=app)
for s in str.split(',')
]
sane_attributes[shortkey] = result
else:
process = partial(processor, entry=entry, key=key)
if default is None and key not in attributes:
missing_keys.append(key)
sane_attributes[key] = process("") if not split else []
else:
value = attributes[key] if key in attributes else default
processed_keys.add(key)
if not split:
sane_attributes[key] = process(value)
else:
sane_attributes[key] = [
process(s.strip()) for s in value.split(',')
]
if missing_keys:
error(u"In entry {0}: missing key(s) {1!s}".format(
entry, missing_keys),
abort=True)
extraneous_keys = set(attributes.keys()) - processed_keys
if extraneous_keys:
warn(u"In entry {0}: unknown key(s) {1!s}. Have you misspelled them?".
format(entry, list(extraneous_keys)))
return sane_attributes
def associate_releases(entries, versions, filename):
for _, attributes in entries.items():
attributes['releases'] = OrderedDict()
prog = re.compile(release_pattern)
warnings = {}
try:
with open(filename) as input:
lines = []
for line in input:
line = line.strip()
result = prog.match(line)
try:
entry, date = result.groups()
except ValueError as ex:
error(u"In file {0}: Malformed release {1}".format(
filename, line.replace),
exception=ex)
else:
if not entry in entries:
if not entry in warnings:
warnings[entry] = [line]
else:
warnings[entry].append(line)
else:
lines.append((entry, date))
for entry, releases in warnings.items():
warn(u"In file {0}: In release(s) {1!s}: Unknown entry {2}".format(
filename, releases, entry))
lines.sort(reverse=True)
for line in lines:
found = False
entry, date = line
for version_number, version_date in versions:
if version_date <= date:
entry_releases = entries[entry]['releases']
if version_number not in entry_releases:
entry_releases[version_number] = []
entry_releases[version_number].append(date)
found = True
break
if not found:
warn(
u"In file {0}: In release {1}: Release date {2} has no matching version"
.format(filename, line, date))
except Exception as ex:
error(u"In file {0}: error".format(filename), exception=ex)
error("Not processing releases")
def parse_name_url(name, entry, key):
if name.find(" and ") != -1:
terminal.warn(
u"In entry {0}: {1} field contains 'and'. Use ',' to separate names."
.format(entry, key))
url_start = name.find('<')
url_end = name.find('>')
if url_start != -1 and url_end != -1:
url = name[url_start + 1:url_end].strip()
if url.startswith("mailto:"):
url = url.replace("@", " /at/ ").replace(".", " /dot/ ")
elif "@" in url:
terminal.warn(
u"In entry {0}: Found mail address without 'mailto:': {1}".
format(entry, url))
url = "mailto:" + url
url = url.replace("@", " /at/ ").replace(".", " /dot/ ")
return name[:url_start].strip(), url
else:
terminal.notice(u"In entry {0}: no URL specified for {1} {2} ".format(
entry, key, name))
return name, None
def main():
usage = "sitegen.py [-h] [--templates TEMPLATES_DIR --dest DEST_DIR] [--status STATUS_FILE] [--deps DEPS_FILE] METADATA_DIR THYS_DIR"
parser = argparse.ArgumentParser(usage=usage)
parser.add_argument("metadata_dir",
metavar="METADATA_DIR",
action="store",
help="metadata location")
parser.add_argument("thys_dir",
metavar="THYS_DIR",
action="store",
help="directory with afp entries")
parser.add_argument("--templates",
action="store",
dest="templates_dir",
help="directory with Jinja2 templates")
parser.add_argument("--dest",
action="store",
dest="dest_dir",
help="destination dir for generated html files")
parser.add_argument("--status",
action="store",
dest="status_file",
help="status file location (devel)")
parser.add_argument("--deps",
action="store",
dest="deps_file",
help="dependencies file location")
parser.parse_args(namespace=options)
options.is_devel = options.status_file is not None
if options.dest_dir and not options.templates_dir:
error("Please specify templates dir", abort=True)
# parse metadata
entries = parse(os.path.join(options.metadata_dir, "metadata"))
versions = read_versions(
os.path.join(options.metadata_dir, "release-dates"))
associate_releases(entries, versions,
os.path.join(options.metadata_dir, "releases"))
if len(entries) == 0:
warn("In metadata: No entries found")
# generate depends-on, used-by entries, lines of code and number of lemmas
# by using an afp_dict object
# TODO: error instead of warn
deps_dict = metadata.empty_deps(entries)
if options.deps_file:
with open(options.deps_file, 'r') as df:
deps_dict = metadata.read_deps(df)
else:
warn("No dependencies file specified")
afp_dict = afpstats.afp_dict(entries, options.thys_dir, deps_dict)
afp_dict.build_stats()
for e in entries:
entries[e]['depends-on'] = list(map(str, afp_dict[e].imports))
entries[e]['used-by'] = list(map(str, afp_dict[e].used))
# perform check
count = check_fs(entries, options.thys_dir)
output = "Checked directory {0}. Found {1} warnings.".format(
options.thys_dir, count)
color = 'yellow' if count > 0 else 'green'
print(colored(output, color, attrs=['bold']))
# perform generation
if options.dest_dir:
if options.status_file is not None:
(build_data, status) = parse_status(options.status_file)
for a in afp_dict:
if a in status:
afp_dict[a].status = status[a]
else:
afp_dict[a].status = "skipped"
else:
build_data = dict()
builder = templates.Builder(options, entries, afp_dict)
builder.generate_topics()
builder.generate_index()
builder.generate_entries()
builder.generate_statistics()
builder.generate_download()
for s in [
"about", "citing", "updating", "search", "submitting", "using"
]:
builder.generate_standard(s + ".html", s + ".tpl")
builder.generate_rss(30)
#TODO: look over it one more time
if options.is_devel:
builder.generate_status(build_data)
