def fetch_new_drafts(since: datetime) -> List[IETF_URI]:
"""Fetch all new drafts since time 'since'"""
trk = datatracker.DataTracker()
draft_itr = trk.documents(
since=since.isoformat(timespec="seconds"),
doctype=trk.document_type(
datatracker.DocumentTypeURI("/api/v1/name/doctypename/draft/")))
urls = []
for draft in draft_itr:
for uri in draft.submissions:
submission = trk.submission(uri)
if not submission:
break
urls += [
IETF_URI(submission.name,
_extn,
rev=submission.rev,
dtype="draft",
url=_url) for _extn, _url in submission.urls()
if _extn in valid_extns
]
return urls
def test_dload_single_draft_allversions(self):
rootdir = pathlib.Path(
tempfile.mkdtemp(dir=pathlib.Path().cwd())).resolve()
draft_name = 'draft-mcquistin-augmented-ascii-diagrams'
argv = f"-d {str(rootdir)} {draft_name}".split()
dt = datatracker.DataTracker()
draft = dt.document_from_draft(draft_name)
self.assertIsNotNone(draft)
files = []
if draft is None:
return
for uri in draft.submissions:
submission = dt.submission(uri)
self.assertIsNotNone(submission)
if submission is None:
continue
files += [
pathlib.Path(_url).name for _ext, _url in submission.urls()
if _ext in ['.xml', '.txt']
]
opts = npt.util.read_usr_opts(argv)
for d in [pathlib.Path(_f) for _f in files]:
name, rev = '-'.join(
d.stem.split(sep='-')[:-1]), d.stem.split(sep='-')[-1]
_dl_file = rootdir / "draft" / name / rev / d
self.assertTrue(_dl_file.exists(),
msg=f"{_dl_file} does not exist")
self.assertTrue(_dl_file.is_file(),
msg=f"{_dl_file} is not a file")
if rootdir.exists():
shutil.rmtree(rootdir)
def _query_datatracker(self, doctype, name, rev, extn):
assert doctype in [
'draft', 'rfc'
], f"Could not resolve document type {doctype}.Only draft,rfc supported."
urls = []
if doctype == 'draft':
dt = datatracker.DataTracker()
draft = dt.document_from_draft(name)
assert draft != None, f"Could not resolve remote draft -- name = {name} , rev = {rev}, extension = {extn}"
if draft is not None:
for uri in draft.submissions:
submission = dt.submission(uri)
if not submission:
continue
if rev and rev != submission.rev:
continue
if extn == None:
urls += [
IETF_URI(submission.name,
_ext,
rev=submission.rev,
dtype="draft",
url=_url)
for _ext, _url in submission.urls()
if _ext in valid_extns
]
elif extn in valid_extns:
for _sub_extn, _sub_url in submission.urls():
if _sub_extn != extn:
continue
urls.append(
IETF_URI(submission.name,
extn,
rev=submission.rev,
dtype="draft",
url=_sub_url))
elif doctype == 'rfc':
ri = rfcindex.RFCIndex()
rfc = ri.rfc(name.upper())
assert rfc is not None, f"Invalid rfc -- {name}"
rfc_txt_label = lambda fmt: ("ASCII" if "ASCII" in fmt else "TEXT")
extn_rfc_convert = lambda _ext: rfc_txt_label(
rfc.formats) if _ext == ".txt" else _ext[1:].upper()
rfc_extn_convert = lambda _ext: ".txt" if _ext in [
"ASCII", "TEXT"
] else f".{_ext.lower()}"
rfc_extensions = [rfc_extn_convert(_ext) for _ext in rfc.formats]
dt_extns = [_ext for _ext in rfc_extensions if _ext in valid_extns]
if extn:
assert extn in dt_extns, f"File format extn of {name}{extn} not amongst {dt_extns}"
urls.append(
IETF_URI(name,
extn,
rev=None,
dtype="rfc",
url=rfc.content_url(extn_rfc_convert(extn))))
else:
urls += [
IETF_URI(name,
_extn,
rev=None,
dtype="rfc",
url=rfc.content_url(extn_rfc_convert(_extn)))
for _extn in dt_extns
]
return urls
def __init__(self, use_cache=False):
self.dt = DT.DataTracker()
self.conn = sqlite3.connect('ietfdata.db')
self.create_tables()
if use_cache:
requests_cache.install_cache("dt_collector_cache")
import os
import sys
sys.path.insert(0,
os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
from pathlib import Path
import ietfdata.datatracker as dt
import ietfdata.mailarchive as mailarchive
from ietfdata.mailhelper_headerdata import *
from ietfdata.mailhelper_datatracker import *
datatracker = dt.DataTracker(cache_dir=Path("cache"))
archive = mailarchive.MailArchive(
cache_dir=Path("cache"),
helpers=[HeaderDataMailHelper(),
DatatrackerMailHelper()])
def pretty_print_message_metadata(msg: mailarchive.MailingListMessage):
subject = msg.message["Subject"].replace('\n', "\\n")
string = f"{msg.metadata('from_name'):50s} | {msg.metadata('from_addr'):30s} | {str(msg.metadata('from_person').id) if msg.metadata('from_person') is not None else '--':6s} | {msg.metadata('timestamp'):%Y-%m-%d %H:%M} | {subject:30s}"
for document in msg.metadata("related_docs"):
name = document.name
if document.rfc is not None:
name = f"RFC{document.rfc}"
string += f"\n\tRelated Document: {document.title} ({name})"
return string