def createLinkDatum(parent_url, url):
"""Create Link CrawlDbDatum mock data."""
key = ndb.Key(CrawlDbDatum, parent_url)
CrawlDbDatum.get_or_insert(url,
parent=key,
url=url,
last_status=pipelines.UNFETCHED)
def createMockFetchedDatum(url, html_text, status):
"""Create FetchedDatum mock data."""
key = ndb.Key(CrawlDbDatum, url)
crawl = CrawlDbDatum.get_or_insert(url, parent=key,
url=url, last_status=status)
if status != pipelines.UNFETCHED:
fetched_datum = FetchedDbDatum(parent=crawl.key,
url=url, fetched_url = url,
fetched_content = html_text, content_type="text/html")
fetched_datum.put()
def createMockFetchedDatum(url, html_text, status):
"""Create FetchedDatum mock data."""
key = ndb.Key(CrawlDbDatum, url)
crawl = CrawlDbDatum.get_or_insert(url,
parent=key,
url=url,
last_status=status)
if status != pipelines.UNFETCHED:
fetched_datum = FetchedDbDatum(parent=crawl.key,
url=url,
fetched_url=url,
fetched_content=html_text,
content_type="text/html")
fetched_datum.put()
def createMockCrawlDbDatum(url):
"""Create CrawlDbDatum mock data."""
CrawlDbDatum.get_or_insert(url,
url=url, last_status=pipelines.UNFETCHED)
def createMockCrawlDbDatum(url):
"""Create CrawlDbDatum mock data."""
CrawlDbDatum.get_or_insert(url, url=url, last_status=pipelines.UNFETCHED)
def createLinkDatum(parent_url, url):
"""Create Link CrawlDbDatum mock data."""
key = ndb.Key(CrawlDbDatum, parent_url)
CrawlDbDatum.get_or_insert(url, parent=key,
url=url, last_status=pipelines.UNFETCHED)
