def _to_datastore_map(kv_content):
"""Map function of store to datastore.
The kind of entity is defined in cooperate.models
Args:
content_csv: The csv format string of exported contents.
Returns:
url: The url string, that will key of inserted entity to datastore.
"""
k, v = kv_content
parsed_csv = v.split(",")
url = ""
title = ""
content = ""
try:
url = parsed_csv[0]
title = parsed_csv[1]
content = parsed_csv[2]
except Exception as e:
logging.warning("Can't parse csv:" + v + ":" + e.message)
ContentsDatum.insert_or_fail(
url,
parent=ndb.Key(ContentsDatum, url),
url=url,
title=title,
content=content,
description=None)
yield url
def testSuccessfulRun(self):
p = cooperate.ImportCloudStoragePipeline("ImportCloudStoragePipeline",
gs_bucket_name=TEST_BUCKET_NAME,
shards=3)
p.start()
test_support.execute_until_empty(self.taskqueue)
cooperate.ImportCloudStoragePipeline.from_id(p.pipeline_id)
# Fetch from
entities = ContentsDatum.query().fetch()
self.assertTrue(len(entities))
def testSuccessfulRun(self):
p = cooperate.ImportCloudStoragePipeline(
"ImportCloudStoragePipeline",
gs_bucket_name=TEST_BUCKET_NAME,
shards=3)
p.start()
test_support.execute_until_empty(self.taskqueue)
cooperate.ImportCloudStoragePipeline.from_id(p.pipeline_id)
# Fetch from
entities = ContentsDatum.query().fetch()
self.assertTrue(len(entities))