def put(self, source_name):
helper = RequestHelper(self)
source_name = urllib.unquote(source_name)
success, values = read_json_fields(helper, "url", "active", logger=logging)
if not success:
return
url, active = values
# a Feed must be sole owner of an ArtifactSource;
# fails if source already exists and is already linked to a feed
source = ArtifactSourceAccessor.get_by_name(source_name, return_none=True)
if source:
source_feed_key = Feed.get_by_source(source, keys_only=True, return_none=True)
if source_feed_key:
msg = "source '%s' is referenced by feed %s" % (source_name, source_feed_key.name())
helper.error(409, msg)
else:
source = ArtifactSourceAccessor.create(source_name)
# creates UrlResource if necessary
resource = UrlResourceAccessor.get_by_url(url, return_none=True)
if not resource:
resource = UrlResourceAccessor.create(url)
# create or update Feed
feed = Feed.get_by_source_name(source_name, return_none=True)
if feed:
feed.artifact_source = source
feed.url_resource = resource
feed.put()
else:
Feed.create(source_name, artifact_source=source, url=url, url_resource=resource, active=bool(active))
helper.set_status(204)
def ingest_feed_entries(feed, user, error_call=None):
"""
yields:
(artifact guid, entry) tuple
"""
# TODO: use etag from previous ingest
for entry in feeds.generate_feed_entries(feed.url):
try:
stripped_content = entry.get("stripped_content")
if stripped_content:
# ensures this is a non-empty entry
link = entry.get("link")
raw_modified = entry.get("modified")
if raw_modified:
modified = datetime(*raw_modified[0:-2])
else:
modified = None
logging.debug("%s modified %s (%s)" %
(link, modified, modified.__class__))
url_resource = UrlResourceAccessor.get_or_create(
link, source_modified=modified, feed=feed)
# TODO: check if there is already an artifact for this resource
info_key, content_key, source_key, created = ArtifactAccessor.find_or_create(
source=feed.artifact_source.name,
content_type="text/plain",
body=stripped_content,
url=link,
url_resource=url_resource,
modified_by=user)
yield info_key.name(), entry, created
except Exception, e:
if error_call:
error_call(entry, e)
else:
raise e
def ingest_feed_entries(feed, user, error_call=None):
"""
yields:
(artifact guid, entry) tuple
"""
# TODO: use etag from previous ingest
for entry in feeds.generate_feed_entries(feed.url):
try:
stripped_content = entry.get("stripped_content")
if stripped_content:
# ensures this is a non-empty entry
link = entry.get("link")
raw_modified = entry.get("modified")
if raw_modified:
modified = datetime(*raw_modified[0:-2])
else:
modified = None
logging.debug("%s modified %s (%s)" % (link, modified, modified.__class__))
url_resource = UrlResourceAccessor.get_or_create(link,
source_modified=modified,
feed=feed)
# TODO: check if there is already an artifact for this resource
info_key, content_key, source_key, created = ArtifactAccessor.find_or_create(source=feed.artifact_source.name,
content_type="text/plain",
body=stripped_content,
url = link,
url_resource=url_resource,
modified_by=user)
yield info_key.name(), entry, created
except Exception, e:
if error_call:
error_call(entry, e)
else:
raise e