def make_source_for(src, LOCATIONS):
slug = slugify(src['collection']).replace('-', '_')
slug_location = slugify(src['location'])
if is_facebook(src['file_url']):
feed_id = "%s_%s_fb_1" % (
slug,
slug_location,
)
else:
feed_id = "%s_%s_1" % (
slug,
slug_location,
)
result = {
"extractor": "", # depends if feed or not
"keep_index_on_update": True,
"enrichers":
[["ocd_backend.enrichers.AS2TranslationEnricher", {}]
# [
# "ocd_backend.enrichers.NEREnricher",
# {}
# ],
# [
# "ocd_backend.enrichers.BinoasEnricher",
# {}
# ]
],
"file_url": '',
"index_name": slug,
"transformer": "ocd_backend.transformers.BaseTransformer",
"loader": "ocd_backend.loaders.AS2Loader",
"item": "", # html grabber
"cleanup": "ocd_backend.tasks.CleanupElasticsearch",
"location": _normalize_location(src['location'], LOCATIONS),
"hidden": False,
"id": feed_id
}
if not is_facebook(src['file_url']):
additional = get_source_info_from_url(src['file_url'])
for k, v in additional.iteritems():
result[k] = v
else:
result[
"extractor"] = "ocd_backend.extractors.facebook.FacebookExtractor"
result["item"] = "ocd_backend.items.facebook.PageItem"
result["facebook"] = {
'app_secret': os.environ.get('FACEBOOK_APP_SECRET', None),
'app_id': os.environ.get('FACEBOOK_APP_ID', None),
"paging": False,
"api_version": "v2.11",
"graph_url": "%s/posts" % (get_facebook_path(src['file_url']), )
}
for k, v in src.iteritems():
if k != 'file_url':
result[k] = v
return result
def _get_doc_type(self, doc, doc_type_spec):
m = re.match(r'^@([\w_]+)', doc_type_spec)
if m is not None:
doc_field = m.group(1)
return slugify(doc[doc_field], '_')
else:
return doc_type_spec
def __init__(self,
source_id=None,
source=None,
supplier=None,
collection=None,
merge_into=None):
# Set defaults
self.skip_validation = None
self.values = dict()
if merge_into:
if not isinstance(merge_into, tuple) or len(merge_into) != 3:
raise ValueError(
'merge_into requires a tuple with 3 elements: (predicate, column, value)'
)
self.merge_into = merge_into
else:
self.merge_into = None
# https://argu.co/voc/mapping/<organization>/<source>/<source_id_key>/<source_id>
# i.e. https://argu.co/voc/mapping/nl/ggm/vrsnummer/6655476
if source_id:
assert source
assert supplier
assert collection
self.had_primary_source = Uri(
Mapping, '{}/{}/{}/{}'.format(source, supplier, collection,
slugify(source_id)))
def __init__(self, source_id=False, organization=None, source=None, source_id_key=None):
# Set defaults
#self.uri = None
#self.prefix = None
self.skip_validation = None
# self.verbose_name = None
self.values = dict()
# https://argu.co/voc/mapping/<organization>/<source>/<source_id_key>/<source_id>
# i.e. https://argu.co/voc/mapping/nl/ggm/vrsnummer/6655476
if source_id is not False:
assert source_id
assert organization
assert source
assert source_id_key
self.had_primary_source = Uri(
Mapping,
'{}/{}/{}/{}'.format(
organization,
source,
source_id_key,
slugify(source_id)
)
)
self._source = source
def __init__(self,
source_id=None,
source=None,
supplier=None,
collection=None,
merge_into=None,
cached_path=None,
canonical_iri=None):
# Set defaults
self.skip_validation = None
self.values = dict()
self.enricher_task = self.enricher_task
if merge_into:
if not isinstance(merge_into, tuple) or len(merge_into) != 3:
raise ValueError(
'merge_into requires a tuple with 3 elements: (predicate, column, value)'
)
self.merge_into = merge_into
else:
self.merge_into = None
self.canonical_id = source_id
self.cached_path = cached_path
try:
# if canonical_iri is a lambda function
self.canonical_iri = canonical_iri(self)
except TypeError:
self.canonical_iri = canonical_iri
# https://argu.co/voc/mapping/<organization>/<source>/<source_id_key>/<source_id>
# i.e. https://argu.co/voc/mapping/nl/ggm/vrsnummer/6655476
if source_id:
assert source
assert supplier
assert collection
self.source_iri = Uri(
Mapping, '{}/{}/{}/{}'.format(source, supplier, collection,
slugify(source_id)))
def get_object_id(self):
return slugify(unicode(self.original_item['Meetingtype']).strip())
def get_object_id(self):
return slugify(unicode(self.original_item['Meetingtype']).strip())
def convert_party(party):
slug = slugify(party['partij']).replace('-', '_')
slug_location = 'nederland'
# feed_type_defs = {
# 'Feed': {
# "extractor": "ocd_backend.extractors.feed.FeedExtractor",
# "item": "ocd_backend.items.feed.FeedContentFromPageItem",
# 'env': {
#
# }
# },
# 'Facebook': {
# "extractor": "ocd_backend.extractors.facebook.FacebookExtractor",
# "item": "ocd_backend.items.facebook.PageItem",
# 'env': {
# 'app_secret': os.environ.get('FACEBOOK_APP_SECRET', None),
# 'app_id': os.environ.get('FACEBOOK_APP_ID', None),
# "paging": False,
# "api_version": "v2.11",
# "graph_url": "%s/posts" % (
# get_facebook_path(party[feed_type]),)
# }
# }
# }
feed_id = "%s_%s_1" % (
slug,
slug_location,
)
result = {
"extractor": "", # depends if feed or not
"keep_index_on_update": True,
"enrichers": [
# [
# "ocd_backend.enrichers.NEREnricher",
# {}
# ],
# [
# "ocd_backend.enrichers.BinoasEnricher",
# {}
# ]
],
"file_url": party['website'],
"index_name": slug,
"transformer": "ocd_backend.transformers.BaseTransformer",
"collection": party['partij'],
"loader": "ocd_backend.loaders.ElasticsearchLoader",
"item": "", # html grabber
"cleanup": "ocd_backend.tasks.CleanupElasticsearch",
"location": 'Nederland',
"hidden": False,
"id": feed_id
}
if party['feed'] != '':
result['extractor'] = "ocd_backend.extractors.feed.FeedExtractor"
result['item'] = "ocd_backend.items.feed.FeedContentFromPageItem"
else:
result[
'extractor'] = "ocd_backend.extractors.staticfile.StaticHtmlExtractor"
result['item_xpath'] = ''
return result
def convert_party(party, feed_type, locations):
slug = slugify(party['Partij']).replace('-', '_')
slug_location = slugify(party['RegioNaam']).replace('-', '_')
feed_type_defs = {
'Feed': {
"extractor": "ocd_backend.extractors.feed.FeedExtractor",
"item": "ocd_backend.items.feed.FeedContentFromPageItem",
'env': {}
},
'Facebook': {
"extractor": "ocd_backend.extractors.facebook.FacebookExtractor",
"item": "ocd_backend.items.facebook.PageItem",
'env': {
'app_secret': os.environ.get('FACEBOOK_APP_SECRET', None),
'app_id': os.environ.get('FACEBOOK_APP_ID', None),
"paging": False,
"api_version": "v2.11",
"graph_url":
"%s/posts" % (get_facebook_path(party[feed_type]), )
}
}
}
# whoops
if feed_type != 'Feed':
feed_id = "%s_%s_1" % (
slug,
slug_location,
)
else:
feed_id = "%s_%s_fb_1" % (
slug,
slug_location,
)
result = {
"extractor":
feed_type_defs[feed_type]['extractor'],
"keep_index_on_update":
True,
"enrichers": [["ocd_backend.enrichers.NEREnricher", {}],
["ocd_backend.enrichers.BinoasEnricher", {}]],
feed_type.lower(): {},
"file_url":
party[feed_type],
"index_name":
slug,
"transformer":
"ocd_backend.transformers.BaseTransformer",
"collection":
party['Partij'],
"loader":
"ocd_backend.loaders.ElasticsearchLoader",
"item":
feed_type_defs[feed_type]['item'],
"cleanup":
"ocd_backend.tasks.CleanupElasticsearch",
"location":
_normalize_location(party['RegioNaam'], locations),
"hidden":
False,
"id":
feed_id
}
for k, v in feed_type_defs[feed_type]['env'].iteritems():
result[feed_type.lower()][k] = v
return result
def get_object_id(self):
return slugify(unicode(self.original_item["name"]).strip())