def __init__(self, options):
self.options = options
self.imported = set()
self.layers = []
settings = {}
with open(".build/config.yaml") as f:
settings = yaml.load(f)
self.languages = settings["available_locale_names"]
exluded_themes_string = settings["excluded_themes_from_search"]
exluded_themes = []
if exluded_themes_string is not None:
exluded_themes = exluded_themes_string.split(",")
# must be done only once we have loaded the project config
from c2cgeoportal.models import DBSession, Interface, Theme, Role
self.session = DBSession()
self._ = {}
self.metadata_service_url = \
'http://shop.geoportail.lu/Portail/inspire/webservices/getMD.jsp'
registry = bootstrap(self.options.app_config)['registry']
request = bootstrap(self.options.app_config)['request']
self.es_layer_index = get_index(request) + '_layers'
self.tdirs = registry.queryUtility(ITranslationDirectories, default=[])
self.tsf = TranslationStringFactory('geoportailv3-client')
self.interfaces = self.session.query(Interface).filter(
Interface.name.in_(options.interfaces)).all()
self.public_theme = {}
self.public_group = {}
for interface in self.interfaces:
self.public_theme[interface.id] = []
self.public_group[interface.id] = []
for theme in self.session.query(Theme).filter_by(public=True).all():
if theme.name not in exluded_themes:
self._add_theme(theme)
for role in self.session.query(Role).all():
for theme in self.session.query(Theme).all():
if theme.name not in exluded_themes:
self._add_theme(theme, role)
ensure_index(get_elasticsearch(request), self.es_layer_index,
options.recreate_index)
try:
helpers.bulk(actions=self.layers,
client=get_elasticsearch(request),
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
statuslog("\n %s" % e)
def __init__(self, options):
self.options = options
self.imported = set()
self.layers = []
settings = {}
with open(".build/config.yaml") as f:
settings = yaml.load(f)
self.languages = settings["available_locale_names"]
# must be done only once we have loaded the project config
from c2cgeoportal.models import DBSession, Interface, Theme, Role
self.session = DBSession()
self._ = {}
self.metadata_service_url = \
'http://shop.geoportail.lu/Portail/inspire/webservices/getMD.jsp'
registry = bootstrap(self.options.app_config)['registry']
request = bootstrap(self.options.app_config)['request']
self.es_layer_index = get_index(request) + '_layers'
self.tdirs = registry.queryUtility(ITranslationDirectories, default=[])
self.tsf = TranslationStringFactory('geoportailv3-client')
self.interfaces = self.session.query(Interface).filter(
Interface.name.in_(options.interfaces)
).all()
self.public_theme = {}
self.public_group = {}
for interface in self.interfaces:
self.public_theme[interface.id] = []
self.public_group[interface.id] = []
for theme in self.session.query(Theme).filter_by(public=True).all():
self._add_theme(theme)
for role in self.session.query(Role).all():
for theme in self.session.query(Theme).all():
self._add_theme(theme, role)
ensure_index(
get_elasticsearch(request),
self.es_layer_index,
options.recreate_index
)
try:
helpers.bulk(actions=self.layers,
client=get_elasticsearch(request),
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
statuslog("\n %s" % e)
def main():
env = bootstrap('development.ini')
request = env['request']
try:
opts, args = getopt.getopt(sys.argv[1:], 'ri', ['reset', 'index'])
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
index = False
reset = False
for o, a in opts:
if o in ('-r', '--reset'):
statuslog('\rResetting Index')
reset = True
if o in ('-i', '--index'):
statuslog('\rChecking Index')
index = True
import time
index_name = get_index(request) + '_' + time.strftime("%Y%m%d")
ensure_index(get_elasticsearch(request), index_name, reset)
if index is True:
statuslog("\rCreating Database Query ")
c = get_cursor()
counter = 1
while True:
multiple = 250
results = c.fetchmany(multiple)
doc_list = []
for result in results:
doc = update_document(get_index(request),
'poi',
result['id'],
result)
doc_list.append(doc)
statuslog("\rIndexed Elements: %i" % int(counter))
counter = counter + 1
try:
helpers.bulk(client=get_elasticsearch(request),
actions=doc_list,
chunk_size=multiple,
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
print "\n %s" % e
if not results:
statuslog("\n")
break
def main():
env = bootstrap('development.ini')
request = env['request']
try:
opts, args = getopt.getopt(sys.argv[1:], 'ri', ['reset', 'index'])
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
index = False
reset = False
for o, a in opts:
if o in ('-r', '--reset'):
statuslog('\rResetting Index')
reset = True
if o in ('-i', '--index'):
statuslog('\rChecking Index')
index = True
import time
index_name = get_index(request) + '_' + time.strftime("%Y%m%d")
ensure_index(get_elasticsearch(request), index_name, reset)
if index is True:
statuslog("\rCreating Database Query ")
c = get_cursor()
counter = 1
while True:
multiple = 250
results = c.fetchmany(multiple)
doc_list = []
for result in results:
doc = update_document(get_index(request), 'poi', result['id'],
result)
doc_list.append(doc)
statuslog("\rIndexed Elements: %i" % int(counter))
counter = counter + 1
try:
helpers.bulk(client=get_elasticsearch(request),
actions=doc_list,
chunk_size=multiple,
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
print "\n %s" % e
if not results:
statuslog("\n")
break
def main():
env = bootstrap('development.ini')
request = env['request']
try:
opts, args = getopt.getopt(sys.argv[1:], '', ['recreate'])
except getopt.GetoptError as err:
print str(err)
sys.exit(2)
recreate = False
for o, a in opts:
if o == '--recreate':
recreate = True
ensure_index(get_elasticsearch(request), get_index(request), recreate)
statuslog("\rCreating Database Query ")
c = get_cursor()
counter = 1
while True:
multiple = 250
results = c.fetchmany(multiple)
doc_list = []
for result in results:
doc = update_document(get_index(request),
'poi',
result['id'],
result)
doc_list.append(doc)
statuslog("\rIndexed Elements: %i" % int(counter))
counter = counter + 1
try:
helpers.bulk(client=get_elasticsearch(request),
actions=doc_list,
chunk_size=multiple,
raise_on_error=True)
except (BulkIndexError, ConnectionTimeout) as e:
print "\n %s" % e
if not results:
statuslog("\n")
break
