def __init__(self, options): self.options = options self.imported = set() self.layers = [] settings = {} with open(".build/config.yaml") as f: settings = yaml.load(f) self.languages = settings["available_locale_names"] exluded_themes_string = settings["excluded_themes_from_search"] exluded_themes = [] if exluded_themes_string is not None: exluded_themes = exluded_themes_string.split(",") # must be done only once we have loaded the project config from c2cgeoportal.models import DBSession, Interface, Theme, Role self.session = DBSession() self._ = {} self.metadata_service_url = \ 'http://shop.geoportail.lu/Portail/inspire/webservices/getMD.jsp' registry = bootstrap(self.options.app_config)['registry'] request = bootstrap(self.options.app_config)['request'] self.es_layer_index = get_index(request) + '_layers' self.tdirs = registry.queryUtility(ITranslationDirectories, default=[]) self.tsf = TranslationStringFactory('geoportailv3-client') self.interfaces = self.session.query(Interface).filter( Interface.name.in_(options.interfaces)).all() self.public_theme = {} self.public_group = {} for interface in self.interfaces: self.public_theme[interface.id] = [] self.public_group[interface.id] = [] for theme in self.session.query(Theme).filter_by(public=True).all(): if theme.name not in exluded_themes: self._add_theme(theme) for role in self.session.query(Role).all(): for theme in self.session.query(Theme).all(): if theme.name not in exluded_themes: self._add_theme(theme, role) ensure_index(get_elasticsearch(request), self.es_layer_index, options.recreate_index) try: helpers.bulk(actions=self.layers, client=get_elasticsearch(request), raise_on_error=True) except (BulkIndexError, ConnectionTimeout) as e: statuslog("\n %s" % e)
def __init__(self, options): self.options = options self.imported = set() self.layers = [] settings = {} with open(".build/config.yaml") as f: settings = yaml.load(f) self.languages = settings["available_locale_names"] # must be done only once we have loaded the project config from c2cgeoportal.models import DBSession, Interface, Theme, Role self.session = DBSession() self._ = {} self.metadata_service_url = \ 'http://shop.geoportail.lu/Portail/inspire/webservices/getMD.jsp' registry = bootstrap(self.options.app_config)['registry'] request = bootstrap(self.options.app_config)['request'] self.es_layer_index = get_index(request) + '_layers' self.tdirs = registry.queryUtility(ITranslationDirectories, default=[]) self.tsf = TranslationStringFactory('geoportailv3-client') self.interfaces = self.session.query(Interface).filter( Interface.name.in_(options.interfaces) ).all() self.public_theme = {} self.public_group = {} for interface in self.interfaces: self.public_theme[interface.id] = [] self.public_group[interface.id] = [] for theme in self.session.query(Theme).filter_by(public=True).all(): self._add_theme(theme) for role in self.session.query(Role).all(): for theme in self.session.query(Theme).all(): self._add_theme(theme, role) ensure_index( get_elasticsearch(request), self.es_layer_index, options.recreate_index ) try: helpers.bulk(actions=self.layers, client=get_elasticsearch(request), raise_on_error=True) except (BulkIndexError, ConnectionTimeout) as e: statuslog("\n %s" % e)
def main(): env = bootstrap('development.ini') request = env['request'] try: opts, args = getopt.getopt(sys.argv[1:], 'ri', ['reset', 'index']) except getopt.GetoptError as err: print str(err) sys.exit(2) index = False reset = False for o, a in opts: if o in ('-r', '--reset'): statuslog('\rResetting Index') reset = True if o in ('-i', '--index'): statuslog('\rChecking Index') index = True import time index_name = get_index(request) + '_' + time.strftime("%Y%m%d") ensure_index(get_elasticsearch(request), index_name, reset) if index is True: statuslog("\rCreating Database Query ") c = get_cursor() counter = 1 while True: multiple = 250 results = c.fetchmany(multiple) doc_list = [] for result in results: doc = update_document(get_index(request), 'poi', result['id'], result) doc_list.append(doc) statuslog("\rIndexed Elements: %i" % int(counter)) counter = counter + 1 try: helpers.bulk(client=get_elasticsearch(request), actions=doc_list, chunk_size=multiple, raise_on_error=True) except (BulkIndexError, ConnectionTimeout) as e: print "\n %s" % e if not results: statuslog("\n") break
def main(): env = bootstrap('development.ini') request = env['request'] try: opts, args = getopt.getopt(sys.argv[1:], '', ['recreate']) except getopt.GetoptError as err: print str(err) sys.exit(2) recreate = False for o, a in opts: if o == '--recreate': recreate = True ensure_index(get_elasticsearch(request), get_index(request), recreate) statuslog("\rCreating Database Query ") c = get_cursor() counter = 1 while True: multiple = 250 results = c.fetchmany(multiple) doc_list = [] for result in results: doc = update_document(get_index(request), 'poi', result['id'], result) doc_list.append(doc) statuslog("\rIndexed Elements: %i" % int(counter)) counter = counter + 1 try: helpers.bulk(client=get_elasticsearch(request), actions=doc_list, chunk_size=multiple, raise_on_error=True) except (BulkIndexError, ConnectionTimeout) as e: print "\n %s" % e if not results: statuslog("\n") break