import click
from elasticsearch import helpers, exceptions, logger as elastic_logger
from lxml import etree
from parse import parse
from msc_pygeoapi.env import (
MSC_PYGEOAPI_BASEPATH,
MSC_PYGEOAPI_ES_TIMEOUT,
MSC_PYGEOAPI_ES_URL,
MSC_PYGEOAPI_ES_AUTH,
)
from msc_pygeoapi.loader.base import BaseLoader
from msc_pygeoapi.util import get_es, json_pretty_print, strftime_rfc3339
LOGGER = logging.getLogger(__name__)
elastic_logger.setLevel(logging.WARNING)
# index settings
INDEX_NAME = 'marine_weather_{}'
MAPPINGS = {
'regular-forecasts': {
'issued_datetime_utc': {
'type': 'date',
'format': 'date_time_no_millis',
'ignore_malformed': False,
},
'issued_datetime_local': {
'type': 'date',
'format': 'date_time_no_millis',
'ignore_malformed': False,
import click
from elasticsearch import logger as elastic_logger
from parse import parse
from msc_pygeoapi import cli_options
from msc_pygeoapi.connector.elasticsearch_ import ElasticsearchConnector
from msc_pygeoapi.env import (MSC_PYGEOAPI_LOGGING_LOGLEVEL)
from msc_pygeoapi.loader.base import BaseLoader
from msc_pygeoapi.util import (
configure_es_connection,
check_es_indexes_to_delete,
)
LOGGER = logging.getLogger(__name__)
elastic_logger.setLevel(getattr(logging, MSC_PYGEOAPI_LOGGING_LOGLEVEL))
# cleanup settings
DAYS_TO_KEEP = 3
# index settings
INDEX_BASENAME = 'aqhi-{}-realtime.'
MAPPINGS = {
'forecasts': {
'properties': {
'geometry': {
'type': 'geo_shape'
},
'properties': {
'properties': {
from string import Template
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import sessionmaker
from dateutil.parser import parse as dateparse
# settings.py
from dotenv import load_dotenv
from pathlib import Path
from elasticsearch import Elasticsearch
from elasticsearch import logger as es_logger
es_logger.setLevel(50)
es_user = os.getenv("ES_USER")
es_pwd = os.getenv("ES_PWD")
es = Elasticsearch(http_auth=(es_user, es_pwd))
env_path = Path('..')/'.env'
load_dotenv(dotenv_path=env_path)
DB_DATABASE = os.getenv("DB_DATABASE")
DB_USERNAME = os.getenv("DB_USERNAME")
DB_PASSWORD = os.getenv("DB_PASSWORD")
# engine = create_engine("mysql+pymysql://"+DB_USERNAME+":"+DB_PASSWORD+"@localhost/Game?charset=utf8", encoding='utf-8', echo=False)
engine = create_engine("mysql+pymysql://"+DB_USERNAME+":"+DB_PASSWORD +
"@localhost/"+DB_DATABASE+"?charset=utf8", encoding='utf-8', echo=False)
Base = declarative_base()
import logging
from copy import deepcopy
from typing import Optional, Dict, Union, Any, Iterable
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
from elasticsearch import logger as es_logger
__all__ = ["ElasticSearchIndexer"]
from forte.common.configuration import Config
# It seems that elastic search shows a lot of debug message, sometimes can
# suffocate travis. Trying to depress the logging.
es_logger.setLevel(logging.INFO)
class ElasticSearchIndexer:
r"""Indexer class for `Elasticsearch`."""
def __init__(self, config: Optional[Union[Dict, Config]] = None):
super().__init__()
self._config = Config(config, self.default_configs())
self.elasticsearch = Elasticsearch(hosts=self._config.hosts)
def index(
self,
document: Dict[str, Any],
index_name: Optional[str] = None,
refresh: Optional[Union[bool, str]] = False,
) -> None:
