def main():
p = build_arg_parser()
cfg = p.parse_args()
logger = setup_logging(cfg.doc_type)
if cfg.dump_config:
logger.info('Running index_ccdb with')
logger.info(p.format_values())
index_alias = cfg.index_name
index_name = "{}-v1".format(index_alias)
backup_index_name = "{}-v2".format(index_alias)
logger.info("Creating Elasticsearch Connection")
if cfg.is_aws_host:
es = get_aws_es_connection(cfg)
logger.info('AWS configured as Elasticsearch host')
else:
es = get_es_connection(cfg)
qas_timestamp = get_qa_timestamp(cfg, logger)
logger.info("Begin indexing data in Elasticsearch")
index_json_data(es,
logger,
cfg.doc_type,
cfg.settings,
cfg.mapping,
cfg.dataset,
index_name,
backup_index_name,
index_alias,
qas_timestamp=qas_timestamp)
def main():
p = build_arg_parser()
cfg = p.parse_args()
logger = setup_logging('taxonomy')
if cfg.dump_config:
logger.info('Running index_taxonomy with')
logger.info(p.format_values())
es = get_es_connection(cfg)
logger.info("Begin indexing taxonomy data in Elasticsearch")
index_taxonomy(es, logger, cfg.taxonomy, cfg.index_name)
def test_get_es_connection(self, mock_es):
options = make_configargs({
'es_host': 'www.example.org',
'es_port': '9222',
'es_username': '******',
'es_password': '******',
})
mock_es.return_value = 'passed'
actual = sut.get_es_connection(options)
self.assertEqual(actual, 'passed')
mock_es.assert_called_once_with('http://www.example.org:9222',
http_auth=('king', 'kong'),
timeout=1000,
user_ssl=True)
default='complaint',
help='Elasticsearch document type')
group = p.add_argument_group('Files')
group.add('--dataset',
dest='dataset',
required=True,
help="Complaint data in NDJSON format")
return p
if __name__ == '__main__':
p = build_arg_parser()
cfg = p.parse_args()
logger = setup_logging(cfg.doc_type)
if cfg.dump_config:
logger.info('Running index_ccdb with')
logger.info(p.format_values())
index_alias = cfg.index_name
index_name = "{}-v1".format(index_alias)
backup_index_name = "{}-v2".format(index_alias)
logger.info("Creating Elasticsearch Connection")
es = get_es_connection(cfg)
logger.info("Begin indexing data in Elasticsearch")
index_json_data(es, logger, cfg.doc_type, cfg.settings, cfg.mapping,
cfg.dataset, index_name, backup_index_name, index_alias)