def main(): p = build_arg_parser() cfg = p.parse_args() logger = setup_logging(cfg.doc_type) if cfg.dump_config: logger.info('Running index_ccdb with') logger.info(p.format_values()) index_alias = cfg.index_name index_name = "{}-v1".format(index_alias) backup_index_name = "{}-v2".format(index_alias) logger.info("Creating Elasticsearch Connection") if cfg.is_aws_host: es = get_aws_es_connection(cfg) logger.info('AWS configured as Elasticsearch host') else: es = get_es_connection(cfg) qas_timestamp = get_qa_timestamp(cfg, logger) logger.info("Begin indexing data in Elasticsearch") index_json_data(es, logger, cfg.doc_type, cfg.settings, cfg.mapping, cfg.dataset, index_name, backup_index_name, index_alias, qas_timestamp=qas_timestamp)
def main(): p = build_arg_parser() cfg = p.parse_args() logger = setup_logging('taxonomy') if cfg.dump_config: logger.info('Running index_taxonomy with') logger.info(p.format_values()) es = get_es_connection(cfg) logger.info("Begin indexing taxonomy data in Elasticsearch") index_taxonomy(es, logger, cfg.taxonomy, cfg.index_name)
def test_get_es_connection(self, mock_es): options = make_configargs({ 'es_host': 'www.example.org', 'es_port': '9222', 'es_username': '******', 'es_password': '******', }) mock_es.return_value = 'passed' actual = sut.get_es_connection(options) self.assertEqual(actual, 'passed') mock_es.assert_called_once_with('http://www.example.org:9222', http_auth=('king', 'kong'), timeout=1000, user_ssl=True)
default='complaint', help='Elasticsearch document type') group = p.add_argument_group('Files') group.add('--dataset', dest='dataset', required=True, help="Complaint data in NDJSON format") return p if __name__ == '__main__': p = build_arg_parser() cfg = p.parse_args() logger = setup_logging(cfg.doc_type) if cfg.dump_config: logger.info('Running index_ccdb with') logger.info(p.format_values()) index_alias = cfg.index_name index_name = "{}-v1".format(index_alias) backup_index_name = "{}-v2".format(index_alias) logger.info("Creating Elasticsearch Connection") es = get_es_connection(cfg) logger.info("Begin indexing data in Elasticsearch") index_json_data(es, logger, cfg.doc_type, cfg.settings, cfg.mapping, cfg.dataset, index_name, backup_index_name, index_alias)