def add_filters_item(filters, es_url, es_in_index, es_out_index): elastic_in = ElasticSearch(es_url, es_in_index) elastic_out = ElasticSearch(es_url, es_out_index) # Time to just copy from in_index to our_index total = elastic_out.bulk_upload_sync(fetch(elastic_in, filters), "uuid")
ARGS = get_params() if ARGS.debug: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s') logging.debug("Debug mode activated") else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') logging.info("Importing items from %s to %s/%s", ARGS.collection, ARGS.elastic_url, ARGS.index) elastic = ElasticSearch(ARGS.elastic_url, ARGS.index) if ARGS.collection: mongo_items = fetch_mongodb_collection(ARGS.collection, ARGS.mongo_host, ARGS.mongo_port) elif ARGS.project: mongo_items = fetch_mongodb_project(ARGS.project, ARGS.mongo_host, ARGS.mongo_port) elif ARGS.all_collections: mongo_items = fetch_mongodb_all(ARGS.mongo_host, ARGS.mongo_port) else: raise RuntimeError('Collection to be processed not provided') if mongo_items: logging.info("Loading collections in Elasticsearch") elastic.bulk_upload_sync(mongo_items, "id")
help="Get metrics for data source") return parser.parse_args() if __name__ == '__main__': args = get_params() if args.debug: logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(message)s') logging.debug("Debug mode activated") else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') logging.getLogger("urllib3").setLevel(logging.WARNING) logging.getLogger("requests").setLevel(logging.WARNING) data_source = args.data_source index = "grimoirelab_metrics" elastic = ElasticSearch(args.elastic_url, index) if args.elastic_metrics_url: elastic = ElasticSearch(args.elastic_metrics_url, index) elastic.bulk_upload_sync(fetch_metric(args.elastic_url, data_source), "id") # for metric in fetch_metric(es_url, data_source): # print(metric)
def export_items(elastic_url, in_index, out_index, elastic_url_out=None, search_after=False, search_after_value=None, limit=None, copy=False): """ Export items from in_index to out_index using the correct mapping """ if not limit: limit = DEFAULT_LIMIT if search_after_value: search_after_value_timestamp = int(search_after_value[0]) search_after_value_uuid = search_after_value[1] search_after_value = [ search_after_value_timestamp, search_after_value_uuid ] logging.info("Exporting items from %s/%s to %s", elastic_url, in_index, out_index) count_res = requests.get('%s/%s/_count' % (elastic_url, in_index)) try: count_res.raise_for_status() except requests.exceptions.HTTPError: if count_res.status_code == 404: logging.error("The index does not exists: %s", in_index) else: logging.error(count_res.text) sys.exit(1) logging.info("Total items to copy: %i", count_res.json()['count']) # Time to upload the items with the correct mapping elastic_in = ElasticSearch(elastic_url, in_index) if not copy: # Create the correct mapping for the data sources detected from in_index ds_mapping = find_mapping(elastic_url, in_index) else: logging.debug('Using the input index mapping') ds_mapping = extract_mapping(elastic_url, in_index) if not elastic_url_out: elastic_out = ElasticSearch(elastic_url, out_index, mappings=ds_mapping) else: elastic_out = ElasticSearch(elastic_url_out, out_index, mappings=ds_mapping) # Time to just copy from in_index to our_index uid_field = find_uuid(elastic_url, in_index) backend = find_perceval_backend(elastic_url, in_index) if search_after: total = elastic_out.bulk_upload_sync( fetch(elastic_in, backend, limit, search_after_value, scroll=False), uid_field) else: total = elastic_out.bulk_upload_sync(fetch(elastic_in, backend, limit), uid_field) logging.info("Total items copied: %i", total)
for dependency in dependencies: eitem = enrich_item(dependency) eitem['project'] = project yield eitem if __name__ == '__main__': args = get_params() if args.debug: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s') logging.debug("Debug mode activated") else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') logging.info("Importing items from %s to %s/%s", args.file, args.elastic_url, args.index) elastic = ElasticSearch(args.elastic_url, args.index) items = fetch_dependencies(args.file, args.project) if items: logging.info("Loading dependencies in Elasticsearch ...") elastic.bulk_upload_sync(items, "uuid") logging.info("Import completed.")