def export_dashboard(elastic_url, dash_id, export_file, es_index=None): # Kibana dashboard fields kibana = { "dashboard": None, "visualizations": [], "index_patterns": [], "searches": [] } # Used to avoid having duplicates search_ids_done = [] index_ids_done = [] logging.debug("Exporting dashboard %s to %s" % (args.dashboard, args.export_file)) if not es_index: es_index = ".kibana" elastic = ElasticSearch(elastic_url, es_index) kibana["dashboard"] = { "id": dash_id, "value": get_dashboard_json(elastic, dash_id) } if "panelsJSON" not in kibana["dashboard"]["value"]: # The dashboard is empty. No visualizations included. return kibana # Export all visualizations and the index patterns and searches in them for panel in json.loads(kibana["dashboard"]["value"]["panelsJSON"]): if panel['type'] in ['visualization']: vis_id = panel['id'] vis_json = get_vis_json(elastic, vis_id) kibana["visualizations"].append({"id": vis_id, "value": vis_json}) search_id = get_search_from_vis(elastic, vis_id) if search_id and search_id not in search_ids_done: search_ids_done.append(search_id) kibana["searches"].append({ "id": search_id, "value": get_search_json(elastic, search_id) }) index_pattern_id = get_index_pattern_from_vis(elastic, vis_id) if index_pattern_id and index_pattern_id not in index_ids_done: index_ids_done.append(index_pattern_id) kibana["index_patterns"].append({ "id": index_pattern_id, "value": get_index_pattern_json(elastic, index_pattern_id) }) logging.debug("Done") with open(export_file, 'w') as f: f.write(json.dumps(kibana))
def create_search(elastic_url, dashboard, index_pattern, es_index=None): """ Create the base search for vis if used :param elastic_url: URL for ElasticSearch (ES) server :param dashboard: kibana dashboard to be used as template :param enrich_index: ES index with enriched items used in the new dashboard """ search_id = None if not es_index: es_index = ".kibana" elastic = ElasticSearch(elastic_url, es_index) dash_data = get_dashboard_json(elastic, dashboard) # First vis if "panelsJSON" not in dash_data: logging.error("Can not find vis in dashboard: %s" % (dashboard)) raise # Get the search from the first vis in the panel for panel in json.loads(dash_data["panelsJSON"]): panel_id = panel["id"] logging.debug("Checking search in %s vis" % (panel_id)) search_id = get_search_from_vis(elastic, panel_id) if search_id: break # And now time to create the search found if not search_id: logging.info("Can't find search %s" % (dashboard)) return logging.debug("Found template search %s" % (search_id)) search_json = get_search_json(elastic, search_id) search_source = search_json['kibanaSavedObjectMeta']['searchSourceJSON'] new_search_source = json.loads(search_source) new_search_source['index'] = index_pattern new_search_source = json.dumps(new_search_source) search_json['kibanaSavedObjectMeta'][ 'searchSourceJSON'] = new_search_source search_json['title'] += " " + index_pattern new_search_id = search_id + "__" + index_pattern url = elastic.index_url + "/search/" + new_search_id requests.post(url, data=json.dumps(search_json), verify=False) logging.debug("New search created: %s" % (url)) return new_search_id
def create_index_pattern(elastic_url, dashboard, enrich_index, es_index=None): """ Create a index pattern using as template the index pattern in dashboard template vis :param elastic_url: URL for ElasticSearch (ES) server :param dashboard: kibana dashboard to be used as template :param enrich_index: ES index with enriched items used in the new dashboard """ index_pattern = None if not es_index: es_index = ".kibana" elastic = ElasticSearch(elastic_url, es_index) dash_data = get_dashboard_json(elastic, dashboard) # First vis if "panelsJSON" not in dash_data: logging.error("Can not find vis in dashboard: %s" % (dashboard)) raise # Get the index pattern from the first vis in the panel # that as index pattern data for panel in json.loads(dash_data["panelsJSON"]): panel_id = panel["id"] logging.debug("Checking index pattern in %s vis" % (panel_id)) index_pattern = get_index_pattern_from_vis(elastic, panel_id) if index_pattern: break # And now time to create the index pattern found if not index_pattern: logging.error("Can't find index pattern for %s" % (dashboard)) raise logging.debug("Found %s template index pattern" % (index_pattern)) new_index_pattern_json = get_index_pattern_json(elastic, index_pattern) new_index_pattern_json['title'] = enrich_index url = elastic.index_url+"/index-pattern/"+enrich_index requests.post(url, data = json.dumps(new_index_pattern_json)) logging.debug("New index pattern created: %s" % (url)) return enrich_index
def get_elastic(url, es_index, clean=None, ocean_backend=None): mapping = None if ocean_backend: mapping = ocean_backend.get_elastic_mappings() try: ocean_index = es_index elastic_ocean = ElasticSearch(url, ocean_index, mapping, clean) except ElasticConnectException: logging.error("Can't connect to Elastic Search. Is it running?") sys.exit(1) return elastic_ocean
def get_elastic(): try: ocean_index = ConfOcean.get_index() elastic_ocean = ElasticSearch(args.elastic_url, ocean_index) except ElasticConnectException: logging.error("Can't connect to Elastic Search. Is it running?") sys.exit(1) except ElasticWriteException: logging.error("Can't write to Elastic Search.") sys.exit(1) return elastic_ocean
def import_dashboard(elastic_url, import_file, es_index=None): logging.debug("Reading from %s the JSON for the dashboard to be imported" % (args.import_file)) with open(import_file, 'r') as f: try: kibana = json.loads(f.read()) except ValueError: logging.error("Wrong file format") sys.exit(1) if 'dashboard' not in kibana: logging.error("Wrong file format. Can't find 'dashboard' field.") sys.exit(1) if not es_index: es_index = ".kibana" elastic = ElasticSearch(elastic_url, es_index) url = elastic.index_url + "/dashboard/" + kibana['dashboard']['id'] requests.post(url, data=json.dumps(kibana['dashboard']['value']), verify=False) if 'searches' in kibana: for search in kibana['searches']: url = elastic.index_url + "/search/" + search['id'] requests.post(url, data=json.dumps(search['value']), verify=False) if 'index_patterns' in kibana: for index in kibana['index_patterns']: url = elastic.index_url + "/index-pattern/" + index['id'] requests.post(url, data=json.dumps(index['value']), verify=False) if 'visualizations' in kibana: for vis in kibana['visualizations']: url = elastic.index_url + "/visualization" + "/" + vis['id'] requests.post(url, data=json.dumps(vis['value']), verify=False) logging.debug("Done")
def list_dashboards(elastic_url, es_index=None): if not es_index: es_index = ".kibana" elastic = ElasticSearch(elastic_url, es_index) dash_json_url = elastic.index_url + "/dashboard/_search?size=10000" print(dash_json_url) r = requests.get(dash_json_url) res_json = r.json() if "hits" not in res_json: logging.error("Can't find dashboards") raise RuntimeError("Can't find dashboards") for dash in res_json["hits"]["hits"]: print(dash["_id"])
task_init = datetime.now() arthur_repos = {"repositories": []} args = get_params() config_logging(args.debug) total_repos = 0 # enrich ocean index_enrich = OCEAN_INDEX + "_" + PERCEVAL_BACKEND + "_enrich" es_enrich = None try: es_enrich = ElasticSearch(args.elastic_url, index_enrich) except ElasticConnectException: logging.error("Can't connect to Elastic Search. Is it running?") # The owner could be an org or an user. for org in args.org: owner_url = get_owner_repos_url(org, args.token) try: repos = get_repositores(owner_url, args.token, args.nrepos) except requests.exceptions.HTTPError: logging.error("Can't get repos for %s" % (owner_url)) continue if args.db_projects_map: insert_projects_mapping(args.db_projects_map, org, repos) for repo in repos:
def create_dashboard(elastic_url, dashboard, enrich_index, kibana_host, es_index=None): """ Create a new dashboard using dashboard as template and reading the data from enriched_index """ def new_panels(elastic, panels, search_id): """ Create the new panels and their vis for the dashboard from the panels in the template dashboard """ dash_vis_ids = [] new_panels = [] for panel in panels: if panel['type'] in ['visualization', 'search']: if panel['type'] == 'visualization': dash_vis_ids.append(panel['id']) panel['id'] += "__"+enrich_index if panel['type'] == 'search': panel['id'] = search_id new_panels.append(panel) create_vis(elastic, dash_vis_ids, search_id) return new_panels def create_vis(elastic, dash_vis_ids, search_id): """ Create new visualizations for the dashboard """ # Create visualizations for the new dashboard item_template_url = elastic.index_url+"/visualization" # Hack: Get all vis if they are <10000. Use scroll API to get all. # Better: use mget to get all vis in dash_vis_ids item_template_url_search = item_template_url+"/_search?size=10000" r = requests.get(item_template_url_search) all_visualizations =r.json()['hits']['hits'] visualizations = [] for vis in all_visualizations: if vis['_id'] in dash_vis_ids: visualizations.append(vis) logging.info("Total template vis found: %i" % (len(visualizations))) for vis in visualizations: vis_data = vis['_source'] vis_name = vis['_id'].split("_")[-1] vis_id = vis_name+"__"+enrich_index vis_data['title'] = vis_id vis_meta = json.loads(vis_data['kibanaSavedObjectMeta']['searchSourceJSON']) vis_meta['index'] = enrich_index vis_data['kibanaSavedObjectMeta']['searchSourceJSON'] = json.dumps(vis_meta) if "savedSearchId" in vis_data: vis_data["savedSearchId"] = search_id url = item_template_url+"/"+vis_id r = requests.post(url, data = json.dumps(vis_data)) logging.debug("Created new vis %s" % (url)) if not es_index: es_index = ".kibana" # First create always the index pattern as data source index_pattern = create_index_pattern(elastic_url, dashboard, enrich_index, es_index) # If search is used create a new search with the new index_pàttern search_id = create_search(elastic_url, dashboard, index_pattern, es_index) elastic = ElasticSearch(elastic_url, es_index) # Create the new dashboard from the template dash_data = get_dashboard_json(elastic, dashboard) dash_data['title'] = enrich_index # Load template panels to create the new ones with their new vis panels = json.loads(dash_data['panelsJSON']) dash_data['panelsJSON'] = json.dumps(new_panels(elastic, panels, search_id)) dash_path = "/dashboard/"+dashboard+"__"+enrich_index url = elastic.index_url+dash_path requests.post(url, data = json.dumps(dash_data)) dash_url = kibana_host+"/app/kibana#"+dash_path return dash_url
connector = get_connector_from_name(backend_name, connectors) backend = connector[0](**vars(args)) ocean_backend = connector[1](backend, **vars(args)) enrich_backend = connector[2](backend, **vars(args)) es_index = backend.get_name() + "_" + backend.get_id() clean = args.no_incremental if args.cache: clean = True try: # Ocean elastic_state = ElasticSearch(args.elastic_url, es_index, ocean_backend.get_elastic_mappings(), clean) # Enriched ocean enrich_index = es_index + "_enrich" elastic = ElasticSearch(args.elastic_url, enrich_index, enrich_backend.get_elastic_mappings(), clean) except ElasticConnectException: logging.error("Can't connect to Elastic Search. Is it running?") sys.exit(1) ocean_backend.set_elastic(elastic_state) enrich_backend.set_elastic(elastic) try:
tweet.update(logstash_fields(tweet)) yield tweet if __name__ == '__main__': args = get_params() if args.debug: logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s') logging.debug("Debug mode activated") else: logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') logging.info("Importing tweets from %s to %s/%s", args.json_dir, args.elastic_url, args.index) elastic = ElasticSearch(args.elastic_url, args.index) total = 0 first_date = None last_date = None ids = [] tweets = [] for tweet in fetch_tweets(args.json_dir): # Check first and last dates tweet_date = parser.parse(tweet['created_at']) if not first_date or tweet_date <= first_date: first_date = tweet_date if not last_date or tweet_date >= last_date: