def restore_dashboard(self, name, dashboard_names, rfrom): grafana_sdk.get_logger().info( "Restoring host {}, dashboard {}, from {}".format( name, dashboard_names, rfrom)) try: all_dashboard = "all" in dashboard_names if rfrom == hourly_backup_type: if all_dashboard: folder_name = "hourly/{}/".format(name) self.__scan_to_restore(folder_name, "*.json") else: for dashboard_name in dashboard_names: folder_name = "hourly/{}/".format(name) self.__scan_to_restore( folder_name, "{}.json".format(dashboard_name)) else: if all_dashboard: folder_name = "daily/{}/".format(rfrom) self.__scan_to_restore(folder_name, "*.json") else: for dashboard_name in dashboard_names: folder_name = "daily/{}/{}".format( rfrom, dashboard_name) self.__scan_to_restore( folder_name, "{}.json".format(dashboard_name)) except Exception as exc: grafana_sdk.get_logger().error( "Error restoring dashboard {}, error : {}".format( name, str(exc)))
def __scan_to_revision(self, name, db_names=None): search_db_response = self.grafana_api.search_db() if len(search_db_response) == 0: grafana_sdk.get_logger().error( "Could not find any revision files for host {}".format(name)) else: grafana_sdk.get_logger().info( "Scanned data for revison - {}".format( len(search_db_response))) for db_response in search_db_response: db_id = db_response['id'] db_uid = db_response['uid'] db_title = db_response['title'].replace(" ", "") meta_version = self.__get_revision_meta("{}/{}/{}_{}/".format( revision_folder, name, db_title.lower(), db_uid.lower())) ver = [1] if not meta_version else [meta_version] if not db_names or db_uid.lower() in db_names: db_title = db_response['title'].replace(" ", "") dashboard_versions = self.grafana_api.dashboard_versions(db_id) for dashboard_version in dashboard_versions: dashboard_version_id = dashboard_version['version'] if not meta_version or int(meta_version) < int( dashboard_version_id): dashboard_version_details = self.grafana_api.dashboard_version_details( db_id, dashboard_version_id) self.__store( "{}/{}/{}_{}/".format(revision_folder, name, db_title.lower(), db_uid.lower()), "version{}.json".format(dashboard_version_id), dashboard_version_details) ver.append(int(dashboard_version_id)) self.__store_revision_meta( "{}/{}/{}_{}/".format(revision_folder, name, db_title.lower(), db_uid.lower()), max(ver))
def __init__(self, name, grafana_url, api_key): self.s3 = True self.name = name current_date = datetime.now().strftime("%d-%m-%Y") self.hourly_folder = "hourly/" + self.name + "/" self.daily_folder = "daily/{}/".format(current_date) + self.name + "/" self.grafana_api = grafana_sdk.GrafanaApi(grafana_url, api_key) if os.path.exists(GrafanaBackupManager.grafana_config) == True: grafana_config_content = GrafanaBackupManager.get_grafana_content( GrafanaBackupManager.grafana_config) s3_backup_content = grafana_config_content['backup'].get( 's3', dict()) local_backup_content = grafana_config_content['backup'].get( 'local', dict()) self.local = local_backup_content.get('enabled', True) == True self.s3 = s3_backup_content.get('enabled', False) == True if self.local: self.backup_folder = local_backup_content.get( 'backup_folder', '') grafana_sdk.get_logger().info( "Local backup is enabled and storing under : {} ".format( self.backup_folder)) if self.s3: self.s3_ins = boto3.resource('s3') self.s3_bucket_name = s3_backup_content['bucket_name'] self.s3_backup_folder = s3_backup_content.get( 'backup_folder', 'grafana/backup/') grafana_sdk.get_logger().info( "s3 backup is enabled for bucket {} and storing under : {}" .format(self.s3_bucket_name, self.s3_backup_folder))
def __scan_to_create(self, folder_name, file_name): backup_file_list = self.__scan_folders(folder_name, file_name) if len(backup_file_list) == 0: grafana_sdk.get_logger().error( "Could not find any data to create under {}".format( folder_name)) else: grafana_sdk.get_logger().info( "Scanned data to create db - {}".format(backup_file_list)) for backup_file in backup_file_list: dashboard_content_json = self.get_backup_meta_content(backup_file) folder_id = dashboard_content_json['meta']['folderId'] folder_title = dashboard_content_json['meta']['folderTitle'] if folder_id != 0: folder_response = self.grafana_api.search_folder(folder_id) if folder_response.status_code != 200: new_folder_response = self.grafana_api.create_folder( folder_title) folder_id = new_folder_response['id'] else: folder_response = folder_response.json() folder_id = folder_response['id'] del dashboard_content_json['dashboard']['uid'] del dashboard_content_json['dashboard']['id'] dashboard_content_json['folderId'] = folder_id dashboard_content_json[ 'message'] = "Updated by grafana backup script with content {}.".format( backup_file) dashboard_content_json['overwrite'] = True self.grafana_api.restore(json.dumps(dashboard_content_json))
def __get_revision_meta(self, folder_name): try: return self.get_backup_meta_content("{}.meta_data".format( self.__get_folder_name(folder_name)))['version'] except: grafana_sdk.get_logger().info( "Revision meta data file is not present.") return None
def get_grafana_content(file_name): try: grafana_url_file = open(file_name) grafana_url_data = json.load(grafana_url_file) grafana_url_file.close() return grafana_url_data except Exception as exc: grafana_sdk.get_logger().error( "error reading file {} , error {}".format(file_name, str(exc)))
def get_grafana_mapper(grafana_url): try: name = grafana_url['name'] url = grafana_url['url'] api_key = grafana_url['api_key'] return name, url, api_key except Exception as exc: grafana_sdk.get_logger().error( "error mapping grafana host config file, {}".format(str(exc))) sys.exit(0)
def __s3_store(self, filename, content): try: grafana_sdk.get_logger().info( "Storing data : {}".format(self.s3_backup_folder + filename)) self.s3_ins.Object(self.s3_bucket_name, self.s3_backup_folder + filename).put(Body=json.dumps(content)) except Exception as exc: grafana_sdk.get_logger().error( "Error storing backup on s3 {}, {}, error : {}".format( self.s3_bucket_name, filename, str(exc)))
def __s3_read(self, filename): try: s3_object_content = self.s3_ins.Object( self.s3_bucket_name, filename).get()["Body"].read().decode('utf-8') return json.loads(s3_object_content) except Exception as exc: grafana_sdk.get_logger().error( "Error reading s3 {}, {}, error : {}".format( self.s3_bucket_name, filename, str(exc))) raise Exception("Error reading s3 bucket " + self.s3_bucket_name)
def _store_meta_info(self, backup_type, mode="Auto"): meta_data = { 'time': datetime.now().strftime("%d-%m-%Y %H:%M:%S"), 'type': backup_type, 'mode': mode } if backup_type == daily_backup_type: folder_name = self.daily_folder else: folder_name = self.hourly_folder self.__store(folder_name, ".meta_data", meta_data) grafana_sdk.get_logger().info( "Taking {} Grafana JSON file Backup for host {}.".format( backup_type.title(), self.name.title()))
def revision_dashboard_backup(self, name, dashboard_names): grafana_sdk.get_logger().info( "taking revision backup of dashboard on host {}, dashboard {}". format(name, dashboard_names)) try: all_dashboard = "all" in dashboard_names if all_dashboard: self.__scan_to_revision(name) else: self.__scan_to_revision(name, dashboard_names) except Exception as exc: grafana_sdk.get_logger().error( "Error taking revision backup {}, error : {}".format( name, str(exc)))
def __store(self, folder_name, file_name, response): if self.s3: self.__s3_store(folder_name + file_name, response) try: if self.local: folder_name = self.backup_folder + folder_name grafana_sdk.get_logger().info( "Storing data on folder : {}".format(folder_name)) os.makedirs(folder_name, exist_ok=True) with open(folder_name + file_name, 'w') as fp: json.dump(response, fp, indent=4, sort_keys=True) fp.close() except Exception as exc: grafana_sdk.get_logger().error( "Error storing backup localy error : {}".format(str(exc)))
def __scan_to_restore(self, folder_name, filename): backup_file_list = self.__scan_folders(folder_name, filename) if len(backup_file_list) == 0: grafana_sdk.get_logger().error( "Could not find any data for restore under {}".format( folder_name)) else: grafana_sdk.get_logger().info( "Scanned data for restore - {}".format(backup_file_list)) for backup_file in backup_file_list: dashboard_content_json = self.get_backup_meta_content(backup_file) dashboard_content_json[ 'message'] = "Updated by grafana backup script with content {}.".format( backup_file) dashboard_content_json['overwrite'] = True self.grafana_api.restore(json.dumps(dashboard_content_json))
def revison_grafana_backup(revision_hosts=["all"], dashboard_names=["all"]): grafana_sdk.get_logger().info("Running Grafana Revision script!") all_hosts = "all" in revision_hosts for grafana_url in GrafanaBackupManager.get_grafana_content( GrafanaBackupManager.grafana_config)['grafana_urls']: name, url, api_key = get_grafana_mapper(grafana_url) if all_hosts or name in revision_hosts: gbm = GrafanaBackupManager(name, url, api_key) pool.apply_async(gbm.revision_dashboard_backup, (name, dashboard_names)) else: grafana_sdk.get_logger().info( "could not find host - {} in {}!".format(name, revision_hosts)) pool.close() pool.join() grafana_sdk.get_logger().info("Completed running Grafana Revision!")
def create_grafana_dashboard(create_hosts=["all"], dashboard_names=["all"], rfrom=hourly_backup_type): grafana_sdk.get_logger().info("Running Grafana Create script!") all_hosts = "all" in create_hosts for grafana_url in GrafanaBackupManager.get_grafana_content( GrafanaBackupManager.grafana_config)['grafana_urls']: name, url, api_key = get_grafana_mapper(grafana_url) if all_hosts or name in create_hosts: gbm = GrafanaBackupManager(name, url, api_key) pool.apply_async(gbm.create_dashboard, (name, dashboard_names, rfrom)) else: grafana_sdk.get_logger().info( "could not find host - {} in {}!".format(name, create_hosts)) pool.close() pool.join() grafana_sdk.get_logger().info("Completed running Grafana Create!")
def backup_grafana_dashboard(backup_type): grafana_sdk.get_logger().info("Running Grafana Backup script!") for grafana_url in GrafanaBackupManager.get_grafana_content( GrafanaBackupManager.grafana_config)['grafana_urls']: name, url, api_key = get_grafana_mapper(grafana_url) gbm = GrafanaBackupManager(name, url, api_key) try: if backup_type == hourly_backup_type: pool.apply_async(gbm.hourly_backup, ()) elif backup_type == daily_backup_type: pool.apply_async(gbm.daily_backup, ()) else: pool.apply_async(gbm.hourly_backup, ()) pool.apply_async(gbm.daily_backup, ()) except Exception as e: grafana_sdk.get_logger().error( "Error running backup tasks : {}".format(str(e))) pool.close() pool.join() grafana_sdk.get_logger().info("Completed taking Grafana JSON Backup!")
def dashboard_backup(self, folder_name): try: dashboards = self.grafana_api.search_db() if len(dashboards) == 0: grafana_sdk.get_logger().error( "Could not find any data for backup under {}".format( folder_name)) else: grafana_sdk.get_logger().info( "Scanned data for backup - {}".format(len(dashboards))) for dashboard in dashboards: dashboard_uri = dashboard['uid'] dashboard_title = dashboard['title'].replace(" ", "") dashboard_details_json = self.grafana_api.dashboard_details( dashboard_uri) self.__store( folder_name, "{}_{}.json".format(dashboard_title.lower(), dashboard_uri.lower()), dashboard_details_json) except Exception as exc: grafana_sdk.get_logger().error( "Error taking backup {}, error : {}".format( folder_name, str(exc)))