def photos(ctx): """ List members who either don't have a photo or their photo it too small. """ config = ctx.obj['config'] directory = Directory(config.path('data:path', 'directory.json')) missing = [] too_small = [] for family in directory.families(): for member in family.members(): if not member.photo or member.photo == 'unknown.jpeg': missing.append(member.name) else: # print config.path('data:path','photos',member.photo) img = Image.open( config.path('data:path', 'photos', member.photo)) size = img.size if size[0] < 300 or size[1] < 300: too_small.append(member.name) print("----- Missing Photo -----") for name in missing: print(name) print("\n") print("----- Photo Too Small -----") for name in too_small: print(name)
def main(): names = [] user = User() client = gdata.spreadsheet.service.SpreadsheetsService() client.ClientLogin(user.email, user.ticket) # Spreadsheets feed = client.GetSpreadsheetsFeed() logging.info(feed) id_parts = feed.entry[0].id.text.split('/') spreadsheet = id_parts[len(id_parts) - 1] # Worksheets within the Spreadsheet wsFeed = client.GetWorksheetsFeed(spreadsheet) id_parts = wsFeed.entry[0].id.text.split('/') worksheet = id_parts[len(id_parts) - 1] # Get cell data feed = client.GetCellsFeed(spreadsheet, worksheet) directory = Directory() for entry in feed.entry: # Only pull data from the 'Name' column ('A' column) if entry.title.text[0] == 'A': person = entry.content.text.strip() directory.add(person)
def edit_member(ctx, name, family_name): """ Edit a Family Member """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family, person = __find_member(directory, name, family_name) if person: need_to_save = False person_data = person.to_json() fields = sorted(person_data.keys()) for field in fields: if field in ('name', 'photo'): continue if field == 'phone': new_value = Prompt.phone() if new_value: person_data['phone'].update(new_value) new_value = person_data['phone'] elif field == 'relationships': new_value = Prompt.relationships() else: old_value = person_data[field] new_value = Prompt.input("%s (%s): " % (field, old_value)) if new_value: need_to_save = True setattr(person, field, new_value) if need_to_save: directory.save() else: print("Member not found: '%s'" % (name))
def edit(ctx, name): """ Edit a Family's Data """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family = directory.get(name) family = Prompt.choose_from_list(family, "Which Family? ") need_to_save = False fam_data = family.to_json() fields = sorted(fam_data.keys()) for name in fields: if name in ('id', 'name', 'members'): continue # TODO: what if notes already exist? if name == 'notes': new_value = Prompt.notes() else: old_value = fam_data[name] new_value = Prompt.input("%s (%s): " % (name, old_value)) if new_value: need_to_save = True setattr(family, name, new_value) if need_to_save: directory.save()
def add_member(ctx, name): """ Add a Family Member """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family = directory.get(name) __add_members(family) directory.save()
def view(ctx, name): """ View a Family """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family = directory.get(name) family = Prompt.choose_from_list(family, "Which Family? ") print(repr(family))
def get_backup_name_list(self): self.log.debug("Metadata - Get backup name list.") directory = Directory(self.log) file_list = directory.get_file_list(self.path, self.BACKUP_EXPORT_INFO_META) if isinstance(file_list, list): return sorted(file_list) self.log.debug("Metadata - Invalid backup name list.") return []
def __init__(self, data_path, file_name): self.__data_path = data_path # Create and Prep page self.__pdf = Canvas("%s/%s.pdf" % (data_path, file_name), pagesize=landscape(letter), pageCompression=0) # Read Directory Data self.__directory = Directory("%s/directory.json" % (data_path))
def vcf(ctx, vcf_file): """Import VCF Card""" config = ctx.obj['config'] directory = Directory(config.path('data:path', 'directory.json')) vcf = VCFParser(photo_dir=config.path('data:path', 'photos')) data = vcf.parse(vcf_file) print("----- %s - %s -----" % (data['last_name'], data['address'])) family = directory.get(data['last_name']) family = Prompt.choose_from_list(family, "Which Family") if family: person = family.get(data['name']) if person: print("%s already exists." % (data['name'])) else: new_person = Person(**data) family.add(new_person) directory.save() else: new_family = Family(name=data['last_name'], address=data['address'], city=data['city'], state=data['state'], zip=data['zip'], members=[data]) directory.add(new_family) directory.save()
def get_backup_rbd_info_list(self): self.log.debug("Metadata - Get backup rbd info list.") directory = Directory(self.log) dir_name_list = directory.get_dir_list(self.path) rbd_info_list = [] for dir_name in dir_name_list: if dir_name != self.BACKUP_EXPORT_INFO_META: rbd_name_list = directory.get_dir_list(self.path, dir_name) for rbd_name in rbd_name_list: rbd_info = self.get_rbd_info(dir_name, rbd_name) rbd_info_list.append((dir_name, rbd_name, rbd_info)) return rbd_info_list
def _load_directory(self): """ """ _spreadsheet = "Matsumoto Family Directory" _worksheet = "Current" directory = Directory() drive = Drive() data = drive.get_data(_spreadsheet, _worksheet) name_list = data[1][1:] directory.adds(name_list) return directory
class Exporter: def __init__(self, data_path): # Read Directory Data self.__data_path = data_path self.__directory = Directory(F"{data_path}/directory.json") def export_markdown(self, filename): with open(F"{self.__data_path}/{filename}.md", "w") as output: output.write("# Massey's Chapel UMC - Member Directory\n\n") # Family -> Members -> Person for family in self.__directory.families(): output.write(F"## {family.name}\n") output.write(F"""{family.address} {family.city}, {family.state} {family.zip} \n""") for member in family.members(): bday = "N/A" if member.birthday == None else datetime.strftime( member.birthday, "%b %d") email = "N/A" if member.email == None else member.email phone = "N/A" if len(member.phone) > 0: phones = [] for type, num in member.phone.items(): phones.append(F"{type}: {num}") phone = " | ".join(phones) person = "* %-25s\t%s\t%-25s\t%s\n" % (member.name, bday, email, phone) output.write(person) output.write("\n")
def __init__(self, log, path, meta_dir=None): self.log = log if meta_dir == None: meta_dir = const.META_DIRNAME self.path = "%s/%s" % (path, meta_dir) self.metafile = MetaFile(log) self.directory = Directory(log) self.CLUSTER_INFO_META = const.META_CLUSTER_INFO self.CLUSTER_RBD_INFO_META = const.META_CLUSTER_RBD_INFO self.CLUSTER_RBD_SNAPSHOT_INFO_META = const.META_CLUSTER_RBD_SNAPSHOT_INFO self.BACKUP_EXPORT_INFO_META = const.META_BACKUP_EXPORT_INFO self.BACKUP_CIRCLE_INFO_META = const.META_BACKUP_CIRCLE_INFO self.BACKUP_INCREMENTAL_INFO_META = const.META_BACKUP_INCREMENTAL_INFO self.LIST_COUNTER_KEY = const.META_LIST_COUNTER_KEY self.MAX_BACKUP_INFO_RETAIN_COUNT = const.META_ROTATION_LENGTH
def delete(ctx, name): """ Delete a Family from the Directory """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family = directory.get(name) family = Prompt.choose_from_list(family, "Which Family? ") choice = Prompt.input("Confirm Delete: %s - %s (yes|no)? " % (family.name, family.address)) if choice == 'yes': directory.delete(family) directory.save() else: print("Not Deleting Family: %s - %s" % (family.name, family.address))
def del_member(ctx, name, family_name): """ Remove a Family Member from a Family """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family, person = __find_member(directory, name, family_name) if person: family.delete(person) directory.delete_photo(person) directory.save() print("'%s' successfully deleted." % (name)) else: print("Member not found: '%s'" % (name))
def add(ctx, name): """ Add a new Family to the Directory """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) family_data = Family(name=name).to_json() fields = sorted(family_data.keys()) print("----- Adding the '%s' Family -----" % (family_data['name'])) for name in fields: if name in ('id', 'name', 'members'): continue new_value = Prompt.input("%s: " % (name)) family_data[name] = new_value if new_value else "" new_family = Family(**family_data) print(new_family) directory.add(new_family) __add_members(new_family) directory.save()
def fix(ctx, thing): """ Run bulk data fix commands """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) # Fix relationships structure if thing == "relationships": for fam in directory.families(): for person in fam.members(): new_rels = [] for rel in person.relationships: new_rels.append({ 'type': rel['type'].capitalize(), 'name': rel['name'] }) person.relationships = new_rels directory.save() # Fix email addr -- set to None if none elif thing == "email": for fam in directory.families(): for person in fam.members(): if person.email == "N/A": person.email = None directory.save() elif thing == "add_notes": directory.save() elif thing == "family_member_to_list": directory.save() elif thing == "directory_to_list": directory.save() elif thing == "format": directory.save() else: print("I'm sorry Dave, I'm afraid I can't fix %s!" % (thing))
if __name__ == '__main__': config = Config().local() files = [] # TODO use argparse # TODO add command to diff 2 files # TODO add summary flag try: path = sys.argv[1] except IndexError: path = '' progress = '--progress' in sys.argv dir = Directory(path or './') for file in dir.files_iter(config.skip): file.snippets = list(file.parse(config.snippet)) files.append(file) print('Analyzing content of "{}"'.format(dir.path)) print('Satisfactory files found: {}'.format(len(files))) print('Please be patient ...') paired = [] similars = [] total = sum([len(file.snippets) for file in files])**2 i = 0 for file_a in files:
class RBD_Backup_Metadata(object): def __init__(self, log, path, meta_dir=None): self.log = log if meta_dir == None: meta_dir = const.META_DIRNAME self.path = "%s/%s" % (path, meta_dir) self.metafile = MetaFile(log) self.directory = Directory(log) self.CLUSTER_INFO_META = const.META_CLUSTER_INFO self.CLUSTER_RBD_INFO_META = const.META_CLUSTER_RBD_INFO self.CLUSTER_RBD_SNAPSHOT_INFO_META = const.META_CLUSTER_RBD_SNAPSHOT_INFO self.BACKUP_EXPORT_INFO_META = const.META_BACKUP_EXPORT_INFO self.BACKUP_CIRCLE_INFO_META = const.META_BACKUP_CIRCLE_INFO self.BACKUP_INCREMENTAL_INFO_META = const.META_BACKUP_INCREMENTAL_INFO self.LIST_COUNTER_KEY = const.META_LIST_COUNTER_KEY self.MAX_BACKUP_INFO_RETAIN_COUNT = const.META_ROTATION_LENGTH def set_cluster_info(self, meta): filepath = os.path.join(self.path, self.CLUSTER_INFO_META) return self.metafile.write(meta, filepath=filepath) def set_backup_export_info(self, export_name, meta): filepath = os.path.join(self.path, self.BACKUP_EXPORT_INFO_META, export_name) return self.metafile.write(meta, filepath=filepath) def set_rbd_info(self, pool_name, rbd_name, meta): filepath = os.path.join(self.path, pool_name, rbd_name, self.CLUSTER_RBD_INFO_META) return self.metafile.write(meta, filepath=filepath) def set_backup_circle_list(self, pool_name, rbd_name, meta): filepath = os.path.join(self.path, pool_name, rbd_name, self.BACKUP_CIRCLE_INFO_META) return self.metafile.write(meta, filepath=filepath) def set_backup_snapshot_list(self, pool_name, rbd_name, meta): filepath = os.path.join(self.path, pool_name, rbd_name, self.CLUSTER_RBD_SNAPSHOT_INFO_META) return self.metafile.write(meta, filepath=filepath) def set_backup_incremental_list(self, pool_name, rbd_name, circle_name, meta): filepath = os.path.join(self.path, pool_name, rbd_name, self.BACKUP_INCREMENTAL_INFO_META, circle_name) return self.metafile.write(meta, filepath=filepath) def get_cluster_info(self): self.log.debug("Metadata - Get backup cluster info.") filepath = os.path.join(self.path, self.CLUSTER_INFO_META) cluster_info = self.metafile.read(filepath=filepath) if isinstance(cluster_info, dict): return cluster_info self.log.debug("Metadata - Invalid backup cluster info.") return {} def get_rbd_info(self, pool_name, rbd_name): self.log.debug("Metadata - Get backup RBD info.") filepath = os.path.join(self.path, pool_name, rbd_name, self.CLUSTER_RBD_INFO_META) rbd_info = self.metafile.read(filepath=filepath) if isinstance(rbd_info, dict): return rbd_info self.log.debug("Metadata - Invalid backup RBD info.") return {} def get_backup_name_list(self): self.log.debug("Metadata - Get backup name list.") directory = Directory(self.log) file_list = directory.get_file_list(self.path, self.BACKUP_EXPORT_INFO_META) if isinstance(file_list, list): return sorted(file_list) self.log.debug("Metadata - Invalid backup name list.") return [] def get_backup_rbd_list(self, backup_name): self.log.debug("Metadata - Get backup rbd list.") filepath = os.path.join(self.path, self.BACKUP_EXPORT_INFO_META, backup_name) rbd_list = self.metafile.read(filepath=filepath) if isinstance(rbd_list, list): return rbd_list self.log.debug("Metadata - Invalid backup rbd list.") return [] def get_backup_rbd_info_list(self): self.log.debug("Metadata - Get backup rbd info list.") directory = Directory(self.log) dir_name_list = directory.get_dir_list(self.path) rbd_info_list = [] for dir_name in dir_name_list: if dir_name != self.BACKUP_EXPORT_INFO_META: rbd_name_list = directory.get_dir_list(self.path, dir_name) for rbd_name in rbd_name_list: rbd_info = self.get_rbd_info(dir_name, rbd_name) rbd_info_list.append((dir_name, rbd_name, rbd_info)) return rbd_info_list def get_backup_circle_list(self, pool_name, rbd_name): self.log.debug("Metadata - Get backup circle list.") filepath = os.path.join(self.path, pool_name, rbd_name, self.BACKUP_CIRCLE_INFO_META) circle_list = self.metafile.read(filepath=filepath) if isinstance(circle_list, list): sorted_circle_list = sort_dict_list(circle_list, self.LIST_COUNTER_KEY, reverse=False) if sorted_circle_list == False: self.log.debug("Metadata - Unable to sort backup circle list.") else: return sorted_circle_list self.log.debug("Metadata - Invalid backup circle list.") return [] def get_backup_snapshot_list(self, pool_name, rbd_name): self.log.debug("Metadata - Get backup snapshot list.") filepath = os.path.join(self.path, pool_name, rbd_name, self.CLUSTER_RBD_SNAPSHOT_INFO_META) snapshot_list = self.metafile.read(filepath=filepath) if isinstance(snapshot_list, list): sorted_snapshot_list = sort_dict_list(snapshot_list, self.LIST_COUNTER_KEY, reverse=False) if sorted_snapshot_list == False: self.log.debug( "Metadata - Unable to sort backup snapshot list.") else: return sorted_snapshot_list self.log.debug("Metadata - Invalid backup snapshot list.") return [] def get_backup_incremental_list(self, pool_name, rbd_name, circle_name): self.log.debug("Metadata - Get backup incremental list.") filepath = os.path.join(self.path, pool_name, rbd_name, self.BACKUP_INCREMENTAL_INFO_META, circle_name) incremental_list = self.metafile.read(filepath=filepath) if isinstance(incremental_list, list): sorted_incremental_list = sort_dict_list(incremental_list, self.LIST_COUNTER_KEY, reverse=False) if sorted_incremental_list == False: self.log.debug( "Metadata - Unable to sort backup incremental list.") else: return sorted_incremental_list self.log.debug("Metadata - Invalid backup incremental list.") return [] def del_backup_circle_info(self, pool_name, rbd_name, delete_circle_info, key=None): try: self.log.debug("Metadata - Delete a backup circle.") if key == None: d_key = self.LIST_COUNTER_KEY else: d_key = key circle_list = self.get_backup_circle_list(pool_name, rbd_name) new_circle_list = [] for circle_info in circle_list: if delete_circle_info[d_key] != circle_info[d_key]: new_circle_list.append(circle_info) return self.set_backup_circle_list(pool_name, rbd_name, new_circle_list) except Exception as e: self.log.error("Metadata - Unable to delete a backup circle. %s" % e) return False def del_backup_snapshot_info(self, pool_name, rbd_name, delete_snapshot_info, key=None): try: self.log.debug("Metadata - Delete a backup snapshot.") if key == None: d_key = self.LIST_COUNTER_KEY else: d_key = key snapshot_list = self.get_backup_snapshot_list(pool_name, rbd_name) new_snapshot_list = [] for snapshot_info in snapshot_list: if delete_snapshot_info[d_key] != snapshot_info[d_key]: new_snapshot_list.append(snapshot_info) return self.set_backup_snapshot_list(pool_name, rbd_name, new_snapshot_list) except Exception as e: self.log.error( "Metadata - Unable to delete a backup snapshot. %s" % e) return False def del_backup_incremental_info(self, pool_name, rbd_name, circle_name, incr_info=None, key=None): try: self.log.debug("Metadata - Delete a backup incremental.") filepath = os.path.join(self.path, pool_name, rbd_name, self.BACKUP_INCREMENTAL_INFO_META, circle_name) if incr_info == None: self.log.debug( "Metadata - Delete backup incremental file '%s'." % filepath) self.directory.delete(filepath) return True incremental_list = self.metafile.read(filepath=filepath) new_incremental_list = [] if isinstance(incremental_list, list): for incremental_name in incremental_list: if incremental_name != incr_info: new_incremental_list.append(incremental_name) return self.set_backup_incremental_list(pool_name, rbd_name, circle_name, new_incremental_list) except Exception as e: self.log.error( "Metadata - Unable to delete a backup incremental. %s" % e) return False def del_backup_info(self, backup_name): try: self.log.debug("Metadata - Delete a backup info.") filepath = os.path.join(self.path, self.BACKUP_EXPORT_INFO_META, backup_name) self.directory.delete(filepath) return True except Exception as e: self.log.error("Metadata - Unable to delete a backup info. %s" % e) return False def del_rbd_info(self, pool_name, rbd_name): try: self.log.debug("Metadata - Delete a rbd meta.") filepath = os.path.join(self.path, pool_name, rbd_name) self.directory.delete(filepath) return True except Exception as e: self.log.error("Metadata - Unable to delete a rbd meta. %s" % e) return False def add_backup_circle_info(self, pool_name, rbd_name, new_circle_info): try: self.log.debug("Metadata - Add new backup circle.") circle_list = self.get_backup_circle_list(pool_name, rbd_name) add_circle_info = new_circle_info if len(circle_list) == 0: add_circle_info[self.LIST_COUNTER_KEY] = 1 else: last_circle_info = circle_list[-1] add_circle_info[self.LIST_COUNTER_KEY] = int( last_circle_info[self.LIST_COUNTER_KEY]) + 1 circle_list.append(add_circle_info) return self.set_backup_circle_list(pool_name, rbd_name, circle_list) except Exception as e: self.log.error("Metadata - Unable to add new backup circle. %s" % e) return False def add_backup_info(self, backup_name, meta): try: self.log.debug("Metadata - Add new backup export.") self.set_backup_export_info(backup_name, meta) rotation_length = self.MAX_BACKUP_INFO_RETAIN_COUNT backup_name_list = self.get_backup_name_list() diff_count = len(backup_name_list) - int(rotation_length) if diff_count > 0: sorted_export_list = sorted(backup_name_list) for i in range(0, diff_count): self.del_backup_info(backup_name_list[i]) except Exception as e: self.log.error( "Metadata - Unable to add new backup export info. %s" % e) return False def add_backup_snapshot_info(self, pool_name, rbd_name, new_snapshot_info): try: self.log.debug("Metadata - Add new backup snapshot.") snapshot_list = self.get_backup_snapshot_list(pool_name, rbd_name) add_snapshot_info = new_snapshot_info if len(snapshot_list) == 0: add_snapshot_info[self.LIST_COUNTER_KEY] = 1 else: last_snapshot_info = snapshot_list[-1] new_counter = int( last_snapshot_info[self.LIST_COUNTER_KEY]) + 1 add_snapshot_info[self.LIST_COUNTER_KEY] = new_counter snapshot_list.append(add_snapshot_info) return self.set_backup_snapshot_list(pool_name, rbd_name, snapshot_list) except Exception as e: self.log.error("Metadata - Unable to add new backup snapshot. %s" % e) return False def add_backup_incremental_info(self, pool_name, rbd_name, circle_name, new_incr_info): try: self.log.debug("Metadata - Add new backup incremental.") incremental_list = self.get_backup_incremental_list( pool_name, rbd_name, circle_name) add_incremental_info = new_incr_info if len(incremental_list) == 0: add_incremental_info[self.LIST_COUNTER_KEY] = 1 else: last_incremental_info = incremental_list[-1] new_counter = int( last_incremental_info[self.LIST_COUNTER_KEY]) + 1 add_incremental_info[self.LIST_COUNTER_KEY] = new_counter incremental_list.append(add_incremental_info) return self.set_backup_incremental_list(pool_name, rbd_name, circle_name, incremental_list) except Exception as e: self.log.error( "Metadata - Unable to add new backup incremental. %s" % e) return False
def main(argument_list): DEFAULT_CONFIG_PATH = const.CONFIG_PATH DEFAULT_CONFIG_SECTION = const.CONFIG_SECTION try: parser = ArgumentParser(add_help=False) parser.add_argument('--config-file') parser.add_argument('--config-section') parser.add_argument('--cluster-name') parser.add_argument('--backup-directory') parser.add_argument('--yes', action='store_true') parser.add_argument('options', nargs='+') args = vars(parser.parse_args(argument_list[1:])) backup_config_file = DEFAULT_CONFIG_PATH backup_config_section = DEFAULT_CONFIG_SECTION if args['config_file'] is not None: backup_config_file = args['config_file'] if args['config_section'] is not None: backup_config_section = args['config_section'] # create config obj and read config file data cfg = Config(backup_config_file, backup_config_section) if not cfg.is_valid(): print("Error, fail to initialize config.") sys.exit(2) if not cfg.set_options(print_options=False): print("Error, fail to set config.") sys.exit(2) backup_directory = cfg.backup_destination_path cluster_name = cfg.ceph_cluster_name if args['backup_directory'] is not None: backup_directory = args['backup_directory'] if args['cluster_name'] is not None: cluster_name = args['cluster_name'] ask_yes_no = True if args['yes']: ask_yes_no = False opts = args['options'] # initial backup logging log = Logger(cfg.log_file_path, cfg.log_level, cfg.log_max_bytes, cfg.log_backup_count, cfg.log_delay, name=const.LOG_DELETE_LOGGER_NAME) if not log.set_log(log_module=cfg.log_module_name): print("Error, unable to set logger.") sys.exit(2) backup_cluster_directory = os.path.join(backup_directory, cluster_name) backup_meta = RBD_Backup_Metadata(log, backup_cluster_directory) directory = Directory(log) if opts[0] == 'rbd': if len(opts) == 3: pool_name = opts[1] rbd_name = opts[2] log.info("Delete backuped RBD.", "pool name: %s" % pool_name, "rbd name: %s" % rbd_name) print("- delete backuped RBD:") print(" pool name: %s" % pool_name) print(" rbd name: %s" % rbd_name) if ask_yes_no: response = raw_input("Are you sure to delete this RBD backup "\ "(yes or no)? : ") response = response.lower() if response != 'yes': sys.exit(0) dir_ret = directory.delete(backup_cluster_directory, pool_name, rbd_name) meta_ret = backup_meta.del_rbd_info(pool_name, rbd_name) if dir_ret != False and meta_ret != False: log.info("The Backuped RBD is deleted successfully.") print("- the backuped RBD is deleted successfully.") else: print("Error, error occur while deleting backuped RBD.") except KeyboardInterrupt as ke: print("- Operation canceled.") sys.exit(2) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) sys.exit(2)
def __init__(self, data_path): # Read Directory Data self.__data_path = data_path self.__directory = Directory(F"{data_path}/directory.json")
def main(argument_list): DEFAULT_CONFIG_PATH = const.CONFIG_PATH DEFAULT_CONFIG_SECTION = const.CONFIG_SECTION try: parser = ArgumentParser(add_help=False) parser.add_argument('--config-file') parser.add_argument('--config-section') parser.add_argument('--cluster-name') parser.add_argument('--backup-directory') parser.add_argument('--show-details', action='store_true') parser.add_argument('options', nargs='+') #args = vars(parser.parse_args(argument_list[1:])) known, unknown = parser.parse_known_args(argument_list[1:]) args = vars(known) # get config file path and config section name restore_config_file = DEFAULT_CONFIG_PATH restore_config_section = DEFAULT_CONFIG_SECTION if args['config_file'] is not None: restore_config_file = args['config_file'] if args['config_section'] is not None: restore_config_section = args['config_section'] # create config obj and read config file data cfg = Config(restore_config_file, restore_config_section) if not cfg.is_valid(): print("Error, fail to initialize config.") sys.exit(2) if not cfg.set_options(print_options=False): print("Error, fail to set config.") sys.exit(2) # get backup directory path and cluster name backup_directory = cfg.backup_destination_path cluster_name = cfg.ceph_cluster_name if args['backup_directory'] is not None: backup_directory = args['backup_directory'] if args['cluster_name'] is not None: cluster_name = args['cluster_name'] show_details = False if args['show_details']: show_details = True opts = args['options'] # initial backup logging log = Logger(cfg.log_file_path, cfg.log_level, cfg.log_max_bytes, cfg.log_backup_count, cfg.log_delay, name=const.LOG_SHOW_LOGGER_NAME) if not log.set_log(log_module=cfg.log_module_name): print("Error, unable to set logger.") sys.exit(2) if opts[0] != 'config': rbd_restore_list = RBD_Restore_List(log) backup_cluster_directory = os.path.join(backup_directory, cluster_name) if not rbd_restore_list.read_meta(backup_cluster_directory): print("Error, unable to get cluster info.") sys.exit(2) directory = Directory(log) print("") if opts[0] == 'backup': if len(opts) == 1: print("*Show all backup name.\n") backup_name_list = rbd_restore_list.get_backup_name_list() if show_details: print("[Backup Name] [RBD Count]") print("-" * 31) else: print("[Backup Name List]") print("-" * 20) for backup_name in backup_name_list: if show_details: rbd_list = rbd_restore_list.get_backup_rbd_list( backup_name) print("%s %s" % (backup_name, len(rbd_list))) else: print("%s" % backup_name) elif len(opts) >= 2: backup_name = normalize_datetime(' '.join(opts[1:])) rbd_list = rbd_restore_list.get_backup_rbd_list(backup_name) print("*Show RBD list in backup name '%s'.\n" % backup_name) if show_details: print("[Backup Time] [Circle name] " \ "[Pool name/RBD name] ... [Status]") print("-" * 74) else: print("[Backup RBD List]") print("-" * 20) for rbd_info in rbd_list: if show_details: if rbd_info[0] == 0: status = 'OK' else: status = 'FAIL' if status == 'OK': exist = directory.exist(backup_cluster_directory, rbd_info[1], rbd_info[2], rbd_info[3]) if exist == False: status = 'Deleted' print("%s %s %s/%s ... %s" % (rbd_info[4], rbd_info[3], rbd_info[1], rbd_info[2], status)) else: print("%s/%s" % (rbd_info[1], rbd_info[2])) elif opts[0] == 'rbd': if len(opts) == 1: print("*Show all backuped RBD name.\n") rbd_list = rbd_restore_list.get_backup_rbd_list() if len(rbd_list) == 0: return 0 arranged_rbd_info = {} for rbd_info in rbd_list: pool_name = rbd_info[0] rbd_name = rbd_info[1] info = rbd_info[2] if not arranged_rbd_info.has_key(pool_name): rbd_info_list = [] else: rbd_info_list = arranged_rbd_info[pool_name] info['name'] = rbd_name rbd_info_list.append(info) arranged_rbd_info[pool_name] = rbd_info_list if show_details: print("[Pool name]") print( " [RBD name] [block name prefix] [Num objects] [size (bytes)]" ) print("-" * 74) else: print("[Pool name]") print(" [RBD name]") print("-" * 20) for pool_name, rbd_list in arranged_rbd_info.iteritems(): print("%s" % pool_name) if show_details: for rbd_info in rbd_list: rbd_name = rbd_info['name'] rbd_size = rbd_info['size'] rbd_objs = rbd_info['num_objs'] rbd_prix = rbd_info['block_name_prefix'] print(" %s %s %s %s" % (rbd_name, rbd_prix, rbd_objs, rbd_size)) else: for rbd_info in rbd_list: print(" %s" % rbd_info['name']) if len(opts) == 3: pool_name = opts[1] rbd_name = opts[2] print("*Show backup time of RBD '%s/%s'.\n" % (pool_name, rbd_name)) if show_details: backup_info_list = rbd_restore_list.get_rbd_backup_info_list( pool_name, rbd_name) if len(backup_info_list) == 0: return 0 print("[Backup time] [Backup name] " \ "[Backup circle] [Backup size]") print("-" * 74) for backup_info in backup_info_list: backup_file = backup_info[0] backup_time = backup_info[1] backup_name = backup_info[2] backup_circ = backup_info[3] backup_size = directory.get_used_size( backup_cluster_directory, pool_name, rbd_name, backup_circ, backup_file) print("%s %s %s %s" % (backup_time, backup_name, backup_circ, backup_size)) else: backup_time_list = rbd_restore_list.get_rbd_backup_time_list( pool_name, rbd_name) if len(backup_time_list) == 0: return 0 print("[Backup time]") print("-" * 20) for backup_time in backup_time_list: print("%s" % backup_time) elif opts[0] == 'cluster': print("*Show backuped cluster info.\n") cluster_info = rbd_restore_list.get_cluster_info() for key, value in cluster_info.iteritems(): print("%s: %s" % (key, value)) elif opts[0] == 'config': if len(opts) == 1: print("*Show backup config.\n") cfg_opts = cfg.get_option() for key, value in cfg_opts.iteritems(): print("%s = %s" % (key, value)) elif len(opts) == 2: if opts[1] == 'openstack': print("*Show openstack yaml. (constructing)\n") elif len(opts) == 3: if opts[1] == 'rbd' and opts[2] == 'list': print("*Show RBD backup list.") print("*Yaml file: %s" % cfg.backup_list_file_path) print("*Cluster name: %s" % cluster_name) print("") rbd_backup_list = RBD_Backup_List(log) rbd_backup_list.read_yaml(cfg.backup_list_file_path) rbd_name_list = rbd_backup_list.get_rbd_name_list( cluster_name) if show_details: print("[Pool name]") print( " [RBD name] [backup_type] [max_incr] [max_circ] [max_snap]" ) print("-" * 74) backup_option = RBD_Backup_Option(log) for pool_name, rbd_name_list in rbd_name_list.iteritems( ): print("%s" % pool_name) for rbd_name in rbd_name_list: options = rbd_backup_list.get_rbd_options( cluster_name, pool_name, rbd_name) backup_option.add_option( pool_name, rbd_name, options) backup_type = backup_option.get_backup_type( pool_name, rbd_name) max_incr = backup_option.get_backup_max_incr_count( pool_name, rbd_name) max_circ = backup_option.get_backup_circle_retain_count( pool_name, rbd_name) max_snap = backup_option.get_snapshot_retain_count( pool_name, rbd_name) print(" %s %s %s %s %s" % (rbd_name, backup_type, max_incr, max_circ, max_snap)) else: print("[Pool name]") print(" [RBD name]") print("-" * 24) for pool_name, rbd_name_list in rbd_name_list.iteritems( ): print("%s" % pool_name) for rbd_name in rbd_name_list: print(" %s" % rbd_name) print("") except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) sys.exit(2)
class PrintDirectory: FONT = "Times-Roman" FONT_BOLD = "Times-Bold" SIDE_LEFT = 0 SIDE_RIGHT = 1 MAX_MEMBERS_PER_SIDE = 3 def __init__(self, data_path, file_name): self.__data_path = data_path # Create and Prep page self.__pdf = Canvas("%s/%s.pdf" % (data_path, file_name), pagesize=landscape(letter), pageCompression=0) # Read Directory Data self.__directory = Directory("%s/directory.json" % (data_path)) def render(self, **kwargs): if kwargs.get("bdays_only", False): self.render_bdays() else: self.render_title() self.render_bdays() self.render_families() # Save PDF self.__pdf.save() def render_title(self): x = 4.0 * inch y = 8.0 * inch # Border self.__pdf.setStrokeColorRGB(0, 0, 0) self.__pdf.setFillColor(lightgrey) self.__pdf.roundRect(3.0 * inch, 7.4 * inch, 5 * inch, 1 * inch, 10, fill=1) # Top Title self.__pdf.setFillColorRGB(0, 0, 0) self.__pdf.setFont(PrintDirectory.FONT, 32) self.__pdf.drawString(x, y, "Massey's Chapel") self.__pdf.drawString(x - (.75 * inch), y - (.50 * inch), "United Methodist Church") # Church Photo photo = "%s/photos/title_photo.jpg" % (self.__data_path) self.__pdf.drawImage(photo, 1.0 * inch, 1.25 * inch, width=650, height=434) # Bottom - Sub-title x = 4.25 * inch y = .75 * inch self.__pdf.setFont(PrintDirectory.FONT, 24) self.__pdf.drawString(x, y, "Member Directory") self.__pdf.drawString( x, y - (.35 * inch), datetime.strftime(datetime.now(), "~~~ %B %Y ~~~")) self.__pdf.showPage() def render_bdays(self): # Month placement adjustment placement = [0, 0, 0, 0, 0, 0, 0, 0, 0, .15 * inch, 0, 0] bdays = [{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}] for family in self.__directory.families(): for person in family.members(): if person.birthday: month = person.birthday.month day = person.birthday.day name = person.name bucket = bdays[month].get(day, []) if not bucket: bdays[month][day] = bucket bucket.append(name) self.__pdf.setFillColorRGB(0, 0, 0) self.__pdf.setFont(PrintDirectory.FONT, 24) self.__pdf.drawString(4.85 * inch, 8.2 * inch, "Birthdays") # Vertical Line Down Center self.__pdf.setStrokeColorRGB(0, 0, 0) self.__pdf.line(5.5 * inch, 0.10 * inch, 5.5 * inch, 8.0 * inch) origin_x = 0.5 * inch origin_y = 7.75 * inch width = 2.25 * inch height = 2.5 * inch row = 1 for num, month in enumerate(bdays): if not month: continue if num % 2 != 0: # ODD offset = row - 1 pos_x = origin_x pos_y = (origin_y - (offset * height)) + placement[num - 1] # print "ODD - %d] x:%d y:%d" % (num, pos_x,pos_y) else: # EVEN offset = row - 1 row += 1 if row > 3: row = 1 pos_x = origin_x + width pos_y = (origin_y - (offset * height)) + placement[num - 1] # print "EVEN - %d] x:%d y:%d" % (num, pos_x,pos_y) if num > 6: pos_x += 5.5 * inch self.__pdf.setFont(PrintDirectory.FONT_BOLD, 9) header = self.__pdf.beginText(pos_x, pos_y) header.textLine("~~~ %s ~~~" % (datetime(1900, num, 1).strftime("%B").upper())) self.__pdf.drawText(header) self.__pdf.setFont(PrintDirectory.FONT, 9) info = self.__pdf.beginText(pos_x, pos_y - 0.15 * inch) for day in sorted(month.keys()): names = month.get(day) for n in names: info.textLine("%02d) %s" % (day, n)) self.__pdf.drawText(info) self.__pdf.showPage() def render_families(self): side_it = itertools.cycle( (PrintDirectory.SIDE_LEFT, PrintDirectory.SIDE_RIGHT)) families = self.__directory.families() family_index = 0 family_count = len(families) while family_index < family_count: # Vertical Line Down Center self.__pdf.setStrokeColorRGB(0, 0, 0) self.__pdf.line(5.5 * inch, 0.10 * inch, 5.5 * inch, 8.4 * inch) # Render the Family family1 = families[family_index] family_index += 1 family1_args = {} family1_side = next(side_it) family2_args = {} family2_side = next(side_it) if len(family1.members()) > PrintDirectory.MAX_MEMBERS_PER_SIDE: family1_args = { 'm_start': 0, 'm_end': PrintDirectory.MAX_MEMBERS_PER_SIDE } family2 = family1 family2_args = { 'm_start': PrintDirectory.MAX_MEMBERS_PER_SIDE + 1, 'm_end': PrintDirectory.MAX_MEMBERS_PER_SIDE * 2, 'family_cont': True } else: if family_index < family_count: family2 = families[family_index] # Don't start rendering a 2 page side family on the LEFT if len(family2.members() ) > PrintDirectory.MAX_MEMBERS_PER_SIDE: family2 = None else: family_index += 1 else: family2 = None self.render_family(family1, family1_side, **family1_args) if family2: self.render_family(family2, family2_side, **family2_args) # End the Page self.__pdf.showPage() def render_family(self, family, side, **kwargs): left_margin = 0 right_margin = 0 if side == PrintDirectory.SIDE_LEFT: left_margin = .25 * inch right_margin = 5.5 * inch elif side == PrintDirectory.SIDE_RIGHT: left_margin = 5.5 * inch + .25 * inch right_margin = 11 * inch else: raise ValueError("Invalid value [%s] for 'side' parameter." % (side)) # Family Name self.__pdf.setFont(PrintDirectory.FONT, 18) draw_pos = {'x': left_margin, 'y': 8.125 * inch} self.__pdf.drawString(draw_pos['x'], draw_pos['y'], family.name) # Family Address if not kwargs.get('family_cont', False): self.__pdf.setFont(PrintDirectory.FONT, 16) draw_pos['y'] -= .20 * inch addr = self.__pdf.beginText(draw_pos['x'], draw_pos['y']) addr.setLeading(15) addr.textLine(family.address) addr.textLine("%s, %s %s" % (family.city, family.state, family.zip)) self.__pdf.drawText(addr) # Family Name/Address Divider self.__pdf.setStrokeColorRGB(.5, .5, .5) self.__pdf.line(left_margin, 7.5 * inch, right_margin - (0.25 * inch), 7.5 * inch) # Member Photo, Info photo_dir = "%s/photos" % (self.__data_path) pos_x = left_margin pos_y = 5.35 * inch all_members = family.members() m_start = kwargs.get('m_start', 0) m_end = kwargs.get('m_end', len(all_members)) members = all_members[m_start:m_end + 1] for person in members: self.render_person(person, pos_x, pos_y, photo_dir) pos_x += 2.80 * inch if pos_x >= right_margin: pos_x = left_margin pos_y = 1.90 * inch def render_person(self, person, pos_x, pos_y, photo_dir): # Photo photo_name = person.photo if person.photo else 'unknown.jpeg' photo = "%s/%s" % (photo_dir, photo_name) self.__pdf.drawImage(photo, pos_x, pos_y, width=150, height=150) # Info info = self.__pdf.beginText(pos_x, pos_y - .25 * inch) # Name info.setFont(PrintDirectory.FONT, 16) info.textLine(person.name) info.setFont(PrintDirectory.FONT, 12) # Bday date_str = "N/A" if person.birthday: date_str = datetime.strftime(person.birthday, "%b %d") info.textLine("Birthday: %s" % date_str) # Email email_addr = person.email if person.email else "N/A" info.textLine("Email: %s" % email_addr) # Phones if person.phone: for type, number in person.phone.items(): info.textLine("%s: %s" % (type.capitalize(), number)) # Relationships if person.relationships: if len(person.relationships) > 2: spouse = None children = [] for rel in person.relationships: if rel['type'] == 'Spouse': spouse = rel['name'] elif rel['type'] == 'Child': children.append(rel['name']) text_lines = [] if spouse: text_lines.append("Spouse: %s" % (spouse)) if children: batch1 = children[:2] batch2 = children[2:] text_lines.append("Children: %s" % (", ".join(batch1))) if batch2: text_lines.append(" %s" % (", ".join(batch2))) if text_lines: info.textLine("_________________________") for line in text_lines: info.textLine(line) else: info.textLine("_________________________") for rel in person.relationships: info.textLine("%s: %s" % (rel['type'], rel['name'])) self.__pdf.drawText(info)
def main(argument_list): #const.EXPORT_TYPE = const.EXPORT_TYPE backup_name = None # total size of RBD image to backup (provisioned size) total_rbd_size = 0 # available and used size of backup directory backup_dir_avai_bytes = 0 backup_dir_used_bytes = 0 # store RBD info list (list of dict) backup_rbd_info_list = [] # task counter submitted_snap_create_task_count = 0 submitted_snap_delete_task_count = 0 submitted_rbd_export_task_count = 0 backup_circle_delete_count = 0 is_rbd_list_from_command_line = False manager = None ceph = None try: # parse arguments parser = ArgumentParser(add_help=False) parser.add_argument('--config-file') parser.add_argument('--config-section') parser.add_argument('--backup-name') parser.add_argument('rbd_list', nargs='*') args = vars(parser.parse_args(argument_list[1:])) backup_config_file = const.CONFIG_PATH backup_config_section = const.CONFIG_SECTION if args['config_file'] is not None: backup_config_file = args['config_file'] if args['config_section'] is not None: backup_config_section = args['config_section'] if args['backup_name'] is not None: backup_name = args['backup_name'] if len(args['rbd_list']) != 0: is_rbd_list_from_command_line = True # create config obj and read config file data cfg = Config(backup_config_file, backup_config_section) if not cfg.is_valid(): print("Error, fail to initialize config.") sys.exit(2) if not cfg.set_options(print_options=False): print("Error, fail to set config.") sys.exit(2) # initial backup logging log = Logger(cfg.log_file_path, cfg.log_level, cfg.log_max_bytes, cfg.log_backup_count, cfg.log_delay, name=const.LOG_BACKUP_LOGGER_NAME) if not log.set_log(log_module=cfg.log_module_name): print("Error, unable to set logger.") sys.exit(2) # set name of this backup begin_backup_datetime = get_datetime() if backup_name == None: backup_name = normalize_datetime(begin_backup_datetime) print("- backup name: %s" % backup_name) # start RBD backup log.blank(line_count=4) log.info("******** Start Ceph RBD backup ********", "pid = %s" % os.getpid(), "config file = %s" % backup_config_file, "section = %s" % backup_config_section) log.info('Config settings:', cfg.get_option()) # ================================================================== # check backup directory environment, space size and metafile if exists. # ================================================================== log.info("________ Check Backup Directory ________") print("- check backup directory.") # Path structure of backup directory: # Dest. Backup Dir/Ceph Name/Pool Name/RBD name/Circle Name/Backup Files directory = Directory(log) log.info("Set backup path:", " - backup destination path = %s" % cfg.backup_destination_path, " - ceph cluster name = %s" % cfg.ceph_cluster_name) cluster_backup_path = os.path.join(cfg.backup_destination_path, cfg.ceph_cluster_name) if not directory.exist(cluster_backup_path): cluster_backup_path = directory.create(cluster_backup_path) if cluster_backup_path == False: log.error("Fail to create directory path.") sys.exit(2) print(" set backup directory: %s" % cluster_backup_path) log.info("Get space size info '%s'." % cluster_backup_path) backup_dir_avai_bytes = directory.get_available_size(cluster_backup_path) backup_dir_used_bytes = directory.get_used_size(cluster_backup_path) if backup_dir_avai_bytes == False: log.error("Fail to get space size of directory '%s'." % cluster_backup_path) sys.exit(2) log.info("Available %s bytes, used %s bytes." % (backup_dir_avai_bytes, backup_dir_used_bytes)) print(" %s Mbytes available." % int(backup_dir_avai_bytes/1024/1024)) print(" %s Mbytes used." % int(backup_dir_used_bytes/1024/1024)) # read metadata file in backup directory # get last snapshot name and backup circle directory name log.info("Check metadata of the backup directory.") backup_meta = RBD_Backup_Metadata(log, cluster_backup_path) meta_cluster_info = backup_meta.get_cluster_info() if all (k in meta_cluster_info for k in ('name', 'fsid')): if meta_cluster_info['name'] != cfg.ceph_cluster_name: log.error("Cluster name is not match.", "name from backup directory: %s" % meta_cluster_info['name'], "name from backup config file: %s" % cfg.ceph_cluster_name) sys.exit(2) ceph_cfg = Config(cfg.ceph_conf_file, const.CEPH_CONFIG_SECTION) if not ceph_cfg.is_valid(): log.error("Unable to read ceph config.") sys.exit(2) ceph_cfg_fsid = ceph_cfg.get_option(key='fsid') if meta_cluster_info['fsid'] != ceph_cfg_fsid: log.error("Cluster fsid is not match.", "fsid from backup directory: %s" % meta_cluster_info['fsid'], "fsid from ceph config file: %s" % ceph_cfg_fsid) sys.exit(2) else: # this maybe the first time of backup # copy ceph config and keyring files to backup directory directory.copy_file(cfg.ceph_conf_file, cluster_backup_path) directory.copy_file(cfg.ceph_keyring_file, cluster_backup_path) # ================================================================== # read rbd backup list, backup list source might either from # Openstack yaml file or backup list file (yaml format) or command line # ================================================================== log.info("________ Read RBD Backup List ________") print("- check backup rbd list.") backup_option = RBD_Backup_Option(log) rbd_backup_list = RBD_Backup_List(log) rbd_name_list = {} # rbd name list : {'pool_name': ['rbd_name', ...], ...} pool_count = 0 rbd_count = 0 if is_rbd_list_from_command_line == True: log.info("Read backup list from command line.") print(" get backup list from command line input.") for rbd_list_input in args['rbd_list']: rbd_info = rbd_list_input.split("/") if len(rbd_info) == 2: pool_name = rbd_info[0] rbd_name = rbd_info[1] if not rbd_name_list.has_key(pool_name): rbd_name_list[pool_name] = [rbd_name] else: rbd_list = rbd_name_list[pool_name] if not rbd_name in rbd_list: rbd_list.append(rbd_name) rbd_name_list[pool_name] = rbd_list else: log.warning("Duplicated RBD name '%s'." % rbd_name) continue rbd_count += 1 print(" %s - %s %s" % (rbd_count, pool_name, rbd_name)) else: log.error("Invalid rbd input list. %s" % rbd_list_input) print("Error, Please input RBD name as '<pool_name>/<rbd_name>'\n" \ "For example, 3 RBDs to backup:\n" \ " rbd/rbd_a rbd/rbd_b volume/rbd_1") sys.exit(2) else: if cfg.backup_list_from_openstack_yaml_file == 'True': log.info("Read backup list from OpenStack YAML file.") print(" get backup list from OpenStack YAML file %s." % cfg.openstack_yaml_file_path) file_path = cfg.openstack_yaml_file_path section_name = cfg.openstack_yaml_section distribution = cfg.openstack_distribution ceph_pool = cfg.openstack_ceph_pool if not os.path.exists(cfg.openstack_yaml_file_path): log.error("Openstack Yaml file '%s' not exists." % cfg.backup_list_file_path) sys.exit(2) rbd_backup_list.read_openstack_yaml(cfg.openstack_yaml_file_path, cfg.openstack_yaml_section) if rbd_backup_list.set_cinder_client(distribution=cfg.openstack_distribution, api_version=cfg.api_version) volume_list = rbd_backup_list.get_cinder_volume_list() if len(volume_list) == 0: log.warning("No any matched volume ID found.") print("Info, No any matched volume ID found.") sys.exit(0) if volume_list == False: log.warning("Unable to get cinder volume ID list from openstack.") print("Error, unable to get cinder volume ID list from openstack") sys.exit(2) rbd_name_list[cfg.openstack_ceph_pool] = volume_list else: log.info("Read RBD list from backup list file.") print(" get RBD backup list from %s." % cfg.backup_list_file_path) if not os.path.exists(cfg.backup_list_file_path): log.error("Backup list file '%s' not exists." % cfg.backup_list_file_path) sys.exit(2) rbd_backup_list.read_yaml(cfg.backup_list_file_path) rbd_name_list = rbd_backup_list.get_rbd_name_list(cfg.ceph_cluster_name) if rbd_name_list == {}: log.warning("No any item in RBD backup list.") print("Info, No any item in RBD backup list.") sys.exit(0) if rbd_name_list == False: log.error("Unable to get rbd name list from backup list file.") print("Error, unable to get rbd name list from backup list file.") sys.exit(2) for pool_name, rbd_list in rbd_name_list.iteritems(): pool_count += 1 if cfg.backup_read_options == 'True': for rbd_name in rbd_list: options = rbd_backup_list.get_rbd_options(cfg.ceph_cluster_name, pool_name, rbd_name) backup_option.add_option(pool_name, rbd_name, options) rbd_count += len(rbd_list) log.info("%s RBD images to backup in pool '%s'." % (rbd_count, pool_name)) log.info("Total %s RBD images configured to backup." % rbd_count) print(" %s RBD(s) to be backuped." % rbd_count) if rbd_count == 0: sys.exit(0) # ================================================================== # check ceph cluster # examine the RBD backup list in CEPH cluster. # ignore not exist RBD in the backup list # ================================================================== log.info("________ Verify RBD Backup List ________") print("- verify RBD backup list.") valid_rbd_count = 0 #ceph_conff_file = cfg.ceph_conf_file ceph = Ceph(log, cfg.ceph_cluster_name, conffile=cfg.ceph_conf_file) if not ceph.connect_cluster(): log.error("Unable to connect ceph cluster.") # you may check user or permission to /etc/ceph directory print("Error, unable to connect ceph cluster.") sys.exit(2) ceph_fsid = ceph.get_fsid() ceph_stats = ceph.get_cluster_stats() if meta_cluster_info.has_key('fsid'): if ceph_fsid != meta_cluster_info['fsid']: log.error("Ceph fsid is not matching to the backup directory.") print("Error, the fsid from the ceph cluster is not matching " \ "to the backup directory.") sys.exit(2) log.info("Update cluster info metadata.") ceph_stats['fsid'] = ceph_fsid ceph_stats['name'] = cfg.ceph_cluster_name ceph_stats['conf'] = os.path.basename(cfg.ceph_conf_file) ceph_stats['keyr'] = os.path.basename(cfg.ceph_keyring_file) backup_meta.set_cluster_info(ceph_stats) all_pool_name_list = ceph.get_pool_list() if all_pool_name_list == False: log.error("Unable to get pool name list from ceph cluster.") print("Error, unable to get pool name list from ceph cluster.") sys.exit(2) log.info("Pool name in Ceph cluster:", all_pool_name_list) for pool_name, rbd_list in rbd_name_list.iteritems(): log.info("Check RBDs in Ceph pool '%s'." % pool_name) if pool_name not in all_pool_name_list: log.warning("Pool '%s' is not found, " \ "skip backup of the pool." % pool_name) continue if not ceph.open_ioctx(pool_name): log.warning("Unable to open ioctx of pool '%s', " \ "skip backup of the pool." % pool_name) continue pool_rbd_name_list = ceph.get_rbd_list() # rbd name list in a pool if pool_rbd_name_list == False: log.warning("Unable to get RBD list from ceph cluster, " \ "skip backup of the pool") continue # just log pool stat first pool_stat = ceph.get_pool_stat() log.info("Pool stat:", pool_stat) for rbd_name in rbd_list: log.info("Check RBD '%s'." % rbd_name) if rbd_name not in pool_rbd_name_list: log.warning("RBD '%s' is not exist." % rbd_name) continue rbd_size = ceph.get_rbd_size(rbd_name) rbd_snap = ceph.get_snap_info_list(rbd_name) # return list of (snap id, snap size, snap name) if rbd_size == False or rbd_snap == False: log.warning("Unable to get size or snapshot list of the RBD, " "skip backup of the RBD.") continue # build rbd backup list rbd_info = pack_rbd_info(pool_name, rbd_name, rbd_size, rbd_snap) backup_rbd_info_list.append(rbd_info) total_rbd_size += rbd_size valid_rbd_count += 1 print(" %s/%s - %s bytes." % (pool_name, rbd_name, rbd_size)) # compare rbd stat rbd_stat = ceph.get_rbd_stat(rbd_name) meta_rbd_stat = backup_meta.get_rbd_info(pool_name, rbd_name) if not cmp(rbd_stat, meta_rbd_stat): log.info("RBD stat has been changed.", "Old: %s" % meta_rbd_stat, "New: %s" % rbd_stat) backup_meta.set_rbd_info(pool_name, rbd_name, rbd_stat) ceph.close_ioctx() print(" %s RBD(s) can be backuped." % valid_rbd_count) log.info("Total %s bytes of RBD images size to backup." % total_rbd_size) print(" total RBDs has %s Mbytes." % int(total_rbd_size/1024/1024)) if valid_rbd_count == 0: sys.exit(0) reserve_space = backup_dir_avai_bytes * 0.01 usable_space_size = backup_dir_avai_bytes - reserve_space if total_rbd_size > usable_space_size: log.error("No enough space size for backup, stop backup work.", " - %s bytes of RBD images to backup." % total_rbd_size, " - %s bytes of usable space size (99 percents of available bytes)." % usable_space_size, " - %s bytes more required." % (total_rbd_size-usable_space_size)) print("Error, No enough space size to backup.") sys.exit(2) # ================================================================== # Verify backup types. # Set backup type of RBD, change from incremental to full backup if # a. backup_type is configured as 'full' in backup list file # b. no last snapshot record found in metafile or metafile not exist # c. last snapshot is not found in ceph cluster # d. reached max incremental backup count # ================================================================== log.info("________ Check RBD backup type ________") print("- check rbd backup type.") # store backup options rbd_backup_type = {} # RBD backup type { rbd_id: 'full' or 'incr' } full_weekday = cfg.weekly_full_backup.replace(" ","") incr_weekday = cfg.weekly_incr_backup.replace(" ","") full_weekdays = full_weekday.split(',') incr_weekdays = incr_weekday.split(',') if len(full_weekdays) == 0: log.warning("There is no full backup weekday configured.") log.info("Check default backup type for today.") weekday = str(int(datetime.datetime.today().weekday()) + 1) if weekday in full_weekdays: weekday_backup_type = const.EXPORT_TYPE[0] elif weekday in incr_weekday: weekday_backup_type = const.EXPORT_TYPE[1] else: log.info("No bacakup triggered on today (weekday=%s)." % weekday) print("Info, No bacakup triggered on today.") sys.exit(0) log.info("Backup type for today is '%s'." % weekday_backup_type) for rbd_info in backup_rbd_info_list: # you may do further manipulation of each rbd backup. # control attributed can be defined in rbd backup list file # and write your control logic in this block to overwrite configured setting pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) rbd_id = convert_rbd_id(pool_name, rbd_name) # verify backup type # -------------------------------------- log.info("Check backup type of '%s'." % rbd_id) if cfg.backup_read_options == 'True': log.info("Check backup type form backup option.") option_backup_type = backup_option.get_backup_type(pool_name, rbd_name) if option_backup_type == False: rbd_backup_type[rbd_id] = weekday_backup_type else: rbd_backup_type[rbd_id] = option_backup_type if rbd_backup_type[rbd_id] == const.EXPORT_TYPE[1]: # rbd snapshot check log.info("Check last backup snapshot.") meta_snap_info_list = backup_meta.get_backup_snapshot_list(pool_name, rbd_name) if len(meta_snap_info_list) == 0: log.warning("No snapshot list metadata found.") rbd_backup_type[rbd_id] = const.EXPORT_TYPE[0] else: meta_last_snap_info = meta_snap_info_list[-1] meta_last_snap_name = meta_last_snap_info['name'] ceph_snap_name_list = [i['name'] for i in rbd_snap] # get snap name list if meta_last_snap_name not in ceph_snap_name_list: log.warning("Snapshot name '%s' is not found in ceph cluster." % meta_last_snap_name) rbd_backup_type[rbd_id] = const.EXPORT_TYPE[0] if rbd_backup_type[rbd_id] == const.EXPORT_TYPE[1]: # backup circle directory check log.info("Check last backup circle.") meta_circle_info_list = backup_meta.get_backup_circle_list(pool_name, rbd_name) if len(meta_circle_info_list) == 0: log.warning("No circle list metadata found.") rbd_backup_type[rbd_id] = const.EXPORT_TYPE[0] else: meta_last_circle_info = meta_circle_info_list[-1] meta_last_circle_name = meta_last_circle_info['name'] if not directory.exist(cluster_backup_path, pool_name, rbd_name, meta_last_circle_name): log.warning("Last backup circle directory is not exist.") rbd_backup_type[rbd_id] = const.EXPORT_TYPE[0] if rbd_backup_type[rbd_id] == const.EXPORT_TYPE[1]: # max incremental backup count check log.info("Check max incremental backup.") max_incr_count = backup_option.get_backup_max_incr_count(pool_name, rbd_name) if max_incr_count == False: max_incr_count = cfg.backup_max_incremental if max_incr_count in [0, '0', False, 'False', None]: log.info("No max incremental backup limited.") else: file_list = directory.get_file_list(cluster_backup_path, pool_name, rbd_name, meta_last_circle_name) if len(file_list) > int(max_incr_count): log.info("Max incremental backup reached (%s/%s)." % (len(file_list), max_incr_count)) rbd_backup_type[rbd_id] = const.EXPORT_TYPE[0] log.info("Set backup type of '%s/%s' to '%s'." % (pool_name, rbd_name, rbd_backup_type[rbd_id])) print(" %s %s - %s backup." % (pool_name, rbd_name, rbd_backup_type[rbd_id])) # ================================================================== # sort rbd backup list by provisioned size of RBD, from large to small. # may implement other sorting method, # by type(full or incr first), priority, or others #(rbd loading or exist object count or created snapshot size) # ================================================================== log.info("________ Sort RBD backup list ________") print("- sort rbd backup list order.") list_sort = RBD_Backup_List_Sort(log) sorted_rbd_backup_list = list_sort.sort_by_rbd_size(backup_rbd_info_list) backup_priority = [] count = 0 for rbd_info in backup_rbd_info_list: pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) count += 1 backup_pos = "%s : %s/%s : %s" % (count, pool_name, rbd_name, rbd_size) backup_priority.append(backup_pos) print(" %s %s %s" % (pool_name, rbd_name, rbd_size)) log.info("RBD backup priority:", backup_priority) # ================================================================== # initial worker manager, task maker # ================================================================== log.info("________ Initial task worker manager and task maker ________") print("- start task workers.") manager = Manager(log, tmp_dir=const.TMP_DIR) manager.set_worker_logger(cfg.log_file_path, name=const.LOG_WORKER_LOGGER_NAME) manager.set_monitor_logger(cfg.log_file_path, name=const.LOG_MONITOR_LOGGER_NAME) manager.run_worker(count=cfg.backup_concurrent_worker_count) print(" %s worker(s) started." % manager.worker_count) # initial task maker task_maker = RBD_Backup_Task_Maker(log, cfg.ceph_cluster_name, cfg.ceph_conf_file, cfg.ceph_keyring_file) task_maker.set_export_full_type(const.EXPORT_TYPE[0]) task_maker.set_export_diff_type(const.EXPORT_TYPE[1]) # for storing task result and write to file task_result = RBD_Backup_Task_Result(log) # ================================================================== # start RBD backup procedure # ================================================================== # create and submit rbd snapshot create tasks # ---------------------------------------------------------- log.info("________ Create and submit RBD snapshot create task ________") print("- start RBD snapshot procedure.") for rbd_info in backup_rbd_info_list: pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) log.info("Creating RBD snapshot create task of '%s/%s'." % (pool_name, rbd_name)) snap_create_task = task_maker.get_rbd_snapshot_create_task(pool_name, rbd_name) if snap_create_task != False: log.info("Submit SnapshotCreateTask '%s'." % snap_create_task) if manager.add_task(snap_create_task): submitted_snap_create_task_count += 1 print(" take snapshot of %s/%s" % (pool_name, rbd_name)) else: log.error("Unable to get RBD snapshot create task.") log.info("Submitted %s snapshot create tasks." % submitted_snap_create_task_count) # verify finished snapshot tasks # ---------------------------------------------------------- log.info("________ Verify finished RBD snapshot create task ________") for i in xrange(0, submitted_snap_create_task_count): try: finished_task = manager.get_finished_task() log.info("Received finished task %s." % finished_task) pool_name = finished_task.pool_name rbd_name = finished_task.rbd_name created_snapshot_name = finished_task.snap_name created_datetime = finished_task.create_datetime result = finished_task.get_result() task_result.add_snapshort_create_result(result) log.info("Task Result: ", result) # remove rbd backup item from backup list because of failure of snapshot create. if result['return_code'] != 0: log.warning("%s is not completed. " % result['task_name'], "remove the RBD from backup list.") print(" snapshot of %s/%s failed." % (pool_name, rbd_name)) temp_rbd_backup_list = [] for rbd_info in backup_rbd_info_list: if rbd_info['pool_name'] != pool_name and rbd_info['rbd_name'] != rbd_name: temp_rbd_backup_list.append(rbd_info) backup_rbd_info_list = temp_rbd_backup_list else: log.info("%s is completed, " % result['task_name']) print(" snapshot of %s/%s completed." % (pool_name, rbd_name)) # just set default snapshot info created_snapshot_info = {'id': None, 'size': None, 'name': created_snapshot_name, 'datetime': created_datetime} if ceph.open_ioctx(pool_name): snapshot_info = ceph.get_snap_info(rbd_name, created_snapshot_name) if snapshot_info == False: log.warning("Unable to get snapshot info") else: # replace with detail info created_snapshot_info = snapshot_info else: log.error("Unable to open ioctx of pool '%s'." % pool_name) log.info("Update backup snapshot info metadata.") backup_meta.add_backup_snapshot_info(pool_name, rbd_name, created_snapshot_info) except Exception as e: log.error("Unable to verify snapshot create task. %s" % e) continue # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # todo: after create new snapshot, we may sort again the rbd_backup list # base on the size of the new created snapshot. (if supported to get used size) # #list_sort = RBD_Backup_List_Sort(log) #sorted_rbd_backup_list = list_sort.sort_by_snap_size(backup_rbd_info_list) # Create and submit export tasks # ---------------------------------------------------------- log.info("________ Create and submit RBD export task ________") print("- start RBD export procedure.") for rbd_info in backup_rbd_info_list: pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) log.info("Creating RBD export task of '%s/%s'." % (pool_name, rbd_name)) rbd_id = convert_rbd_id(pool_name, rbd_name) backup_type = rbd_backup_type[rbd_id] # get snapshot name form metadata meta_snap_info_list = backup_meta.get_backup_snapshot_list(pool_name, rbd_name) if len(meta_snap_info_list) == 0: log.warning("No last snapshot found.") continue new_created_snap_info = meta_snap_info_list[-1] # from this backup new_created_snap_name = new_created_snap_info['name'] log.info("Backup type is %s" % backup_type) if backup_type == const.EXPORT_TYPE[0]: # full # create the circle dir and get the path new_circle_name = new_created_snap_name backup_circle_path = directory.create(cluster_backup_path, pool_name, rbd_name, new_circle_name) if backup_circle_path == False: log.error("Unable to create RBD backup destination path, " \ "skip this RBD export.") continue export_task = task_maker.get_rbd_export_full_task(pool_name, rbd_name, new_created_snap_name, backup_circle_path) elif backup_type == const.EXPORT_TYPE[1]: # incr # get snapshot created from last backup last_created_snap_info = meta_snap_info_list[-2] # from last backup last_created_snap_name = last_created_snap_info['name'] # get belonging circle dir name meta_circle_info_list = backup_meta.get_backup_circle_list(pool_name, rbd_name) meta_last_circle_info = meta_circle_info_list[-1] meta_last_circle_path = meta_last_circle_info['path'] # get the circle dir path backup_circle_path = os.path.join(cluster_backup_path, pool_name, rbd_name, meta_last_circle_path) export_task = task_maker.get_rbd_export_diff_task(pool_name, rbd_name, new_created_snap_name, last_created_snap_name, backup_circle_path) else: log.warning("Unknown backup type '%s'. skip." % rbd_backup_type[rbd_id]) continue if export_task != False: log.info("Submit RBD export task '%s'" % export_task) if manager.add_task(export_task): submitted_rbd_export_task_count += 1 print(" export %s/%s" % (pool_name, rbd_name)) else: log.error("Unable to get RBD export task.") log.info("Submitted %s RBD export tasks." % submitted_rbd_export_task_count) # verify finished export tasks # ---------------------------------------------------------- log.info("________ Verify finished RBD export task ________") backup_list_info = [] for i in xrange(0, submitted_rbd_export_task_count): try: finished_task = manager.get_finished_task() log.info("Received finished task %s." % finished_task) pool_name = finished_task.pool_name rbd_name = finished_task.rbd_name snap_name = finished_task.snap_name created_datetime = finished_task.create_datetime circle_dir_name = directory.get_basename(finished_task.dest_path) result = finished_task.get_result() task_result.add_export_task_result(result) log.info("Task Result: ", result) if result['return_code'] != 0: log.warning("%s is not completed, " % result['task_name']) print(" export %s/%s failed." % (pool_name, rbd_name)) # remove export incompleted file if exist if finished_task.export_type == const.EXPORT_TYPE[0]: directory.delete(finished_task.dest_path) elif finished_task.export_type == const.EXPORT_TYPE[1]: directory.delete(finished_task.dest_filepath) else: log.info("%s is completed, " % result['task_name']) print(" export of %s/%s completed." % (pool_name, rbd_name)) if finished_task.export_type == const.EXPORT_TYPE[0]: log.info("Update backup circle info metadata.") circle_info = {'backup_name': backup_name, 'name': finished_task.dest_file, 'path': circle_dir_name, 'datetime': created_datetime} backup_meta.add_backup_circle_info(pool_name, rbd_name, circle_info) elif finished_task.export_type == const.EXPORT_TYPE[1]: log.info("Update incremental backup info metadata.") incr_info = {'backup_name': backup_name, 'name': finished_task.dest_file, 'from': finished_task.from_snap, 'to': finished_task.snap_name, 'datetime': created_datetime} backup_meta.add_backup_incremental_info(pool_name, rbd_name, circle_dir_name, incr_info) backup_list_info.append((result['return_code'], pool_name, rbd_name, circle_dir_name, snap_name)) except Exception as e: log.error("Unable to verify export task. %s" % e) continue log.info("Update backup export info metadata.") backup_meta.add_backup_info(backup_name, backup_list_info) # remove exceed snapshot # ---------------------------------------------------------- log.info("________ Delete exceed RBD snapshots ________") print("- check exceed RBD snapshot.") # reduce number of worker to 1 only for sequence exec of snapshot delete task stop_worker_count = int(cfg.backup_concurrent_worker_count) - 1 if stop_worker_count != 0: manager.stop_worker(count=stop_worker_count) for rbd_info in backup_rbd_info_list: pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) log.info("Check snapshots of RBD '%s/%s'." % (pool_name, rbd_name)) max_snap_count = backup_option.get_snapshot_retain_count(pool_name, rbd_name) if max_snap_count == False: snap_retain_count = cfg.backup_snapshot_retain_count else: snap_retain_count = max_snap_count # retrieve snapshot name to a list ceph_snap_name_list = [i['name'] for i in rbd_snap] matched_snapshot_naem_list = [] meta_snap_info_list = backup_meta.get_backup_snapshot_list(pool_name, rbd_name) for meta_snap_info in meta_snap_info_list: meta_snap_name = meta_snap_info['name'] if meta_snap_name in ceph_snap_name_list: matched_snapshot_naem_list.append(meta_snap_name) # do a trick to correct count of matched snapshot name list # add one more count for the new created snapshot before. matched_snapshot_count = (len(matched_snapshot_naem_list)+1) # create snapshot delete task. diff_count = (matched_snapshot_count - int(snap_retain_count)) if diff_count > 0: log.info("%s exceed snapshot to be deleted." % diff_count) for i in range(0, diff_count): snap_name = matched_snapshot_naem_list[i] # get snap name for matched snapshot name snap_delete_task = task_maker.get_rbd_snapshot_delete_task(pool_name, rbd_name, snap_name) if snap_create_task != False: log.info("Submit SnapshotDeleteTask '%s'" % snap_delete_task) if manager.add_task(snap_delete_task): # check result after submit the task finished_task = manager.get_finished_task() log.info("%s is completed." % (finished_task)) pool_name = finished_task.pool_name rbd_name = finished_task.rbd_name deleted_snap_name = finished_task.snap_name result = finished_task.get_result() task_result.add_snapshort_delete_result(result) log.info("Task Result: ", result) # mark deleted snapshot, if result['return_code'] != 0: log.error("%s is not completed." % result['task_name']) continue else: log.info("Update backup snapshot info metadata.") meta_snap_info_list = backup_meta.get_backup_snapshot_list(pool_name, rbd_name) for meta_snap_info in meta_snap_info_list: if meta_snap_info['name'] == deleted_snap_name: backup_meta.del_backup_snapshot_info(pool_name, rbd_name, meta_snap_info, key='name') break print(" delete snapshot %s of %s/%s" % (snap_name, pool_name, rbd_name)) submitted_snap_delete_task_count += 1 else: log.error("Unable to get RBD snapshot delete task.") else: log.info("No snapshot to be deleted.") log.info("Total deleted %s RBD snapshots." % submitted_snap_delete_task_count) # remove exceed backup circle # ---------------------------------------------------------- log.info("________ Delete exceed backup circle ________") print("- check execeed RBD backup circle.") for rbd_info in backup_rbd_info_list: try: pool_name, rbd_name, rbd_size, rbd_snap = unpack_rbd_info(rbd_info) log.info("Check backup circle of RBD '%s/%s'." % (pool_name, rbd_name)) max_circ_count = backup_option.get_backup_circle_retain_count(pool_name, rbd_name) if max_circ_count == False: circle_retain_count = cfg.backup_circle_retain_count else: circle_retain_count = max_circ_count backup_circle_dir_list = directory.get_dir_list(cluster_backup_path, pool_name, rbd_name) meta_circle_info_list = backup_meta.get_backup_circle_list(pool_name, rbd_name) circle_counter = 0 matched_circle_dir_list = [] for meta_circle_info in meta_circle_info_list: if meta_circle_info['path'] in backup_circle_dir_list: matched_circle_dir_list.append(meta_circle_info) else: log.warning("Missing circle directory '%s'." % meta_circle_info['path']) matched_circle_dir_count = len(matched_circle_dir_list) log.info("%s matched backup circle, " \ "%s backup circle to retain." % (matched_circle_dir_count, circle_retain_count)) diff_count = (matched_circle_dir_count - int(circle_retain_count)) if diff_count <= 0: log.info("No backup circle to be deleted.") continue log.info("%s exceed backup circle to be deleted." % diff_count) for i in range(0, diff_count): delete_backup_circle_info = matched_circle_dir_list[i] circle_path = delete_backup_circle_info['path'] circle_name = delete_backup_circle_info['name'] log.info("Delete backup circle dir '%s'." % circle_path) delete_circle_path = directory.delete(cluster_backup_path, pool_name, rbd_name, circle_path) if delete_circle_path == False: log.warning("Unable to delete the backup circle dir.") continue log.info("Update backup circle info metadata.") backup_meta.del_backup_circle_info(pool_name, rbd_name, delete_backup_circle_info, key='name') log.info("Update incremental backup info metadata.") backup_meta.del_backup_incremental_info(pool_name, rbd_name, circle_name) print(" delete backup circle %s of %s/%s" % (circle_name, pool_name, rbd_name)) task_result.add_backup_circle_delete_result(delete_circle_path) backup_circle_delete_count += 1 except Exception as e: log.error("Unable to complete delete of exceed backup circle. %s" % e) continue log.info("Total deleted %s backup circle directory." % backup_circle_delete_count) # finalize RBD backup # ---------------------------------------------------------- log.info("________ Finalize RBD backup ________") task_result.write_to_file(backup_name) manager.stop() ceph.disconnect_cluster() begin_backup_timestamp = get_timestamp(begin_backup_datetime) log.info("******** Ceph RBD backup complete ********", "use %s seconds " % get_elapsed_time(begin_backup_timestamp)) except Exception as e: exc_type,exc_value,exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, file=sys.stdout) if manager != None: manager.stop() if ceph != None: ceph.disconnect_cluster() sys.exit(2)
def stats(ctx): """ Show some directory stats """ directory = Directory(ctx.obj['config'].path('data:path', 'directory.json')) directory.stats()