def generate_md5(self, button): """ Generate md5 hashes for media files. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_media() progress.set_pass(_('Generating media hashes'), length) with DbTxn(_("Set media hashes"), self.db, batch=True) as trans: for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) full_path = media_path_full(self.db, media.get_path()) md5sum = create_checksum(full_path) if not md5sum: error_msg = 'IOError: %s' % full_path self.models[5].append((error_msg, None)) progress.step() continue media.set_checksum(md5sum) self.db.commit_media(media, trans) progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def saveit(self, button): """ Commit the changes to the database """ self.update_changelist() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = len(self.changelist) progress.set_pass(_('Saving Notes'), length) with DbTxn(_("Saving Cleaned Notes"), self.db, batch=False) as trans: for changed in self.changelist: note = self.db.get_note_from_handle(changed[0]) note.set_styledtext(changed[2]) self.db.commit_note(note, trans) msg = _("Note Cleanup") trans.set_description(msg) progress.step() if progress.get_cancelled(): break self.clear_models() self.show_tabs() progress.close()
def generate_md5(self, button): """ Generate md5 hashes for media files. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_media() progress.set_pass(_('Generating media hashes'), length) with DbTxn(_("Set media hashes"), self.db, batch=True) as trans: for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) full_path = media_path_full(self.db, media.get_path()) md5sum = create_checksum(full_path) if not md5sum: error_msg = 'IOError: %s' % full_path self.models[5].append((error_msg, None)) progress.step() continue media.set_checksum(md5sum) self.db.commit_media(media, trans) progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def cleanup(self, _button): """ Cleanup Notes. """ self.clear_models() StyledText.__getitem__ = MyStyled.__getitem__ # patch in slice func progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_notes() progress.set_pass(_('Scanning Notes'), length) for handle in self.db.get_note_handles(): note = self.db.get_note_from_handle(handle) g_id = note.gramps_id stext = note.get_styledtext() optype = -1 # find the notes and do cleanup #if not stext.tags: text = StyledText(stext._string, stext._tags) # make a copy result = self.convert_to_styled(text) indx = len(self.changelist) for styledtext_tag in result.tags: if (int(styledtext_tag.name) == StyledTextTagType.HIGHLIGHT and '#FFFF00' == styledtext_tag.value): optype = ISSUE break elif int(styledtext_tag.name) == StyledTextTagType.LINK: optype = LINK while True: if optype == ISSUE: # make list of notes with errors self.models[ISSUE].append((self.preview(stext, g_id), indx)) elif stext._string != result._string: # Make list of edited notes self.models[CLEANED].append((self.preview(stext, g_id), indx)) elif optype == LINK: # make list of notes with only links self.models[LINK].append((self.preview(stext, g_id), indx)) else: break self.changelist.append((handle, stext, result)) break progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def cleanup(self, button): """ Cleanup Notes. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_notes() progress.set_pass(_('Scanning Notes'), length) for handle in self.db.get_note_handles(): note = self.db.get_note_from_handle(handle) g_id = note.gramps_id stext = note.get_styledtext() handle = handle.decode('utf-8') optype = -1 ## find the notes and do cleanup if not stext.tags: result = self.convert_to_styled(stext.string) indx = len(self.changelist) for styledtext_tag in result.tags: if int(styledtext_tag.name) == StyledTextTagType.HIGHLIGHT: optype = ISSUE break elif int(styledtext_tag.name) == StyledTextTagType.LINK: optype = LINK while True: if optype == ISSUE: # make list of notes with errors self.models[ISSUE].append((self.preview(stext, g_id), indx)) elif stext.string != result.string: # Make list of edited notes self.models[CLEANED].append((self.preview(stext, g_id), indx)) elif optype == LINK: # make list of notes with only links self.models[LINK].append((self.preview(stext, g_id), indx)) else: break self.changelist.append((handle, stext, result)) break progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def generate_md5(self, button): """ Generate md5 hashes for media files and attach them as attributes to media objects. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_media_objects() progress.set_pass(_('Generating media hashes'), length) with DbTxn(_("Set media hashes"), self.db, batch=True) as trans: for handle in self.db.get_media_object_handles(): media = self.db.get_object_from_handle(handle) full_path = media_path_full(self.db, media.get_path()) try: with io.open(full_path, 'rb') as media_file: md5sum = hashlib.md5(media_file.read()).hexdigest() except IOError as err: error_msg = '%s: %s' % (err.strerror, full_path) self.models[5].append((error_msg, None)) progress.step() continue for attr in media.get_attribute_list(): if str(attr.get_type()) == 'md5': media.remove_attribute(attr) break attr = Attribute() attr.set_type(AttributeType('md5')) attr.set_value(md5sum) media.add_attribute(attr) self.db.commit_media_object(media, trans) progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def fix_media(self, button): """ Fix paths to moved media files. """ progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) progress.set_pass(_('Fixing file paths'), len(self.moved_files)) with DbTxn(_("Fix media paths"), self.db, batch=True) as trans: for handle, new_path in self.moved_files: media = self.db.get_media_from_handle(handle) media.set_path(new_path) self.db.commit_media(media, trans) progress.step() if progress.get_cancelled(): break self.models[0].clear() self.show_tabs() progress.close()
def fix_media(self, button): """ Fix paths to moved media files. """ progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) progress.set_pass(_('Fixing file paths'), len(self.moved_files)) with DbTxn(_("Fix media paths"), self.db, batch=True) as trans: for handle, new_path in self.moved_files: media = self.db.get_object_from_handle(handle) media.set_path(new_path) self.db.commit_media_object(media, trans) progress.step() if progress.get_cancelled(): break self.models[0].clear() self.show_tabs() progress.close()
def generate_md5(self, button): """ Generate md5 hashes for media files. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_media() progress.set_pass(_('Generating media hashes'), length) with DbTxn(_("Set media hashes"), self.db, batch=True) as trans: for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) full_path = media_path_full(self.db, media.get_path()) try: with io.open(full_path, 'rb') as media_file: md5sum = hashlib.md5(media_file.read()).hexdigest() except IOError as err: error_msg = '%s: %s' % (err.strerror, full_path) self.models[5].append((error_msg, None)) progress.step() continue media.set_checksum(md5sum) self.db.commit_media(media, trans) progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def generate_md5(self, button): """ Generate md5 hashes for media files. """ self.clear_models() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = self.db.get_number_of_media() progress.set_pass(_('Generating media hashes'), length) with DbTxn(_("Set media hashes"), self.db, batch=True) as trans: for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) full_path = media_path_full(self.db, media.get_path()) try: with io.open(full_path, 'rb') as media_file: md5sum = hashlib.md5(media_file.read()).hexdigest() except IOError as err: error_msg = '%s: %s' % (err.strerror, full_path) self.models[5].append((error_msg, None)) progress.step() continue media.set_checksum(md5sum) self.db.commit_media(media, trans) progress.step() if progress.get_cancelled(): break self.show_tabs() progress.close()
def saveit(self, button): """ Commit the changes to the database """ self.update_changelist() progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = len(self.changelist) progress.set_pass(_('Saving Notes'), length) with DbTxn(_("Saving Cleaned Notes"), self.db, batch=False) as trans: for changed in self.changelist: note = self.db.get_note_from_handle(changed[0]) note.set_styledtext(changed[2]) self.db.commit_note(note, trans) msg = _("Note Cleanup") trans.set_description(msg) progress.step() if progress.get_cancelled(): break self.clear_models() self.show_tabs() progress.close()
def verify_media(self, button): """ Verify media objects have the correct path to files in the media directory. List missing files, duplicate files, and files that do not yet have a media file in Gramps. """ self.clear_models() self.moved_files = [] media_path = self.db.get_mediapath() if media_path is None: WarningDialog(self.window_name, _('Media path not set. You must set the "Base path ' 'for relative media paths" in the Preferences.'), self.window) return progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = 0 for root, dirs, files in os.walk(media_path): length += len(files) progress.set_pass(_('Finding files'), length) all_files = {} for root, dirs, files in os.walk(media_path): for file_name in files: full_path = os.path.join(root, file_name) try: with io.open(full_path, 'rb') as media_file: md5sum = hashlib.md5(media_file.read()).hexdigest() except IOError as err: error_msg = '%s: %s' % (err.strerror, full_path) self.models[5].append((error_msg, None)) progress.step() continue rel_path = relative_path(full_path, media_path) if md5sum in all_files: all_files[md5sum].append(rel_path) else: all_files[md5sum] = [rel_path] progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_media_objects() progress.set_pass(_('Checking paths'), length) in_gramps = [] for handle in self.db.get_media_object_handles(): handle = handle.decode('utf-8') media = self.db.get_object_from_handle(handle) md5sum = media.get_checksum() in_gramps.append(md5sum) # Moved files gramps_path = media.get_path() if md5sum in all_files: file_path = all_files[md5sum] if gramps_path not in file_path: if len(file_path) == 1: self.moved_files.append((handle, file_path[0])) text = '%s -> %s' % (gramps_path, file_path[0]) self.models[0].append((text, handle)) else: gramps_name = os.path.basename(gramps_path) for path in file_path: if os.path.basename(path) == gramps_name: self.moved_files.append((handle, path)) text = '%s -> %s' % (gramps_path, path) self.models[0].append((text, handle)) elif md5sum is None: text = '[%s] %s' % (media.get_gramps_id(), gramps_path) self.models[4].append((text, str(handle))) else: self.models[1].append((gramps_path, handle)) progress.step() if progress.get_cancelled(): break # Duplicate files or files not in Gramps for md5sum in all_files: if len(all_files[md5sum]) > 1: text = ', '.join(all_files[md5sum]) self.models[2].append((text, all_files[md5sum][0])) if md5sum not in in_gramps: text = ', '.join(all_files[md5sum]) self.models[3].append((text, all_files[md5sum][0])) self.show_tabs() progress.close()
class RelationTab(tool.Tool, ManagedWindow): def __init__(self, dbstate, user, options_class, name, callback=None): uistate = user.uistate self.label = _("Relation and distances with root") self.dbstate = dbstate FilterClass = GenericFilterFactory('Person') self.path = '.' filter = FilterClass() tool.Tool.__init__(self, dbstate, options_class, name) if uistate: window = Gtk.Window() window.set_default_size(880, 600) box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0) window.add(box) # dirty work-around for Gtk.HeaderBar() and FolderChooser chooser = Gtk.FileChooserDialog( _("Folder Chooser"), parent=uistate.window, action=Gtk.FileChooserAction.SELECT_FOLDER, buttons=(_('_Cancel'), Gtk.ResponseType.CANCEL, _('_Select'), Gtk.ResponseType.OK)) chooser.set_tooltip_text(_("Please, select a folder")) status = chooser.run() if status == Gtk.ResponseType.OK: # work-around 'IsADirectoryError' with self() # TypeError: invalid file: gi.FunctionInfo() self.path = chooser.get_current_folder() chooser.destroy() ManagedWindow.__init__(self, uistate, [], self.__class__) self.titles = [ (_('Rel_id'), 0, 40, INTEGER), # would be INTEGER (_('Relation'), 1, 300, str), (_('Name'), 2, 200, str), (_('up'), 3, 35, INTEGER), (_('down'), 4, 35, INTEGER), (_('Common MRA'), 5, 40, INTEGER), (_('Rank'), 6, 40, INTEGER), (_('Period'), 7, 40, str), ] treeview = Gtk.TreeView() model = ListModel(treeview, self.titles) s = Gtk.ScrolledWindow() s.add(treeview) box.pack_start(s, True, True, 0) button = Gtk.Button(label=_("Save")) button.connect("clicked", self.button_clicked) box.pack_end(button, False, True, 0) self.stats_list = [] # behavior can be different according to CPU and generation depth max_level = config.get('behavior.generation-depth') # compact and interlinked tree # single core 2.80 Ghz needs +/- 0.1 second per person if max_level >= 15: var = max_level * 0.01 elif 10 <= max_level < 15: var = max_level * 0.02 else: var = max_level * 0.025 plist = self.dbstate.db.iter_person_handles() length = self.dbstate.db.get_number_of_people() default_person = self.dbstate.db.get_default_person() if uistate: self.progress = ProgressMeter(self.label, can_cancel=True, parent=window) else: self.progress = ProgressMeter(self.label) if default_person: # rather designed for run via GUI... root_id = default_person.get_gramps_id() ancestors = rules.person.IsAncestorOf([str(root_id), True]) descendants = rules.person.IsDescendantOf([str(root_id), True]) related = rules.person.IsRelatedWith([str(root_id)]) # filtering people can be useful on some large data set # counter on filtering pass was not efficient # Not the proper solution, but a lazy one providing expected message filter.add_rule(related) self.progress.set_pass(_('Please wait, filtering...')) filtered_list = filter.apply(self.dbstate.db, plist) relationship = get_relationship_calculator() else: # TODO: provide selection widget for CLI and GUI WarningDialog(_("No default_person")) return count = 0 filtered_people = len(filtered_list) self.progress.set_pass(_('Generating relation map...'), filtered_people) if self.progress.get_cancelled(): self.progress.close() return step_one = time.clock() # init for counters for handle in filtered_list: nb = len(self.stats_list) count += 1 self.progress.step() step_two = time.clock() start = 99 if count > start: # provide a basic interface for counters need = (step_two - step_one) / count wait = need * filtered_people remain = int(wait) - int(step_two - step_one) # sorry, lazy header = _("%d/%d \n %d/%d seconds \n %d/%d \n%f|\t%f" % (count, filtered_people, remain, int(wait), nb, length, float(need), float(var))) self.progress.set_header(header) if self.progress.get_cancelled(): self.progress.close() return person = dbstate.db.get_person_from_handle(handle) timeout_one = time.clock() # for delta and timeout estimations dist = relationship.get_relationship_distance_new(dbstate.db, default_person, person, only_birth=True) timeout_two = time.clock() rank = dist[0][0] if rank == -1 or rank > max_level: # not related and ignored people continue limit = timeout_two - timeout_one expect = (limit - var) / max_level if limit > var: n = name_displayer.display(person) _LOG.debug("Sorry! '{0}' needs {1} second, \ variation = '{2}'".format(n, limit, expect)) continue else: _LOG.debug("variation = '{}'".format( limit)) # delta, see above max_level 'wall' section rel = relationship.get_one_relationship( dbstate.db, default_person, person) rel_a = dist[0][2] Ga = len(rel_a) rel_b = dist[0][4] Gb = len(rel_b) mra = 1 # m: mother; f: father if Ga > 0: for letter in rel_a: if letter == 'm': mra = mra * 2 + 1 if letter == 'f': mra = mra * 2 # design: mra gender will be often female (m: mother) if rel_a[-1] == "f" and Gb != 0: # male gender, look at spouse mra = mra + 1 name = name_displayer.display(person) # pseudo privacy; sample for DNA stuff and mapping import hashlib no_name = hashlib.sha384(name.encode() + handle.encode()).hexdigest() _LOG.info(no_name) # own internal password via handle kekule = number.get_number(Ga, Gb, rel_a, rel_b) # workaround - possible unique ID and common numbers uuid = str(uuid4()) _LOG.info("Random UUID: {}".format(uuid)) if kekule == "u": # TODO: cousin(e)s need a key kekule = 0 if kekule == "nb": # non-birth kekule = -1 try: test = int(kekule) except: # 1: related to mother; 0.x : no more girls lineage kekule = 1 period = get_timeperiod(self.dbstate.db, handle) # sometimes 'iterator' (generator) is more faster #handle_list = map(handle, filtered_list) iterator = (handle for handle in filtered_list) # experimentations; not used yet new_list = [int(kekule), int(Ga), int(Gb), int(mra), int(rank)] line = (iterator, array('b', new_list)) self.stats_list.append( (int(kekule), rel, name, int(Ga), int(Gb), int(mra), int(rank), str(period))) self.progress.close() from itertools import groupby for key, items in groupby(self.stats_list, lambda x: x[0]): for subitem in items: _LOG.info(subitem) _LOG.debug("total: {}".format(nb)) for entry in self.stats_list: if uistate: model.add(entry, entry[0]) else: print(entry) if uistate: window.show() self.set_window(window, None, self.label) self.show() def save(self): """ save action """ doc = ODSTab(len(self.stats_list)) doc.creator(self.db.get_researcher().get_name()) name = self.dbstate.db.get_default_person().get_handle() + '.ods' if self.path != '.': name = os.path.join(self.path, name) try: import io io.open(name, "w", encoding='utf8') except PermissionError or IsADirectoryError: WarningDialog(_("You do not have write rights on this folder")) return spreadsheet = TableReport(name, doc) new_titles = [] skip_columns = [] index = 0 for title in self.titles: if title == 'sort': skip_columns.append(index) else: new_titles.append(title) index += 1 spreadsheet.initialize(len(new_titles)) spreadsheet.write_table_head(new_titles) index = 0 for top in self.stats_list: spreadsheet.set_row(index % 2) index += 1 spreadsheet.write_table_data(top, skip_columns) spreadsheet.finalize() def build_menu_names(self, obj): return (self.label, None) def button_clicked(self, button): self.save()
class RelationTab(tool.Tool, ManagedWindow): def __init__(self, dbstate, user, options_class, name, callback=None): uistate = user.uistate self.label = _("Relation and distances with root") self.dbstate = dbstate FilterClass = GenericFilterFactory('Person') self.path = '.' filter = FilterClass() tool.Tool.__init__(self, dbstate, options_class, name) if uistate: window = Gtk.Window() window.set_default_size(880, 600) box = Gtk.Box(orientation=Gtk.Orientation.VERTICAL, spacing=0) window.add(box) # dirty work-around for Gtk.HeaderBar() and FolderChooser chooser = Gtk.FileChooserDialog(_("Folder Chooser"), parent=uistate.window, action=Gtk.FileChooserAction.SELECT_FOLDER, buttons=(_('_Cancel'), Gtk.ResponseType.CANCEL, _('_Select'), Gtk.ResponseType.OK)) chooser.set_tooltip_text(_("Please, select a folder")) status = chooser.run() if status == Gtk.ResponseType.OK: # work-around 'IsADirectoryError' with self() # TypeError: invalid file: gi.FunctionInfo() self.path = chooser.get_current_folder() chooser.destroy() ManagedWindow.__init__(self, uistate, [], self.__class__) self.titles = [ (_('Rel_id'), 0, 40, INTEGER), # would be INTEGER (_('Relation'), 1, 300, str), (_('Name'), 2, 200, str), (_('up'), 3, 35, INTEGER), (_('down'), 4, 35, INTEGER), (_('Common MRA'), 5, 40, INTEGER), (_('Rank'), 6, 40, INTEGER), (_('Period'), 7, 40, str), ] treeview = Gtk.TreeView() model = ListModel(treeview, self.titles) s = Gtk.ScrolledWindow() s.add(treeview) box.pack_start(s, True, True, 0) button = Gtk.Button(label=_("Save")) button.connect("clicked", self.button_clicked) box.pack_end(button, False, True, 0) self.stats_list = [] # behavior can be different according to CPU and generation depth max_level = config.get('behavior.generation-depth') # compact and interlinked tree # single core 2.80 Ghz needs +/- 0.1 second per person if max_level >= 15: var = max_level * 0.01 elif 10 <= max_level < 15: var = max_level * 0.02 else: var = max_level * 0.025 plist = self.dbstate.db.iter_person_handles() length = self.dbstate.db.get_number_of_people() default_person = self.dbstate.db.get_default_person() if uistate: self.progress = ProgressMeter(self.label, can_cancel=True, parent=window) else: self.progress = ProgressMeter(self.label) if default_person: # rather designed for run via GUI... root_id = default_person.get_gramps_id() ancestors = rules.person.IsAncestorOf([str(root_id), True]) descendants = rules.person.IsDescendantOf([str(root_id), True]) related = rules.person.IsRelatedWith([str(root_id)]) # filtering people can be useful on some large data set # counter on filtering pass was not efficient # Not the proper solution, but a lazy one providing expected message filter.add_rule(related) self.progress.set_pass(_('Please wait, filtering...')) filtered_list = filter.apply(self.dbstate.db, plist) relationship = get_relationship_calculator() else: # TODO: provide selection widget for CLI and GUI WarningDialog(_("No default_person")) return count = 0 filtered_people = len(filtered_list) self.progress.set_pass(_('Generating relation map...'), filtered_people) if self.progress.get_cancelled(): self.progress.close() return step_one = time.clock() # init for counters for handle in filtered_list: nb = len(self.stats_list) count += 1 self.progress.step() step_two = time.clock() start = 99 if count > start: # provide a basic interface for counters need = (step_two - step_one) / count wait = need * filtered_people remain = int(wait) - int(step_two - step_one) # sorry, lazy header = _("%d/%d \n %d/%d seconds \n %d/%d \n%f|\t%f" % (count, filtered_people, remain, int(wait), nb, length, float(need), float(var)) ) self.progress.set_header(header) if self.progress.get_cancelled(): self.progress.close() return person = dbstate.db.get_person_from_handle(handle) timeout_one = time.clock() # for delta and timeout estimations dist = relationship.get_relationship_distance_new( dbstate.db, default_person, person, only_birth=True) timeout_two = time.clock() rank = dist[0][0] if rank == -1 or rank > max_level: # not related and ignored people continue limit = timeout_two - timeout_one expect = (limit - var) / max_level if limit > var: n = name_displayer.display(person) _LOG.debug("Sorry! '{0}' needs {1} second, \ variation = '{2}'".format(n, limit, expect ) ) continue else: _LOG.debug("variation = '{}'".format(limit)) # delta, see above max_level 'wall' section rel = relationship.get_one_relationship( dbstate.db, default_person, person) rel_a = dist[0][2] Ga = len(rel_a) rel_b = dist[0][4] Gb = len(rel_b) mra = 1 # m: mother; f: father if Ga > 0: for letter in rel_a: if letter == 'm': mra = mra * 2 + 1 if letter == 'f': mra = mra * 2 # design: mra gender will be often female (m: mother) if rel_a[-1] == "f" and Gb != 0: # male gender, look at spouse mra = mra + 1 name = name_displayer.display(person) # pseudo privacy; sample for DNA stuff and mapping import hashlib no_name = hashlib.sha384(name.encode() + handle.encode()).hexdigest() _LOG.info(no_name) # own internal password via handle kekule = number.get_number(Ga, Gb, rel_a, rel_b) # workaround - possible unique ID and common numbers uuid = str(uuid4()) _LOG.info("Random UUID: {}".format(uuid)) if kekule == "u": # TODO: cousin(e)s need a key kekule = 0 if kekule == "nb": # non-birth kekule = -1 try: test = int(kekule) except: # 1: related to mother; 0.x : no more girls lineage kekule = 1 period = get_timeperiod(self.dbstate.db, handle) # sometimes 'iterator' (generator) is more faster #handle_list = map(handle, filtered_list) iterator = (handle for handle in filtered_list) # experimentations; not used yet new_list=[int(kekule), int(Ga), int(Gb), int(mra), int(rank)] if max_level > 7: line = (iterator, array('l', new_list)) else: line = (iterator, array('b', new_list)) self.stats_list.append((int(kekule), rel, name, int(Ga), int(Gb), int(mra), int(rank), str(period))) self.progress.close() from itertools import groupby for key, items in groupby(self.stats_list, lambda x: x[0]): for subitem in items: _LOG.info(subitem) _LOG.debug("total: {}".format(nb)) for entry in self.stats_list: if uistate: model.add(entry, entry[0]) else: print(entry) if uistate: window.show() self.set_window(window, None, self.label) self.show() def save(self): """ save action """ doc = ODSTab(len(self.stats_list)) doc.creator(self.db.get_researcher().get_name()) name = self.dbstate.db.get_default_person().get_handle() + '.ods' if self.path != '.': name = os.path.join(self.path, name) try: import io io.open(name, "w", encoding='utf8') except PermissionError or IsADirectoryError: WarningDialog(_("You do not have write rights on this folder")) return spreadsheet = TableReport(name, doc) new_titles = [] skip_columns = [] index = 0 for title in self.titles: if title == 'sort': skip_columns.append(index) else: new_titles.append(title) index += 1 spreadsheet.initialize(len(new_titles)) spreadsheet.write_table_head(new_titles) index = 0 for top in self.stats_list: spreadsheet.set_row(index%2) index += 1 spreadsheet.write_table_data(top, skip_columns) spreadsheet.finalize() def build_menu_names(self, obj): return (self.label, None) def button_clicked(self, button): self.save()
def verify_media(self, button): """ Verify media objects have the correct path to files in the media directory. List missing files, duplicate files, and files that do not yet have a media file in Gramps. """ self.clear_models() self.moved_files = [] media_path = self.db.get_mediapath() if media_path is None: WarningDialog(self.window_name, _('Media path not set. You must set the "Base path ' 'for relative media paths" in the Preferences.'), self.window) return progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = 0 for root, dirs, files in os.walk(media_path): length += len(files) progress.set_pass(_('Finding files'), length) all_files = {} for root, dirs, files in os.walk(media_path): for file_name in files: full_path = os.path.join(root, file_name) md5sum = create_checksum(full_path) if not md5sum: error_msg = 'IOError: %s' % full_path self.models[5].append((error_msg, None)) progress.step() continue rel_path = relative_path(full_path, media_path) if md5sum in all_files: all_files[md5sum].append(rel_path) else: all_files[md5sum] = [rel_path] progress.step() if progress.get_cancelled(): break # the following allows cancelling with subdirectries else: # normal exit of for file_name loop continue # just continue with outer loop break # inner loop had break, so break outer as well. length = self.db.get_number_of_media() progress.set_pass(_('Checking paths'), length) in_gramps = [] for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) md5sum = media.get_checksum() in_gramps.append(md5sum) # Moved files gramps_path = media.get_path() if md5sum in all_files: file_path = all_files[md5sum] if gramps_path not in file_path: if len(file_path) == 1: self.moved_files.append((handle, file_path[0])) text = '%s -> %s' % (gramps_path, file_path[0]) self.models[0].append((text, handle)) else: gramps_name = os.path.basename(gramps_path) for path in file_path: if os.path.basename(path) == gramps_name: self.moved_files.append((handle, path)) text = '%s -> %s' % (gramps_path, path) self.models[0].append((text, handle)) elif md5sum is None: text = '[%s] %s' % (media.get_gramps_id(), gramps_path) self.models[4].append((text, str(handle))) else: self.models[1].append((gramps_path, handle)) progress.step() if progress.get_cancelled(): break # Duplicate files or files not in Gramps for md5sum in all_files: if len(all_files[md5sum]) > 1: text = ', '.join(all_files[md5sum]) self.models[2].append((text, all_files[md5sum][0])) if md5sum not in in_gramps: text = ', '.join(all_files[md5sum]) self.models[3].append((text, all_files[md5sum][0])) self.show_tabs() progress.close()
def __init__(self, dbstate, user, options_class, name, callback=None): uistate = user.uistate tool.Tool.__init__(self, dbstate, options_class, name) self.db = dbstate.db progress = ProgressMeter(_('Thumbnail Generator'), can_cancel=True) length = self.db.get_number_of_media_objects() progress.set_pass(_('Generating media thumbnails'), length) for media in dbstate.db.iter_media_objects(): full_path = media_path_full(dbstate.db, media.get_path()) mime_type = media.get_mime_type() generate_thumbnail(full_path, mime_type) progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_people() progress.set_pass(_('Generating thumbnails for person references'), length) for person in dbstate.db.iter_people(): self.generate_thumbnails(person) progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_families() progress.set_pass(_('Generating thumbnails for family references'), length) for family in dbstate.db.iter_families(): self.generate_thumbnails(family) progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_events() progress.set_pass(_('Generating thumbnails for event references'), length) for event in dbstate.db.iter_events(): self.generate_thumbnails(event) progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_places() progress.set_pass(_('Generating thumbnails for place references'), length) for place in dbstate.db.iter_places(): self.generate_thumbnails(place) progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_sources() progress.set_pass(_('Generating thumbnails for source references'), length) for source in dbstate.db.iter_sources(): self.generate_thumbnails(source) progress.step() if progress.get_cancelled(): break progress.close()
def verify_media(self, button): """ Verify media objects have the correct path to files in the media directory. List missing files, duplicate files, and files that do not yet have a media file in Gramps. """ self.clear_models() self.moved_files = [] media_path = self.db.get_mediapath() if media_path is None: WarningDialog( self.window_name, _('Media path not set. You must set the "Base path ' 'for relative media paths" in the Preferences.'), self.window) return progress = ProgressMeter(self.window_name, can_cancel=True, parent=self.window) length = 0 for root, dirs, files in os.walk(media_path): length += len(files) progress.set_pass(_('Finding files'), length) all_files = {} for root, dirs, files in os.walk(media_path): for file_name in files: full_path = os.path.join(root, file_name) try: with io.open(full_path, 'rb') as media_file: md5sum = hashlib.md5(media_file.read()).hexdigest() except IOError as err: error_msg = '%s: %s' % (err.strerror, full_path) self.models[5].append((error_msg, None)) progress.step() continue rel_path = relative_path(full_path, media_path) if md5sum in all_files: all_files[md5sum].append(rel_path) else: all_files[md5sum] = [rel_path] progress.step() if progress.get_cancelled(): break length = self.db.get_number_of_media() progress.set_pass(_('Checking paths'), length) in_gramps = [] for handle in self.db.get_media_handles(): media = self.db.get_media_from_handle(handle) md5sum = media.get_checksum() in_gramps.append(md5sum) # Moved files gramps_path = media.get_path() if md5sum in all_files: file_path = all_files[md5sum] if gramps_path not in file_path: if len(file_path) == 1: self.moved_files.append((handle, file_path[0])) text = '%s -> %s' % (gramps_path, file_path[0]) self.models[0].append((text, handle)) else: gramps_name = os.path.basename(gramps_path) for path in file_path: if os.path.basename(path) == gramps_name: self.moved_files.append((handle, path)) text = '%s -> %s' % (gramps_path, path) self.models[0].append((text, handle)) elif md5sum is None: text = '[%s] %s' % (media.get_gramps_id(), gramps_path) self.models[4].append((text, str(handle))) else: self.models[1].append((gramps_path, handle)) progress.step() if progress.get_cancelled(): break # Duplicate files or files not in Gramps for md5sum in all_files: if len(all_files[md5sum]) > 1: text = ', '.join(all_files[md5sum]) self.models[2].append((text, all_files[md5sum][0])) if md5sum not in in_gramps: text = ', '.join(all_files[md5sum]) self.models[3].append((text, all_files[md5sum][0])) self.show_tabs() progress.close()