def GetServiceKeysToContentUpdates( self, status: int, media_result: ClientMediaResult.MediaResult, filterable_tags: typing.Iterable[str], external_filterable_tags=None, external_additional_service_keys_to_tags=None): if external_filterable_tags is None: external_filterable_tags = set() if external_additional_service_keys_to_tags is None: external_additional_service_keys_to_tags = ClientTags.ServiceKeysToTags( ) filterable_tags = HydrusTags.CleanTags(filterable_tags) service_keys_to_tags = ClientTags.ServiceKeysToTags() for service_key in HG.client_controller.services_manager.GetServiceKeys( HC.REAL_TAG_SERVICES): service_additional_tags = set() if service_key in external_additional_service_keys_to_tags: service_additional_tags.update( external_additional_service_keys_to_tags[service_key]) if service_key in self._service_keys_to_service_tag_import_options: service_tag_import_options = self._service_keys_to_service_tag_import_options[ service_key] service_filterable_tags = set(filterable_tags) service_filterable_tags.update(external_filterable_tags) service_tags = service_tag_import_options.GetTags( service_key, status, media_result, service_filterable_tags, service_additional_tags) else: service_tags = service_additional_tags if len(service_tags) > 0: service_keys_to_tags[service_key] = service_tags hash = media_result.GetHash() service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags) return service_keys_to_content_updates
def _ImportFiles(self, job_key): did_work = False time_to_save = HydrusData.GetNow() + 600 num_files_imported = 0 presentation_hashes = [] presentation_hashes_fast = set() i = 0 num_total = len(self._file_seed_cache) num_total_unknown = self._file_seed_cache.GetFileSeedCount( CC.STATUS_UNKNOWN) num_total_done = num_total - num_total_unknown while True: file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN) p1 = HC.options['pause_import_folders_sync'] or self._paused p2 = HydrusThreading.IsThreadShuttingDown() p3 = job_key.IsCancelled() if file_seed is None or p1 or p2 or p3: break did_work = True if HydrusData.TimeHasPassed(time_to_save): HG.client_controller.WriteSynchronous('serialisable', self) time_to_save = HydrusData.GetNow() + 600 gauge_num_done = num_total_done + num_files_imported + 1 job_key.SetVariable( 'popup_text_1', 'importing file ' + HydrusData.ConvertValueRangeToPrettyString( gauge_num_done, num_total)) job_key.SetVariable('popup_gauge_1', (gauge_num_done, num_total)) path = file_seed.file_seed_data file_seed.ImportPath(self._file_seed_cache, self._file_import_options, limited_mimes=self._mimes) if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES: if file_seed.HasHash(): hash = file_seed.GetHash() if self._tag_import_options.HasAdditionalTags(): media_result = HG.client_controller.Read( 'media_result', hash) downloaded_tags = [] service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates( file_seed.status, media_result, downloaded_tags) # additional tags if len(service_keys_to_content_updates) > 0: HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates) service_keys_to_tags = ClientTags.ServiceKeysToTags() for (tag_service_key, filename_tagging_options) in list( self._tag_service_keys_to_filename_tagging_options. items()): if not HG.client_controller.services_manager.ServiceExists( tag_service_key): continue try: tags = filename_tagging_options.GetTags( tag_service_key, path) if len(tags) > 0: service_keys_to_tags[tag_service_key] = tags except Exception as e: HydrusData.ShowText( 'Trying to parse filename tags in the import folder "' + self._name + '" threw an error!') HydrusData.ShowException(e) if len(service_keys_to_tags) > 0: service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags) HG.client_controller.WriteSynchronous( 'content_updates', service_keys_to_content_updates) num_files_imported += 1 if hash not in presentation_hashes_fast: if file_seed.ShouldPresent(self._file_import_options): presentation_hashes.append(hash) presentation_hashes_fast.add(hash) elif file_seed.status == CC.STATUS_ERROR: HydrusData.Print( 'A file failed to import from import folder ' + self._name + ':' + path) i += 1 if i % 10 == 0: self._ActionPaths() if num_files_imported > 0: HydrusData.Print('Import folder ' + self._name + ' imported ' + HydrusData.ToHumanInt(num_files_imported) + ' files.') if len(presentation_hashes) > 0: ClientImporting.PublishPresentationHashes( self._name, presentation_hashes, self._publish_files_to_popup_button, self._publish_files_to_page) self._ActionPaths() return did_work
def test_dict_to_content_updates(self): hash = HydrusData.GenerateKey() hashes = {hash} local_key = CC.DEFAULT_LOCAL_TAG_SERVICE_KEY remote_key = HG.test_controller.example_tag_repo_service_key service_keys_to_tags = ClientTags.ServiceKeysToTags({local_key: {'a'}}) content_updates = { local_key: [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ('a', hashes)) ] } self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags), content_updates) service_keys_to_tags = ClientTags.ServiceKeysToTags( {remote_key: {'c'}}) content_updates = { remote_key: [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ('c', hashes)) ] } self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags), content_updates) service_keys_to_tags = ClientTags.ServiceKeysToTags({ local_key: ['a', 'character:b'], remote_key: ['c', 'series:d'] }) content_updates = {} content_updates[local_key] = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ('a', hashes)), HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_ADD, ('character:b', hashes)) ] content_updates[remote_key] = [ HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ('c', hashes)), HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, ('series:d', hashes)) ] self.assertEqual( HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, 'c'), HydrusData.ContentUpdate(HC.CONTENT_TYPE_MAPPINGS, HC.CONTENT_UPDATE_PEND, 'c')) self.assertEqual( ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates( {hash}, service_keys_to_tags), content_updates)