def DoWork( self ):
     
     if HG.view_shutdown:
         
         return
         
     
     if HC.options[ 'pause_import_folders_sync' ] or self._paused:
         
         return
         
     
     checked_folder = False
     
     did_import_file_work = False
     
     error_occured = False
     
     stop_time = HydrusData.GetNow() + 3600
     
     job_key = ClientThreading.JobKey( pausable = False, cancellable = True, stop_time = stop_time )
     
     try:
         
         if not os.path.exists( self._path ) or not os.path.isdir( self._path ):
             
             raise Exception( 'Path "' + self._path + '" does not seem to exist, or is not a directory.' )
             
         
         pubbed_job_key = False
         
         job_key.SetStatusTitle( 'import folder - ' + self._name )
         
         due_by_check_now = self._check_now
         due_by_period = self._check_regularly and HydrusData.TimeHasPassed( self._last_checked + self._period )
         
         if due_by_check_now or due_by_period:
             
             if not pubbed_job_key and self._show_working_popup:
                 
                 HG.client_controller.pub( 'message', job_key )
                 
                 pubbed_job_key = True
                 
             
             self._CheckFolder( job_key )
             
             checked_folder = True
             
         
         file_seed = self._file_seed_cache.GetNextFileSeed( CC.STATUS_UNKNOWN )
         
         if file_seed is not None:
             
             if not pubbed_job_key and self._show_working_popup:
                 
                 HG.client_controller.pub( 'message', job_key )
                 
                 pubbed_job_key = True
                 
             
             did_import_file_work = self._ImportFiles( job_key )
             
         
     except Exception as e:
         
         error_occured = True
         self._paused = True
         
         HydrusData.ShowText( 'The import folder "' + self._name + '" encountered an exception! It has been paused!' )
         HydrusData.ShowException( e )
         
     
     if checked_folder or did_import_file_work or error_occured:
         
         HG.client_controller.WriteSynchronous( 'serialisable', self )
         
     
     job_key.Delete()
    def Search(self, hash_id, max_hamming_distance):

        if max_hamming_distance == 0:

            similar_hash_ids = self._STL(
                self._c.execute(
                    'SELECT hash_id FROM shape_perceptual_hash_map WHERE phash_id IN ( SELECT phash_id FROM shape_perceptual_hash_map WHERE hash_id = ? );',
                    (hash_id, )))

            similar_hash_ids_and_distances = [
                (similar_hash_id, 0) for similar_hash_id in similar_hash_ids
            ]

        else:

            search_radius = max_hamming_distance

            top_node_result = self._c.execute(
                'SELECT phash_id FROM shape_vptree WHERE parent_id IS NULL;'
            ).fetchone()

            if top_node_result is None:

                return []

            (root_node_phash_id, ) = top_node_result

            search = self._STL(
                self._c.execute(
                    'SELECT phash FROM shape_perceptual_hashes NATURAL JOIN shape_perceptual_hash_map WHERE hash_id = ?;',
                    (hash_id, )))

            if len(search) == 0:

                return []

            similar_phash_ids_to_distances = {}

            num_cycles = 0
            total_nodes_searched = 0

            for search_phash in search:

                next_potentials = [root_node_phash_id]

                while len(next_potentials) > 0:

                    current_potentials = next_potentials
                    next_potentials = []

                    num_cycles += 1
                    total_nodes_searched += len(current_potentials)

                    for group_of_current_potentials in HydrusData.SplitListIntoChunks(
                            current_potentials, 10000):

                        # this is split into fixed lists of results of subgroups because as an iterable it was causing crashes on linux!!
                        # after investigation, it seemed to be SQLite having a problem with part of Get64BitHammingDistance touching phashes it presumably was still hanging on to
                        # the crash was in sqlite code, again presumably on subsequent fetch
                        # adding a delay in seemed to fix it as well. guess it was some memory maintenance buffer/bytes thing
                        # anyway, we now just get the whole lot of results first and then work on the whole lot
                        '''
                        #old method
                        select_statement = 'SELECT phash_id, phash, radius, inner_id, outer_id FROM shape_perceptual_hashes NATURAL JOIN shape_vptree WHERE phash_id = ?;'
                        
                        results = list( self._ExecuteManySelectSingleParam( select_statement, group_of_current_potentials ) )
                        '''

                        with HydrusDB.TemporaryIntegerTable(
                                self._c, group_of_current_potentials,
                                'phash_id') as temp_table_name:

                            # temp phash_ids to actual phashes and tree info
                            results = self._c.execute(
                                'SELECT phash_id, phash, radius, inner_id, outer_id FROM {} CROSS JOIN shape_perceptual_hashes USING ( phash_id ) CROSS JOIN shape_vptree USING ( phash_id );'
                                .format(temp_table_name)).fetchall()

                        for (node_phash_id, node_phash, node_radius,
                             inner_phash_id, outer_phash_id) in results:

                            # first check the node itself--is it similar?

                            node_hamming_distance = HydrusData.Get64BitHammingDistance(
                                search_phash, node_phash)

                            if node_hamming_distance <= search_radius:

                                if node_phash_id in similar_phash_ids_to_distances:

                                    current_distance = similar_phash_ids_to_distances[
                                        node_phash_id]

                                    similar_phash_ids_to_distances[
                                        node_phash_id] = min(
                                            node_hamming_distance,
                                            current_distance)

                                else:

                                    similar_phash_ids_to_distances[
                                        node_phash_id] = node_hamming_distance

                            # now how about its children?

                            if node_radius is not None:

                                # we have two spheres--node and search--their centers separated by node_hamming_distance
                                # we want to search inside/outside the node_sphere if the search_sphere intersects with those spaces
                                # there are four possibles:
                                # (----N----)-(--S--)    intersects with outer only - distance between N and S > their radii
                                # (----N---(-)-S--)      intersects with both
                                # (----N-(--S-)-)        intersects with both
                                # (---(-N-S--)-)         intersects with inner only - distance between N and S + radius_S does not exceed radius_N

                                if inner_phash_id is not None:

                                    spheres_disjoint = node_hamming_distance > (
                                        node_radius + search_radius)

                                    if not spheres_disjoint:  # i.e. they intersect at some point

                                        next_potentials.append(inner_phash_id)

                                if outer_phash_id is not None:

                                    search_sphere_subset_of_node_sphere = (
                                        node_hamming_distance +
                                        search_radius) <= node_radius

                                    if not search_sphere_subset_of_node_sphere:  # i.e. search sphere intersects with non-node sphere space at some point

                                        next_potentials.append(outer_phash_id)

            if HG.db_report_mode:

                HydrusData.ShowText(
                    'Similar file search touched {} nodes over {} cycles.'.
                    format(HydrusData.ToHumanInt(total_nodes_searched),
                           HydrusData.ToHumanInt(num_cycles)))

            # so, now we have phash_ids and distances. let's map that to actual files.
            # files can have multiple phashes, and phashes can refer to multiple files, so let's make sure we are setting the smallest distance we found

            similar_phash_ids = list(similar_phash_ids_to_distances.keys())

            with HydrusDB.TemporaryIntegerTable(self._c, similar_phash_ids,
                                                'phash_id') as temp_table_name:

                # temp phashes to hash map
                similar_phash_ids_to_hash_ids = HydrusData.BuildKeyToListDict(
                    self._c.execute(
                        'SELECT phash_id, hash_id FROM {} CROSS JOIN shape_perceptual_hash_map USING ( phash_id );'
                        .format(temp_table_name)))

            similar_hash_ids_to_distances = {}

            for (phash_id, hash_ids) in similar_phash_ids_to_hash_ids.items():

                distance = similar_phash_ids_to_distances[phash_id]

                for hash_id in hash_ids:

                    if hash_id not in similar_hash_ids_to_distances:

                        similar_hash_ids_to_distances[hash_id] = distance

                    else:

                        current_distance = similar_hash_ids_to_distances[
                            hash_id]

                        if distance < current_distance:

                            similar_hash_ids_to_distances[hash_id] = distance

            similar_hash_ids_and_distances = list(
                similar_hash_ids_to_distances.items())

        return similar_hash_ids_and_distances
Exemple #3
0
    def _ActionPaths(self):

        for status in (CC.STATUS_SUCCESSFUL_AND_NEW,
                       CC.STATUS_SUCCESSFUL_BUT_REDUNDANT, CC.STATUS_DELETED,
                       CC.STATUS_ERROR):

            action = self._actions[status]

            if action == CC.IMPORT_FOLDER_DELETE:

                while True:

                    file_seed = self._file_seed_cache.GetNextFileSeed(status)

                    if file_seed is None or HG.view_shutdown:

                        break

                    path = file_seed.file_seed_data

                    try:

                        if os.path.exists(path) and not os.path.isdir(path):

                            ClientPaths.DeletePath(path)

                        txt_path = path + '.txt'

                        if os.path.exists(txt_path):

                            ClientPaths.DeletePath(txt_path)

                        self._file_seed_cache.RemoveFileSeeds((file_seed, ))

                    except Exception as e:

                        raise Exception(
                            'Tried to delete "{}", but could not.'.format(
                                path))

            elif action == CC.IMPORT_FOLDER_MOVE:

                while True:

                    file_seed = self._file_seed_cache.GetNextFileSeed(status)

                    if file_seed is None or HG.view_shutdown:

                        break

                    path = file_seed.file_seed_data

                    try:

                        dest_dir = self._action_locations[status]

                        if not os.path.exists(dest_dir):

                            raise Exception(
                                'Tried to move "{}" to "{}", but the destination directory did not exist.'
                                .format(path, dest_dir))

                        if os.path.exists(path) and not os.path.isdir(path):

                            filename = os.path.basename(path)

                            dest_path = os.path.join(dest_dir, filename)

                            dest_path = HydrusPaths.AppendPathUntilNoConflicts(
                                dest_path)

                            HydrusPaths.MergeFile(path, dest_path)

                        txt_path = path + '.txt'

                        if os.path.exists(txt_path):

                            txt_filename = os.path.basename(txt_path)

                            txt_dest_path = os.path.join(
                                dest_dir, txt_filename)

                            txt_dest_path = HydrusPaths.AppendPathUntilNoConflicts(
                                txt_dest_path)

                            HydrusPaths.MergeFile(txt_path, txt_dest_path)

                        self._file_seed_cache.RemoveFileSeeds((file_seed, ))

                    except Exception as e:

                        HydrusData.ShowText('Import folder tried to move ' +
                                            path + ', but could not:')

                        HydrusData.ShowException(e)

                        HydrusData.ShowText('Import folder has been paused.')

                        self._paused = True

                        return

            elif status == CC.IMPORT_FOLDER_IGNORE:

                pass
 def __init__( self, media, target_resolution = None, init_position = 0 ):
     
     RasterContainer.__init__( self, media, target_resolution )
     
     self._init_position = init_position
     
     self._initialised = False
     
     self._renderer = None
     
     self._frames = {}
     
     self._buffer_start_index = -1
     self._buffer_end_index = -1
     
     self._times_to_play_gif = 0
     
     self._stop = False
     
     self._render_event = threading.Event()
     
     ( x, y ) = self._target_resolution
     
     new_options = HG.client_controller.new_options
     
     video_buffer_size_mb = new_options.GetInteger( 'video_buffer_size_mb' )
     
     duration = self._media.GetDuration()
     num_frames_in_video = self._media.GetNumFrames()
     
     if duration is None or duration == 0:
         
         message = 'The file with hash ' + media.GetHash().hex() + ', had an invalid duration.'
         message += os.linesep * 2
         message += 'You may wish to try regenerating its metadata through the advanced mode right-click menu.'
         
         HydrusData.ShowText( message )
         
         duration = 1.0
         
     
     if num_frames_in_video is None or num_frames_in_video == 0:
         
         message = 'The file with hash ' + media.GetHash().hex() + ', had an invalid number of frames.'
         message += os.linesep * 2
         message += 'You may wish to try regenerating its metadata through the advanced mode right-click menu.'
         
         HydrusData.ShowText( message )
         
         num_frames_in_video = 1
         
     
     self._average_frame_duration = duration / num_frames_in_video
     
     frame_buffer_length = ( video_buffer_size_mb * 1024 * 1024 ) // ( x * y * 3 )
     
     # if we can't buffer the whole vid, then don't have a clunky massive buffer
     
     max_streaming_buffer_size = max( 48, int( num_frames_in_video / ( duration / 3.0 ) ) ) # 48 or 3 seconds
     
     if max_streaming_buffer_size < frame_buffer_length and frame_buffer_length < num_frames_in_video:
         
         frame_buffer_length = max_streaming_buffer_size
         
     
     self._num_frames_backwards = frame_buffer_length * 2 // 3
     self._num_frames_forwards = frame_buffer_length // 3
     
     self._lock = threading.Lock()
     
     self._last_index_rendered = -1
     self._next_render_index = -1
     self._rendered_first_frame = False
     self._ideal_next_frame = 0
     
     HG.client_controller.CallToThread( self.THREADRender )
Exemple #5
0
    def MainLoop(self):

        while not HydrusThreading.IsThreadShuttingDown():

            time.sleep(0.00001)

            with self._lock:

                do_wait = len(self._waterfall_queue) == 0 and len(
                    self._delayed_regeneration_queue) == 0

            if do_wait:

                self._waterfall_event.wait(1)

                self._waterfall_event.clear()

            start_time = HydrusData.GetNowPrecise()
            stop_time = start_time + 0.005  # a bit of a typical frame

            page_keys_to_rendered_medias = collections.defaultdict(list)

            num_done = 0
            max_at_once = 16

            while not HydrusData.TimeHasPassedPrecise(
                    stop_time) and num_done <= max_at_once:

                with self._lock:

                    if len(self._waterfall_queue) == 0:

                        break

                    result = self._waterfall_queue.pop()

                    if len(self._waterfall_queue) == 0:

                        self._waterfall_queue_empty_event.set()

                    self._waterfall_queue_quick.discard(result)

                (page_key, media) = result

                if media.GetDisplayMedia() is not None:

                    self.GetThumbnail(media)

                    page_keys_to_rendered_medias[page_key].append(media)

                num_done += 1

            if len(page_keys_to_rendered_medias) > 0:

                for (page_key,
                     rendered_medias) in page_keys_to_rendered_medias.items():

                    self._controller.pub('waterfall_thumbnails', page_key,
                                         rendered_medias)

                time.sleep(0.00001)

            # now we will do regen if appropriate

            with self._lock:

                # got more important work or no work to do
                if len(self._waterfall_queue) > 0 or len(
                        self._delayed_regeneration_queue
                ) == 0 or HG.client_controller.CurrentlyPubSubbing():

                    continue

                media_result = self._delayed_regeneration_queue.pop()

                self._delayed_regeneration_queue_quick.discard(media_result)

            if HG.file_report_mode:

                hash = media_result.GetHash()

                HydrusData.ShowText(
                    'Thumbnail {} now regenerating from source.'.format(
                        hash.hex()))

            try:

                self._controller.files_maintenance_manager.RunJobImmediately(
                    [media_result],
                    ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL,
                    pub_job_key=False)

            except HydrusExceptions.FileMissingException:

                pass

            except Exception as e:

                hash = media_result.GetHash()

                summary = 'The thumbnail for file {} was incorrect, but a later attempt to regenerate it or load the new file back failed.'.format(
                    hash.hex())

                self._HandleThumbnailException(e, summary)
Exemple #6
0
def SetInitialTLWSizeAndPosition(tlw: QW.QWidget, frame_key):

    new_options = HG.client_controller.new_options

    (remember_size, remember_position, last_size, last_position,
     default_gravity, default_position, maximised,
     fullscreen) = new_options.GetFrameLocation(frame_key)

    parent = tlw.parentWidget()

    if parent is None:

        parent_window = None

    else:

        parent_window = parent.window()

    if remember_size and last_size is not None:

        (width, height) = last_size

        new_size = QC.QSize(width, height)

    else:

        new_size = GetSafeSize(tlw, tlw.sizeHint(), default_gravity)

    tlw.resize(new_size)

    min_width = min(240, new_size.width())
    min_height = min(240, new_size.height())

    tlw.setMinimumSize(QC.QSize(min_width, min_height))

    #

    child_position_point = QC.QPoint(CHILD_POSITION_PADDING,
                                     CHILD_POSITION_PADDING)

    desired_position = child_position_point

    we_care_about_off_screen_messages = True
    slide_up_and_left = False

    if remember_position and last_position is not None:

        (x, y) = last_position

        desired_position = QC.QPoint(x, y)

    elif default_position == 'topleft':

        if parent_window is None:

            we_care_about_off_screen_messages = False

            screen = ClientGUIFunctions.GetMouseScreen()

            if screen is not None:

                desired_position = screen.availableGeometry().topLeft(
                ) + QC.QPoint(CHILD_POSITION_PADDING, CHILD_POSITION_PADDING)

        else:

            parent_tlw = parent_window.window()

            desired_position = parent_tlw.pos() + QC.QPoint(
                CHILD_POSITION_PADDING, CHILD_POSITION_PADDING)

        slide_up_and_left = True

    elif default_position == 'center':

        if parent_window is None:

            we_care_about_off_screen_messages = False

            screen = ClientGUIFunctions.GetMouseScreen()

            if screen is not None:

                desired_position = screen.availableGeometry().center()

        else:

            desired_position = parent_window.frameGeometry().center(
            ) - tlw.rect().center()

    (safe_position, position_message) = GetSafePosition(desired_position)

    if we_care_about_off_screen_messages and position_message is not None:

        HydrusData.ShowText(position_message)

    if safe_position is not None:

        tlw.move(safe_position)

    if slide_up_and_left:

        SlideOffScreenTLWUpAndLeft(tlw)

    # Comment from before the Qt port: if these aren't callafter, the size and pos calls don't stick if a restore event happens

    if maximised:

        tlw.showMaximized()

    if fullscreen and not HC.PLATFORM_MACOS:

        tlw.showFullScreen()
Exemple #7
0
    def GetTags(self, service_key, path):

        tags = set()

        tags.update(self._tags_for_all)

        if self._load_from_neighbouring_txt_files:

            txt_path = path + '.txt'

            if os.path.exists(txt_path):

                try:

                    with open(txt_path, 'r', encoding='utf-8') as f:

                        txt_tags_string = f.read()

                except:

                    HydrusData.ShowText('Could not parse the tags from ' +
                                        txt_path + '!')

                    tags.add(
                        '___had problem reading .txt file--is it not in utf-8?'
                    )

                try:

                    txt_tags = [
                        tag for tag in HydrusText.DeserialiseNewlinedTexts(
                            txt_tags_string)
                    ]

                    if True in (len(txt_tag) > 1024 for txt_tag in txt_tags):

                        HydrusData.ShowText(
                            'Tags were too long--I think this was not a regular text file!'
                        )

                        raise Exception()

                    tags.update(txt_tags)

                except:

                    HydrusData.ShowText('Could not parse the tags from ' +
                                        txt_path + '!')

                    tags.add('___had problem parsing .txt file')

        (base, filename) = os.path.split(path)

        (filename, any_ext_gumpf) = os.path.splitext(filename)

        (filename_boolean, filename_namespace) = self._add_filename

        if filename_boolean:

            if filename_namespace != '':

                tag = filename_namespace + ':' + filename

            else:

                tag = filename

            tags.add(tag)

        (drive, directories) = os.path.splitdrive(base)

        while directories.startswith(os.path.sep):

            directories = directories[1:]

        directories = directories.split(os.path.sep)

        for (index, (dir_boolean,
                     dir_namespace)) in list(self._directories_dict.items()):

            # we are talking -3 through 2 here

            if not dir_boolean:

                continue

            try:

                directory = directories[index]

            except IndexError:

                continue

            if dir_namespace != '':

                tag = dir_namespace + ':' + directory

            else:

                tag = directory

            tags.add(tag)

        #

        for regex in self._regexes:

            try:

                result = re.findall(regex, path)

                for match in result:

                    if isinstance(match, tuple):

                        for submatch in match:

                            tags.add(submatch)

                    else:

                        tags.add(match)

            except:

                pass

        for (namespace, regex) in self._quick_namespaces:

            try:

                result = re.findall(regex, path)

                for match in result:

                    if isinstance(match, tuple):

                        for submatch in match:

                            tags.add(namespace + ':' + submatch)

                    else:

                        tags.add(namespace + ':' + match)

            except:

                pass

        #

        tags = HydrusTags.CleanTags(tags)

        tags = HG.client_controller.tag_display_manager.FilterTags(
            ClientTags.TAG_DISPLAY_STORAGE, service_key, tags)

        return tags
Exemple #8
0
    def do_it(launch_path):

        if HC.PLATFORM_WINDOWS and launch_path is None:

            os.startfile(path)

        else:

            if launch_path is None:

                launch_path = GetDefaultLaunchPath()

            complete_launch_path = launch_path.replace('%path%', path)

            hide_terminal = False

            if HC.PLATFORM_WINDOWS:

                cmd = complete_launch_path

                preexec_fn = None

            else:

                cmd = shlex.split(complete_launch_path)

                preexec_fn = getattr(os, 'setsid', None)

            if HG.subprocess_report_mode:

                message = 'Attempting to launch ' + path + ' using command ' + repr(
                    cmd) + '.'

                HydrusData.ShowText(message)

            try:

                sbp_kwargs = HydrusData.GetSubprocessKWArgs(
                    hide_terminal=hide_terminal, text=True)

                HydrusData.CheckProgramIsNotShuttingDown()

                process = subprocess.Popen(cmd,
                                           preexec_fn=preexec_fn,
                                           stdin=subprocess.PIPE,
                                           stdout=subprocess.PIPE,
                                           stderr=subprocess.PIPE,
                                           **sbp_kwargs)

                (stdout,
                 stderr) = HydrusThreading.SubprocessCommunicate(process)

                if HG.subprocess_report_mode:

                    if stdout is None and stderr is None:

                        HydrusData.ShowText('No stdout or stderr came back.')

                    if stdout is not None:

                        HydrusData.ShowText('stdout: ' + repr(stdout))

                    if stderr is not None:

                        HydrusData.ShowText('stderr: ' + repr(stderr))

            except Exception as e:

                HydrusData.ShowText(
                    'Could not launch a file! Command used was:' + os.linesep +
                    str(cmd))

                HydrusData.ShowException(e)
Exemple #9
0
    def _PopulateHashIdsToHashesCache(self,
                                      hash_ids,
                                      exception_on_error=False):

        if len(self._hash_ids_to_hashes_cache) > 100000:

            if not isinstance(hash_ids, set):

                hash_ids = set(hash_ids)

            self._hash_ids_to_hashes_cache = {
                hash_id: hash
                for (hash_id, hash) in self._hash_ids_to_hashes_cache.items()
                if hash_id in hash_ids
            }

        uncached_hash_ids = {
            hash_id
            for hash_id in hash_ids
            if hash_id not in self._hash_ids_to_hashes_cache
        }

        if len(uncached_hash_ids) > 0:

            pubbed_error = False

            if len(uncached_hash_ids) == 1:

                (uncached_hash_id, ) = uncached_hash_ids

                rows = self._Execute(
                    'SELECT hash_id, hash FROM hashes WHERE hash_id = ?;',
                    (uncached_hash_id, )).fetchall()

            else:

                with self._MakeTemporaryIntegerTable(
                        uncached_hash_ids, 'hash_id') as temp_table_name:

                    # temp hash_ids to actual hashes
                    rows = self._Execute(
                        'SELECT hash_id, hash FROM {} CROSS JOIN hashes USING ( hash_id );'
                        .format(temp_table_name)).fetchall()

            uncached_hash_ids_to_hashes = dict(rows)

            if len(uncached_hash_ids_to_hashes) < len(uncached_hash_ids):

                for hash_id in uncached_hash_ids:

                    if hash_id not in uncached_hash_ids_to_hashes:

                        if exception_on_error:

                            raise HydrusExceptions.DataMissing(
                                'Did not find all entries for those hash ids!')

                        HydrusData.DebugPrint('Database hash error: hash_id ' +
                                              str(hash_id) + ' was missing!')
                        HydrusData.PrintException(
                            Exception('Missing file identifier stack trace.'))

                        if not pubbed_error:

                            HydrusData.ShowText(
                                'A file identifier was missing! This is a serious error that means your client database has an orphan file id! Think about contacting hydrus dev!'
                            )

                            pubbed_error = True

                        hash = bytes.fromhex('aaaaaaaaaaaaaaaa') + os.urandom(
                            16)

                        uncached_hash_ids_to_hashes[hash_id] = hash

            self._hash_ids_to_hashes_cache.update(uncached_hash_ids_to_hashes)
Exemple #10
0
    def GetHashedJSONDumps(self, hashes):

        shown_missing_dump_message = False
        shown_broken_dump_message = False

        hashes_to_objs = {}

        for hash in hashes:

            result = self._Execute(
                'SELECT version, dump_type, dump FROM json_dumps_hashed WHERE hash = ?;',
                (sqlite3.Binary(hash), )).fetchone()

            if result is None:

                if not shown_missing_dump_message:

                    message = 'A hashed serialised object was missing! Its hash is "{}".'.format(
                        hash.hex())
                    message += os.linesep * 2
                    message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
                    message += os.linesep * 2
                    message += 'Your client may have lost one or more session pages.'
                    message += os.linesep * 2
                    message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'

                    HydrusData.ShowText(message)

                    shown_missing_dump_message = True

                HydrusData.Print(
                    'Was asked to fetch named JSON object "{}", but it was missing!'
                    .format(hash.hex()))

                continue

            (version, dump_type, dump) = result

            try:

                if isinstance(dump, bytes):

                    dump = str(dump, 'utf-8')

                serialisable_info = json.loads(dump)

            except:

                self._Execute('DELETE FROM json_dumps_hashed WHERE hash = ?;',
                              (sqlite3.Binary(hash), ))

                self._cursor_transaction_wrapper.CommitAndBegin()

                ExportBrokenHashedJSONDump(
                    self._db_dir, dump,
                    'hash {} dump_type {}'.format(hash.hex(), dump_type))

                if not shown_broken_dump_message:

                    message = 'A hashed serialised object failed to load! Its hash is "{}".'.format(
                        hash.hex())
                    message += os.linesep * 2
                    message += 'This error could be due to several factors, but is most likely a hard drive fault (perhaps your computer recently had a bad power cut?).'
                    message += os.linesep * 2
                    message += 'The database has attempted to delete the broken object, and the object\'s dump written to your database directory. Your client may have lost one or more session pages.'
                    message += os.linesep * 2
                    message += 'Please review the \'help my db is broke.txt\' file in your install_dir/db directory as background reading, and if the situation or fix here is not obvious, please contact hydrus dev.'

                    HydrusData.ShowText(message)

                    shown_broken_dump_message = True

                HydrusData.Print(
                    'Was asked to fetch named JSON object "{}", but it was malformed!'
                    .format(hash.hex()))

            obj = HydrusSerialisable.CreateFromSerialisableTuple(
                (dump_type, version, serialisable_info))

            hashes_to_objs[hash] = obj

        return hashes_to_objs
Exemple #11
0
    def SetJSONDump(self, obj, force_timestamp=None):

        if isinstance(obj, HydrusSerialisable.SerialisableBaseNamed):

            (dump_type, dump_name, version,
             serialisable_info) = obj.GetSerialisableTuple()

            store_backups = False
            backup_depth = 1

            if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_GUI_SESSION_CONTAINER:

                if not obj.HasAllDirtyPageData():

                    raise Exception(
                        'A session with name "{}" was set to save, but it did not have all its dirty page data!'
                        .format(dump_name))

                hashes_to_page_data = obj.GetHashesToPageData()

                self.SetHashedJSONDumps(hashes_to_page_data)

                if force_timestamp is None:

                    store_backups = True
                    backup_depth = HG.client_controller.new_options.GetInteger(
                        'number_of_gui_session_backups')

            try:

                dump = json.dumps(serialisable_info)

            except Exception as e:

                HydrusData.ShowException(e)
                HydrusData.Print(obj)
                HydrusData.Print(serialisable_info)

                raise Exception(
                    'Trying to json dump the object ' + str(obj) +
                    ' with name ' + dump_name +
                    ' caused an error. Its serialisable info has been dumped to the log.'
                )

            if force_timestamp is None:

                object_timestamp = HydrusData.GetNow()

                if store_backups:

                    existing_timestamps = sorted(
                        self._STI(
                            self._Execute(
                                'SELECT timestamp FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
                                (dump_type, dump_name))))

                    if len(existing_timestamps) > 0:

                        # the user has changed their system clock, so let's make sure the new timestamp is larger at least

                        largest_existing_timestamp = max(existing_timestamps)

                        if largest_existing_timestamp > object_timestamp:

                            object_timestamp = largest_existing_timestamp + 1

                    deletee_timestamps = existing_timestamps[:
                                                             -backup_depth]  # keep highest n values

                    deletee_timestamps.append(
                        object_timestamp
                    )  # if save gets spammed twice in one second, we'll overwrite

                    self._ExecuteMany(
                        'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ? AND timestamp = ?;',
                        [(dump_type, dump_name, timestamp)
                         for timestamp in deletee_timestamps])

                else:

                    self._Execute(
                        'DELETE FROM json_dumps_named WHERE dump_type = ? AND dump_name = ?;',
                        (dump_type, dump_name))

            else:

                object_timestamp = force_timestamp

            dump_buffer = GenerateBigSQLiteDumpBuffer(dump)

            try:

                self._Execute(
                    'INSERT INTO json_dumps_named ( dump_type, dump_name, version, timestamp, dump ) VALUES ( ?, ?, ?, ?, ? );',
                    (dump_type, dump_name, version, object_timestamp,
                     dump_buffer))

            except:

                HydrusData.DebugPrint(dump)
                HydrusData.ShowText(
                    'Had a problem saving a JSON object. The dump has been printed to the log.'
                )

                try:

                    HydrusData.Print('Dump had length {}!'.format(
                        HydrusData.ToHumanBytes(len(dump_buffer))))

                except:

                    pass

                raise

        else:

            (dump_type, version,
             serialisable_info) = obj.GetSerialisableTuple()

            if dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER:

                deletee_session_names = obj.GetDeleteeSessionNames()
                dirty_session_containers = obj.GetDirtySessionContainers()

                if len(deletee_session_names) > 0:

                    for deletee_session_name in deletee_session_names:

                        self.DeleteJSONDumpNamed(
                            HydrusSerialisable.
                            SERIALISABLE_TYPE_NETWORK_SESSION_MANAGER_SESSION_CONTAINER,
                            dump_name=deletee_session_name)

                if len(dirty_session_containers) > 0:

                    for dirty_session_container in dirty_session_containers:

                        self.SetJSONDump(dirty_session_container)

                if not obj.IsDirty():

                    return

            elif dump_type == HydrusSerialisable.SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER:

                deletee_tracker_names = obj.GetDeleteeTrackerNames()
                dirty_tracker_containers = obj.GetDirtyTrackerContainers()

                if len(deletee_tracker_names) > 0:

                    for deletee_tracker_name in deletee_tracker_names:

                        self.DeleteJSONDumpNamed(
                            HydrusSerialisable.
                            SERIALISABLE_TYPE_NETWORK_BANDWIDTH_MANAGER_TRACKER_CONTAINER,
                            dump_name=deletee_tracker_name)

                if len(dirty_tracker_containers) > 0:

                    for dirty_tracker_container in dirty_tracker_containers:

                        self.SetJSONDump(dirty_tracker_container)

                if not obj.IsDirty():

                    return

            try:

                dump = json.dumps(serialisable_info)

            except Exception as e:

                HydrusData.ShowException(e)
                HydrusData.Print(obj)
                HydrusData.Print(serialisable_info)

                raise Exception(
                    'Trying to json dump the object ' + str(obj) +
                    ' caused an error. Its serialisable info has been dumped to the log.'
                )

            self._Execute('DELETE FROM json_dumps WHERE dump_type = ?;',
                          (dump_type, ))

            dump_buffer = GenerateBigSQLiteDumpBuffer(dump)

            try:

                self._Execute(
                    'INSERT INTO json_dumps ( dump_type, version, dump ) VALUES ( ?, ?, ? );',
                    (dump_type, version, dump_buffer))

            except:

                HydrusData.DebugPrint(dump)
                HydrusData.ShowText(
                    'Had a problem saving a JSON object. The dump has been printed to the log.'
                )

                raise
Exemple #12
0
    def _DoPreCall(self):

        if HG.daemon_report_mode:

            HydrusData.ShowText(self._name + ' doing a job.')
Exemple #13
0
    def GenerateInfo(self, status_hook=None):

        if self._pre_import_file_status.mime is None:

            if status_hook is not None:

                status_hook('generating filetype')

            mime = HydrusFileHandling.GetMime(self._temp_path)

            self._pre_import_file_status.mime = mime

        else:

            mime = self._pre_import_file_status.mime

        if HG.file_import_report_mode:

            HydrusData.ShowText('File import job mime: {}'.format(
                HC.mime_string_lookup[mime]))

        new_options = HG.client_controller.new_options

        if mime in HC.DECOMPRESSION_BOMB_IMAGES and not self._file_import_options.AllowsDecompressionBombs(
        ):

            if HG.file_import_report_mode:

                HydrusData.ShowText(
                    'File import job testing for decompression bomb')

            if HydrusImageHandling.IsDecompressionBomb(self._temp_path):

                if HG.file_import_report_mode:

                    HydrusData.ShowText(
                        'File import job: it was a decompression bomb')

                raise HydrusExceptions.DecompressionBombException(
                    'Image seems to be a Decompression Bomb!')

        if status_hook is not None:

            status_hook('generating file metadata')

        self._file_info = HydrusFileHandling.GetFileInfo(self._temp_path,
                                                         mime=mime)

        (size, mime, width, height, duration, num_frames, has_audio,
         num_words) = self._file_info

        if HG.file_import_report_mode:

            HydrusData.ShowText('File import job file info: {}'.format(
                self._file_info))

        if mime in HC.MIMES_WITH_THUMBNAILS:

            if status_hook is not None:

                status_hook('generating thumbnail')

            if HG.file_import_report_mode:

                HydrusData.ShowText('File import job generating thumbnail')

            bounding_dimensions = HG.client_controller.options[
                'thumbnail_dimensions']
            thumbnail_scale_type = HG.client_controller.new_options.GetInteger(
                'thumbnail_scale_type')

            (clip_rect, target_resolution
             ) = HydrusImageHandling.GetThumbnailResolutionAndClipRegion(
                 (width, height), bounding_dimensions, thumbnail_scale_type)

            percentage_in = HG.client_controller.new_options.GetInteger(
                'video_thumbnail_percentage_in')

            try:

                self._thumbnail_bytes = HydrusFileHandling.GenerateThumbnailBytes(
                    self._temp_path,
                    target_resolution,
                    mime,
                    duration,
                    num_frames,
                    clip_rect=clip_rect,
                    percentage_in=percentage_in)

            except Exception as e:

                raise HydrusExceptions.DamagedOrUnusualFileException(
                    'Could not render a thumbnail: {}'.format(str(e)))

        if mime in HC.FILES_THAT_HAVE_PERCEPTUAL_HASH:

            if status_hook is not None:

                status_hook('generating similar files metadata')

            if HG.file_import_report_mode:

                HydrusData.ShowText(
                    'File import job generating perceptual_hashes')

            self._perceptual_hashes = ClientImageHandling.GenerateShapePerceptualHashes(
                self._temp_path, mime)

            if HG.file_import_report_mode:

                HydrusData.ShowText(
                    'File import job generated {} perceptual_hashes: {}'.
                    format(len(self._perceptual_hashes), [
                        perceptual_hash.hex()
                        for perceptual_hash in self._perceptual_hashes
                    ]))

        if HG.file_import_report_mode:

            HydrusData.ShowText('File import job generating other hashes')

        if status_hook is not None:

            status_hook('generating additional hashes')

        self._extra_hashes = HydrusFileHandling.GetExtraHashesFromPath(
            self._temp_path)

        has_icc_profile = False

        if mime in HC.FILES_THAT_CAN_HAVE_ICC_PROFILE:

            try:

                pil_image = HydrusImageHandling.RawOpenPILImage(
                    self._temp_path)

                has_icc_profile = HydrusImageHandling.HasICCProfile(pil_image)

            except:

                pass

        self._has_icc_profile = has_icc_profile

        if mime in HC.FILES_THAT_CAN_HAVE_PIXEL_HASH and duration is None:

            try:

                self._pixel_hash = HydrusImageHandling.GetImagePixelHash(
                    self._temp_path, mime)

            except:

                pass

        self._file_modified_timestamp = HydrusFileHandling.GetFileModifiedTimestamp(
            self._temp_path)
Exemple #14
0
    def DoWork(self, status_hook=None) -> FileImportStatus:

        if HG.file_import_report_mode:

            HydrusData.ShowText('File import job starting work.')

        self.GeneratePreImportHashAndStatus(status_hook=status_hook)

        if self._pre_import_file_status.ShouldImport(
                self._file_import_options):

            self.GenerateInfo(status_hook=status_hook)

            try:

                self.CheckIsGoodToImport()

                ok_to_go = True

            except HydrusExceptions.FileImportRulesException as e:

                ok_to_go = False

                not_ok_file_import_status = self._pre_import_file_status.Duplicate(
                )

                not_ok_file_import_status.status = CC.STATUS_VETOED
                not_ok_file_import_status.note = str(e)

            if ok_to_go:

                hash = self._pre_import_file_status.hash
                mime = self._pre_import_file_status.mime

                if status_hook is not None:

                    status_hook('copying file into file storage')

                HG.client_controller.client_files_manager.AddFile(
                    hash,
                    mime,
                    self._temp_path,
                    thumbnail_bytes=self._thumbnail_bytes)

                if status_hook is not None:

                    status_hook('importing to database')

                self._file_import_options.CheckReadyToImport()

                self._post_import_file_status = HG.client_controller.WriteSynchronous(
                    'import_file', self)

            else:

                self._post_import_file_status = not_ok_file_import_status

        else:

            # if the file is already in the database but not in all the desired file services, let's push content updates to make it happen
            if self._pre_import_file_status.status == CC.STATUS_SUCCESSFUL_BUT_REDUNDANT:

                media_result = HG.client_controller.Read(
                    'media_result', self._pre_import_file_status.hash)

                destination_location_context = self._file_import_options.GetDestinationLocationContext(
                )

                desired_file_service_keys = destination_location_context.current_service_keys
                current_file_service_keys = media_result.GetLocationsManager(
                ).GetCurrent()

                file_service_keys_to_add_to = set(
                    desired_file_service_keys).difference(
                        current_file_service_keys)

                if len(file_service_keys_to_add_to) > 0:

                    file_info_manager = media_result.GetFileInfoManager()
                    now = HydrusData.GetNow()

                    service_keys_to_content_updates = {}

                    for service_key in file_service_keys_to_add_to:

                        service_keys_to_content_updates[service_key] = [
                            HydrusData.ContentUpdate(HC.CONTENT_TYPE_FILES,
                                                     HC.CONTENT_UPDATE_ADD,
                                                     (file_info_manager, now))
                        ]

                    HG.client_controller.WriteSynchronous(
                        'content_updates', service_keys_to_content_updates)

            self._post_import_file_status = self._pre_import_file_status.Duplicate(
            )

        if HG.file_import_report_mode:

            HydrusData.ShowText(
                'File import job is done, now publishing content updates')

        self.PubsubContentUpdates()

        return self._post_import_file_status
Exemple #15
0
    def _RefreshUPnP(self, force_wipe=False):

        running_service_with_upnp = True in (service.GetPort() is not None and
                                             service.GetUPnPPort() is not None
                                             for service in self._services)

        if not force_wipe:

            if not running_service_with_upnp:

                return

        if running_service_with_upnp and UPNPC_IS_MISSING:

            return  # welp

        try:

            local_ip = GetLocalIP()

        except:

            return  # can't get local IP, we are wewlad atm, probably some complicated multiple network situation we'll have to deal with later

        try:

            current_mappings = GetUPnPMappings()

        except FileNotFoundError:

            if not force_wipe:

                global UPNPC_MANAGER_ERROR_PRINTED

                if not UPNPC_MANAGER_ERROR_PRINTED:

                    HydrusData.ShowText(
                        'Hydrus was set up to manage your services\' port forwards with UPnP, but the miniupnpc executable is not available. Please check install_dir/bin/upnpc_readme.txt for more details.'
                    )

                    UPNPC_MANAGER_ERROR_PRINTED = True

            return  # in this case, most likely miniupnpc could not be found, so skip for now

        except:

            return  # This IGD probably doesn't support UPnP, so don't spam the user with errors they can't fix!

        our_mappings = {
            (internal_client, internal_port): external_port
            for (description, internal_client, internal_port, external_port,
                 protocol, enabled) in current_mappings
        }

        for service in self._services:

            internal_port = service.GetPort()
            upnp_port = service.GetUPnPPort()

            if (local_ip, internal_port) in our_mappings:

                current_external_port = our_mappings[(local_ip, internal_port)]

                port_is_incorrect = upnp_port is None or upnp_port != current_external_port

                if port_is_incorrect or force_wipe:

                    RemoveUPnPMapping(current_external_port, 'TCP')

        for service in self._services:

            internal_port = service.GetPort()
            upnp_port = service.GetUPnPPort()

            if upnp_port is not None:

                service_type = service.GetServiceType()

                protocol = 'TCP'

                description = HC.service_string_lookup[
                    service_type] + ' at ' + local_ip + ':' + str(
                        internal_port)

                duration = 86400

                try:

                    AddUPnPMapping(local_ip,
                                   internal_port,
                                   upnp_port,
                                   protocol,
                                   description,
                                   duration=duration)

                except HydrusExceptions.RouterException:

                    HydrusData.Print(
                        'The UPnP Daemon tried to add {}:{}->external:{} but it failed. Please try it manually to get a full log of what happened.'
                        .format(local_ip, internal_port, upnp_port))

                    return
def VideoHasAudio(path):

    info_lines = HydrusVideoHandling.GetFFMPEGInfoLines(path)

    (audio_found, audio_format) = ParseFFMPEGAudio(info_lines)

    if not audio_found:

        return False

    # just because video metadata has an audio stream doesn't mean it has audio. some vids have silent audio streams lmao
    # so, let's read it as PCM and see if there is any noise
    # this obviously only works for single audio stream vids, we'll adapt this if someone discovers a multi-stream mkv with a silent channel that doesn't work here

    cmd = [HydrusVideoHandling.FFMPEG_PATH]

    # this is perhaps not sensible for eventual playback and I should rather go for wav file-like and feed into python 'wave' in order to maintain stereo/mono and so on and have easy chunk-reading

    cmd.extend(['-i', path, '-loglevel', 'quiet', '-f', 's16le', '-'])

    sbp_kwargs = HydrusData.GetSubprocessKWArgs()

    HydrusData.CheckProgramIsNotShuttingDown()

    try:

        process = subprocess.Popen(cmd,
                                   bufsize=65536,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   **sbp_kwargs)

    except FileNotFoundError as e:

        HydrusData.ShowText('Cannot render audio--FFMPEG not found!')

        raise

    # silent PCM data is just 00 bytes
    # every now and then, you'll get a couple ffs for some reason, but this is not legit audio data

    try:

        chunk_of_pcm_data = process.stdout.read(65536)

        while len(chunk_of_pcm_data) > 0:

            # iterating over bytes gives you ints, recall
            if True in (b != 0 and b != 255 for b in chunk_of_pcm_data):

                return True

            chunk_of_pcm_data = process.stdout.read(65536)

        return False

    finally:

        process.terminate()

        process.stdout.close()
        process.stderr.close()
    def MainLoop(self):

        try:

            self._InitDBCursor(
            )  # have to reinitialise because the thread id has changed

            self._InitDiskCache()

            self._InitCaches()

        except:

            self._DisplayCatastrophicError(traceback.format_exc())

            self._could_not_initialise = True

            return

        self._ready_to_serve_requests = True

        error_count = 0

        while not ((self._local_shutdown or HG.model_shutdown)
                   and self._jobs.empty()):

            try:

                job = self._jobs.get(timeout=1)

                self._currently_doing_job = True
                self._current_job_name = job.ToString()

                self.publish_status_update()

                try:

                    if HG.db_report_mode:

                        summary = 'Running ' + job.ToString()

                        HydrusData.ShowText(summary)

                    if HG.db_profile_mode:

                        summary = 'Profiling ' + job.ToString()

                        HydrusData.ShowText(summary)

                        HydrusData.Profile(summary, 'self._ProcessJob( job )',
                                           globals(), locals())

                    else:

                        self._ProcessJob(job)

                    error_count = 0

                except:

                    error_count += 1

                    if error_count > 5:

                        raise

                    self._jobs.put(
                        job)  # couldn't lock db; put job back on queue

                    time.sleep(5)

                self._currently_doing_job = False
                self._current_job_name = ''

                self.publish_status_update()

            except queue.Empty:

                if self._transaction_contains_writes and HydrusData.TimeHasPassed(
                        self._transaction_started +
                        self.TRANSACTION_COMMIT_TIME):

                    self._Commit()

                    self._BeginImmediate()

                    self._transaction_contains_writes = False

            if HydrusData.TimeHasPassed(
                    self._connection_timestamp + CONNECTION_REFRESH_TIME
            ):  # just to clear out the journal files

                self._InitDBCursor()

            if self._pause_and_disconnect:

                self._CloseDBCursor()

                while self._pause_and_disconnect:

                    if self._local_shutdown or HG.model_shutdown:

                        break

                    time.sleep(1)

                self._InitDBCursor()

        self._CleanUpCaches()

        self._CloseDBCursor()

        temp_path = os.path.join(self._db_dir, self._durable_temp_db_filename)

        HydrusPaths.DeletePath(temp_path)

        self._loop_finished = True
def GenerateNumPyImage(path, mime, force_pil=False) -> numpy.array:

    if HG.media_load_report_mode:

        HydrusData.ShowText('Loading media: ' + path)

    if not OPENCV_OK:

        force_pil = True

    if not force_pil:

        try:

            pil_image = RawOpenPILImage(path)

            try:

                pil_image.verify()

            except:

                raise HydrusExceptions.UnsupportedFileException()

            # I and F are some sort of 32-bit monochrome or whatever, doesn't seem to work in PIL well, with or without ICC
            if pil_image.mode not in ('I', 'F'):

                if pil_image.mode == 'LAB':

                    force_pil = True

                if HasICCProfile(pil_image):

                    if HG.media_load_report_mode:

                        HydrusData.ShowText(
                            'Image has ICC, so switching to PIL')

                    force_pil = True

        except HydrusExceptions.UnsupportedFileException:

            # pil had trouble, let's cross our fingers cv can do it
            pass

    if mime in PIL_ONLY_MIMETYPES or force_pil:

        if HG.media_load_report_mode:

            HydrusData.ShowText('Loading with PIL')

        pil_image = GeneratePILImage(path)

        numpy_image = GenerateNumPyImageFromPILImage(pil_image)

    else:

        if HG.media_load_report_mode:

            HydrusData.ShowText('Loading with OpenCV')

        if mime in (HC.IMAGE_JPEG, HC.IMAGE_TIFF):

            flags = CV_IMREAD_FLAGS_JPEG

        elif mime == HC.IMAGE_PNG:

            flags = CV_IMREAD_FLAGS_PNG

        else:

            flags = CV_IMREAD_FLAGS_WEIRD

        numpy_image = cv2.imread(path, flags=flags)

        if numpy_image is None:  # doesn't support some random stuff

            if HG.media_load_report_mode:

                HydrusData.ShowText('OpenCV Failed, loading with PIL')

            pil_image = GeneratePILImage(path)

            numpy_image = GenerateNumPyImageFromPILImage(pil_image)

        else:

            numpy_image = DequantizeNumPyImage(numpy_image)

    if NumPyImageHasOpaqueAlphaChannel(numpy_image):

        convert = cv2.COLOR_RGBA2RGB

        numpy_image = cv2.cvtColor(numpy_image, convert)

    return numpy_image
Exemple #19
0
    def OnData( self, mime_data, result ):
        
        media_dnd = isinstance( mime_data, QMimeDataHydrusFiles )
        urls_dnd = mime_data.hasUrls()
        text_dnd = mime_data.hasText()
        
        if media_dnd and self._media_callable is not None:
            
            result = mime_data.hydrusFiles()
            
            if result is not None:
                
                ( page_key, hashes ) = result
                
                if page_key is not None:
                    
                    QP.CallAfter( self._media_callable, page_key, hashes )  # callafter so we can terminate dnd event now
                    
                
            
            result = QC.Qt.MoveAction
            
            # old way of doing it that messed up discord et al
            '''
        elif mime_data.formats().count( 'application/hydrus-media' ) and self._media_callable is not None:
            
            mview = mime_data.data( 'application/hydrus-media' )

            data_bytes = mview.data()

            data_str = str( data_bytes, 'utf-8' )

            (encoded_page_key, encoded_hashes) = json.loads( data_str )

            if encoded_page_key is not None:
                
                page_key = bytes.fromhex( encoded_page_key )
                hashes = [ bytes.fromhex( encoded_hash ) for encoded_hash in encoded_hashes ]

                QP.CallAfter( self._media_callable, page_key, hashes )  # callafter so we can terminate dnd event now
                

            result = QC.Qt.MoveAction
            '''
        elif urls_dnd or text_dnd:
            
            paths = []
            urls = []
            
            if urls_dnd:
                
                dnd_items = mime_data.urls()
                
                for dnd_item in dnd_items:
                    
                    if dnd_item.isLocalFile():
                        
                        paths.append( os.path.normpath( dnd_item.toLocalFile() ) )
                        
                    else:
                        
                        urls.append( dnd_item.url() )
                        
                    
                
            else:
                
                text = mime_data.text()
                
                text_lines = HydrusText.DeserialiseNewlinedTexts( text )
                
                for text_line in text_lines:
                    
                    if text_line.startswith( 'http' ):
                        
                        urls.append( text_line )
                        
                        # ignore 'paths'
                        
                    
                
            
            if self._filenames_callable is not None:
                
                if len( paths ) > 0:
                    
                    QP.CallAfter( self._filenames_callable, paths ) # callafter to terminate dnd event now
                    
                
            
            if self._url_callable is not None:
                
                if len( urls ) > 0:
                    
                    for url in urls:
                        
                        # https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URIs
                        # data:image/png;base64,(data)
                        # so what I prob have to do here is parse the file, decode from base64 or whatever, and then write to a fresh temp location and call self._filenames_callable
                        # but I need to figure out a way to reproduce this on my own. Chrome is supposed to do it on image DnD, but didn't for me
                        if url.startswith( 'data:' ) or len( url ) > 8 * 1024:
                            
                            HydrusData.ShowText( 'This drag and drop was in the unsupported \'Data URL\' format. hydev would like to know more about this so he can fix it.' )
                            
                            continue
                            
                        
                        QP.CallAfter( self._url_callable, url ) # callafter to terminate dnd event now
                        
                    
                
            
            result = QC.Qt.IgnoreAction
            
        else:
            
            result = QC.Qt.IgnoreAction
            
        
        return result
 def _TryEndModal( self, value ):
     
     if not self.isModal(): # in some rare cases (including spammy AutoHotkey, looks like), this can be fired before the dialog can clean itself up
         
         return False
         
     
     if not self._TestValidityAndPresentVetoMessage( value ):
         
         return False
         
     
     if not self._UserIsOKToClose( value ):
         
         return False
         
     
     if value == QW.QDialog.Rejected:
         
         self.SetCancelled( True )
         
     elif value == QW.QDialog.Accepted:
         
         self._SaveOKPosition()
         
     
     self._DoClose( value )
     
     self.CleanBeforeDestroy()
     
     try:
         
         self.done( value )
         
     except Exception as e:
         
         HydrusData.ShowText( 'This dialog seems to have been unable to close for some reason. I am printing the stack to the log. The dialog may have already closed, or may attempt to close now. Please inform hydrus dev of this situation. I recommend you restart the client if you can. If the UI is locked, you will have to kill it via task manager.' )
         
         HydrusData.PrintException( e )
         
         import traceback
         
         HydrusData.DebugPrint( ''.join( traceback.format_stack() ) )
         
         try:
             
             self.close()
             
         except:
             
             HydrusData.ShowText( 'The dialog would not close on command.' )
             
         
         try:
             
             self.deleteLater()
             
         except:
             
             HydrusData.ShowText( 'The dialog would not destroy on command.' )
             
         
     
     return True
Exemple #21
0
 def _InitialiseFromSerialisableInfo( self, serialisable_info ):
     
     have_shown_load_error = False
     
     ( simple_key_simple_value_pairs, simple_key_serialisable_value_pairs, serialisable_key_simple_value_pairs, serialisable_key_serialisable_value_pairs ) = serialisable_info
     
     for ( key, value ) in simple_key_simple_value_pairs:
         
         self[ key ] = value
         
     
     for ( key, serialisable_value ) in simple_key_serialisable_value_pairs:
         
         try:
             
             value = CreateFromSerialisableTuple( serialisable_value )
             
         except HydrusExceptions.SerialisationException as e:
             
             if not have_shown_load_error:
                 
                 HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                 HydrusData.ShowException( e )
                 
                 have_shown_load_error = True
                 
             
             continue
             
         
         self[ key ] = value
         
     
     for ( serialisable_key, value ) in serialisable_key_simple_value_pairs:
         
         try:
             
             key = CreateFromSerialisableTuple( serialisable_key )
             
         except HydrusExceptions.SerialisationException as e:
             
             if not have_shown_load_error:
                 
                 HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                 HydrusData.ShowException( e )
                 
                 have_shown_load_error = True
                 
             
             continue
             
         
         self[ key ] = value
         
     
     for ( serialisable_key, serialisable_value ) in serialisable_key_serialisable_value_pairs:
         
         try:
             
             key = CreateFromSerialisableTuple( serialisable_key )
             
             value = CreateFromSerialisableTuple( serialisable_value )
             
         except HydrusExceptions.SerialisationException as e:
             
             if not have_shown_load_error:
                 
                 HydrusData.ShowText( 'An object in a dictionary could not load. It has been discarded from the dictionary. More may also have failed to load, but to stop error spam, they will go silently. Your client may be running on code versions behind its database. Depending on the severity of this error, you may need to rollback to a previous backup. If you have no backup, you may want to kill your hydrus process now to stop the cleansed dictionary being saved back to the db.' )
                 HydrusData.ShowException( e )
                 
                 have_shown_load_error = True
                 
             
             continue
             
         
         self[ key ] = value
Exemple #22
0
    def _Initialise(self):

        # do this here so we are off the main thread and can wait
        client_files_manager = HG.client_controller.client_files_manager

        self._path = client_files_manager.GetFilePath(self._hash, self._mime)

        try:

            self._numpy_image = ClientImageHandling.GenerateNumPyImage(
                self._path, self._mime)

        except Exception as e:

            HydrusData.ShowText(
                'Problem rendering image at "{}"! Error follows:'.format(
                    self._path))

            HydrusData.ShowException(e)

        if not self._this_is_for_metadata_alone:

            if self._numpy_image is None:

                m = 'There was a problem rendering the image with hash {}! It may be damaged.'.format(
                    self._hash.hex())

                m += os.linesep * 2
                m += 'Jobs to check its integrity and metadata have been scheduled. If it is damaged, it may be redownloaded or removed from the client completely. If it is not damaged, it may be fixed automatically or further action may be required.'

                HydrusData.ShowText(m)

                HG.client_controller.Write(
                    'file_maintenance_add_jobs_hashes', {self._hash},
                    ClientFiles.
                    REGENERATE_FILE_DATA_JOB_FILE_INTEGRITY_DATA_TRY_URL_ELSE_REMOVE_RECORD
                )
                HG.client_controller.Write(
                    'file_maintenance_add_jobs_hashes', {self._hash},
                    ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_METADATA)

            else:

                my_resolution_size = QC.QSize(self._resolution[0],
                                              self._resolution[1])
                my_numpy_size = QC.QSize(self._numpy_image.shape[1],
                                         self._numpy_image.shape[0])

                if my_resolution_size != my_numpy_size:

                    m = 'There was a problem rendering the image with hash {}! Hydrus thinks its resolution is {}, but it was actually {}.'.format(
                        self._hash.hex(), my_resolution_size, my_numpy_size)

                    m += os.linesep * 2
                    m += 'You may see some black squares in the image. A metadata regeneration has been scheduled, so with luck the image will fix itself soon.'

                    HydrusData.ShowText(m)

                    HG.client_controller.Write(
                        'file_maintenance_add_jobs_hashes', {self._hash},
                        ClientFiles.REGENERATE_FILE_DATA_JOB_FILE_METADATA)
Exemple #23
0
 def MainLoop( self ):
     
     try:
         
         self._InitDBCursor() # have to reinitialise because the thread id has changed
         
         self._InitCaches()
         
     except:
         
         self._DisplayCatastrophicError( traceback.format_exc() )
         
         self._could_not_initialise = True
         
         return
         
     
     self._ready_to_serve_requests = True
     
     error_count = 0
     
     while not ( ( self._local_shutdown or HG.model_shutdown ) and self._jobs.empty() ):
         
         try:
             
             job = self._jobs.get( timeout = 1 )
             
             self._currently_doing_job = True
             self._current_job_name = job.ToString()
             
             self.publish_status_update()
             
             try:
                 
                 if HG.db_report_mode:
                     
                     summary = 'Running ' + job.ToString()
                     
                     HydrusData.ShowText( summary )
                     
                 
                 if HG.db_profile_mode:
                     
                     summary = 'Profiling ' + job.ToString()
                     
                     HydrusData.Profile( summary, 'self._ProcessJob( job )', globals(), locals(), show_summary = True )
                     
                 else:
                     
                     self._ProcessJob( job )
                     
                 
                 error_count = 0
                 
             except:
                 
                 error_count += 1
                 
                 if error_count > 5:
                     
                     raise
                     
                 
                 self._jobs.put( job ) # couldn't lock db; put job back on queue
                 
                 time.sleep( 5 )
                 
             
             self._currently_doing_job = False
             self._current_job_name = ''
             
             self.publish_status_update()
             
         except queue.Empty:
             
             if self._cursor_transaction_wrapper.TimeToCommit():
                 
                 self._cursor_transaction_wrapper.CommitAndBegin()
                 
             
         
         if self._pause_and_disconnect:
             
             self._CloseDBCursor()
             
             while self._pause_and_disconnect:
                 
                 if self._local_shutdown or HG.model_shutdown:
                     
                     break
                     
                 
                 time.sleep( 1 )
                 
             
             self._InitDBCursor()
             
         
     
     self._CloseDBCursor()
     
     temp_path = os.path.join( self._db_dir, self._durable_temp_db_filename )
     
     HydrusPaths.DeletePath( temp_path )
     
     self._loop_finished = True
def GenerateNumPyImage( path, mime, force_pil = False ):
    
    if HG.media_load_report_mode:
        
        HydrusData.ShowText( 'Loading media: ' + path )
        
    
    if not OPENCV_OK:
        
        force_pil = True
        
    
    if mime in PIL_ONLY_MIMETYPES or force_pil:
        
        if HG.media_load_report_mode:
            
            HydrusData.ShowText( 'Loading with PIL' )
            
        
        pil_image = GeneratePILImage( path )
        
        numpy_image = GenerateNumPyImageFromPILImage( pil_image )
        
    else:
        
        if HG.media_load_report_mode:
            
            HydrusData.ShowText( 'Loading with OpenCV' )
            
        
        if mime == HC.IMAGE_JPEG:
            
            flags = CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION
            
        else:
            
            flags = CV_IMREAD_FLAGS_SUPPORTS_ALPHA
            
        
        numpy_image = cv2.imread( path, flags = flags )
        
        if numpy_image is None: # doesn't support static gifs and some random other stuff
            
            if HG.media_load_report_mode:
                
                HydrusData.ShowText( 'OpenCV Failed, loading with PIL' )
                
            
            pil_image = GeneratePILImage( path )
            
            numpy_image = GenerateNumPyImageFromPILImage( pil_image )
            
        else:
            
            if numpy_image.dtype == 'uint16':
                
                numpy_image //= 256
                
                numpy_image = numpy.array( numpy_image, dtype = 'uint8' )
                
            
            shape = numpy_image.shape
            
            if len( shape ) == 2:
                
                # monochrome image
                
                convert = cv2.COLOR_GRAY2RGB
                
            else:
                
                ( im_y, im_x, depth ) = shape
                
                if depth == 4:
                    
                    convert = cv2.COLOR_BGRA2RGBA
                    
                else:
                    
                    convert = cv2.COLOR_BGR2RGB
                    
                
            
            numpy_image = cv2.cvtColor( numpy_image, convert )
            
        
    
    return numpy_image
Exemple #25
0
    def _GetThumbnailHydrusBitmap(self, display_media):

        bounding_dimensions = self._controller.options['thumbnail_dimensions']

        hash = display_media.GetHash()
        mime = display_media.GetMime()

        thumbnail_mime = HC.IMAGE_JPEG

        # we don't actually know this, it comes down to detailed stuff, but since this is png vs jpeg it isn't a huge deal down in the guts of image loading
        # ain't like I am encoding EXIF rotation in my jpeg thumbs
        if mime in (HC.IMAGE_APNG, HC.IMAGE_PNG, HC.IMAGE_GIF, HC.IMAGE_ICON):

            thumbnail_mime = HC.IMAGE_PNG

        locations_manager = display_media.GetLocationsManager()

        try:

            path = self._controller.client_files_manager.GetThumbnailPath(
                display_media)

        except HydrusExceptions.FileMissingException as e:

            if locations_manager.IsLocal():

                summary = 'Unable to get thumbnail for file {}.'.format(
                    hash.hex())

                self._HandleThumbnailException(e, summary)

            return self._special_thumbs['hydrus']

        try:

            numpy_image = ClientImageHandling.GenerateNumPyImage(
                path, thumbnail_mime)

        except Exception as e:

            try:

                # file is malformed, let's force a regen
                self._controller.files_maintenance_manager.RunJobImmediately(
                    [display_media],
                    ClientFiles.REGENERATE_FILE_DATA_JOB_FORCE_THUMBNAIL,
                    pub_job_key=False)

            except Exception as e:

                summary = 'The thumbnail for file {} was not loadable. An attempt to regenerate it failed.'.format(
                    hash.hex())

                self._HandleThumbnailException(e, summary)

                return self._special_thumbs['hydrus']

            try:

                numpy_image = ClientImageHandling.GenerateNumPyImage(
                    path, thumbnail_mime)

            except Exception as e:

                summary = 'The thumbnail for file {} was not loadable. It was regenerated, but that file would not render either. Your image libraries or hard drive connection are unreliable. Please inform the hydrus developer what has happened.'.format(
                    hash.hex())

                self._HandleThumbnailException(e, summary)

                return self._special_thumbs['hydrus']

        (current_width,
         current_height) = HydrusImageHandling.GetResolutionNumPy(numpy_image)

        (media_width, media_height) = display_media.GetResolution()

        (expected_width,
         expected_height) = HydrusImageHandling.GetThumbnailResolution(
             (media_width, media_height), bounding_dimensions)

        exactly_as_expected = current_width == expected_width and current_height == expected_height

        rotation_exception = current_width == expected_height and current_height == expected_width

        correct_size = exactly_as_expected or rotation_exception

        if not correct_size:

            it_is_definitely_too_big = current_width >= expected_width and current_height >= expected_height

            if it_is_definitely_too_big:

                if HG.file_report_mode:

                    HydrusData.ShowText('Thumbnail {} too big.'.format(
                        hash.hex()))

                # the thumb we have is larger than desired. we can use it to generate what we actually want without losing significant data

                # this is _resize_, not _thumbnail_, because we already know the dimensions we want
                # and in some edge cases, doing getthumbresolution on existing thumb dimensions results in float/int conversion imprecision and you get 90px/91px regen cycles that never get fixed
                numpy_image = HydrusImageHandling.ResizeNumPyImage(
                    numpy_image, (expected_width, expected_height))

                if locations_manager.IsLocal():

                    # we have the master file, so it is safe to save our resized thumb back to disk since we can regen from source if needed

                    if HG.file_report_mode:

                        HydrusData.ShowText(
                            'Thumbnail {} too big, saving back to disk.'.
                            format(hash.hex()))

                    try:

                        try:

                            thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesNumPy(
                                numpy_image, thumbnail_mime)

                        except HydrusExceptions.CantRenderWithCVException:

                            thumbnail_bytes = HydrusImageHandling.GenerateThumbnailBytesFromStaticImagePath(
                                path, (expected_width, expected_height),
                                thumbnail_mime)

                    except:

                        summary = 'The thumbnail for file {} was too large, but an attempt to shrink it failed.'.format(
                            hash.hex())

                        self._HandleThumbnailException(e, summary)

                        return self._special_thumbs['hydrus']

                    try:

                        self._controller.client_files_manager.AddThumbnailFromBytes(
                            hash, thumbnail_bytes, silent=True)

                        self._controller.files_maintenance_manager.ClearJobs(
                            {hash}, ClientFiles.
                            REGENERATE_FILE_DATA_JOB_REFIT_THUMBNAIL)

                    except:

                        summary = 'The thumbnail for file {} was too large, but an attempt to save back the shrunk file failed.'.format(
                            hash.hex())

                        self._HandleThumbnailException(e, summary)

                        return self._special_thumbs['hydrus']

            else:

                # the thumb we have is either too small or completely messed up due to a previous ratio misparse

                media_is_same_size_as_current_thumb = current_width == media_width and current_height == media_height

                if media_is_same_size_as_current_thumb:

                    # the thumb is smaller than expected, but this is a 32x32 pixilart image or whatever, so no need to scale

                    if HG.file_report_mode:

                        HydrusData.ShowText(
                            'Thumbnail {} too small due to small source file.'.
                            format(hash.hex()))

                else:

                    numpy_image = HydrusImageHandling.ResizeNumPyImage(
                        numpy_image, (expected_width, expected_height))

                    if locations_manager.IsLocal():

                        # we have the master file, so we should regen the thumb from source

                        if HG.file_report_mode:

                            HydrusData.ShowText(
                                'Thumbnail {} too small, scheduling regeneration from source.'
                                .format(hash.hex()))

                        delayed_item = display_media.GetMediaResult()

                        with self._lock:

                            if delayed_item not in self._delayed_regeneration_queue_quick:

                                self._delayed_regeneration_queue_quick.add(
                                    delayed_item)

                                self._delayed_regeneration_queue.append(
                                    delayed_item)

                    else:

                        # we do not have the master file, so we have to scale up from what we have

                        if HG.file_report_mode:

                            HydrusData.ShowText(
                                'Thumbnail {} was too small, only scaling up due to no local source.'
                                .format(hash.hex()))

        hydrus_bitmap = ClientRendering.GenerateHydrusBitmapFromNumPyImage(
            numpy_image)

        return hydrus_bitmap
Exemple #26
0
def GenerateNumPyImage(path, mime, force_pil=False) -> numpy.array:

    if HG.media_load_report_mode:

        HydrusData.ShowText('Loading media: ' + path)

    if not OPENCV_OK:

        force_pil = True

    if not force_pil:

        try:

            pil_image = RawOpenPILImage(path)

            if HG.media_load_report_mode:

                HydrusData.ShowText('Image has ICC, so switching to PIL')

            if HasICCProfile(pil_image):

                force_pil = True

        except HydrusExceptions.UnsupportedFileException:

            # pil had trouble, let's cross our fingers cv can do it
            pass

    if mime in PIL_ONLY_MIMETYPES or force_pil:

        if HG.media_load_report_mode:

            HydrusData.ShowText('Loading with PIL')

        pil_image = GeneratePILImage(path)

        numpy_image = GenerateNumPyImageFromPILImage(pil_image)

    else:

        if HG.media_load_report_mode:

            HydrusData.ShowText('Loading with OpenCV')

        if mime == HC.IMAGE_JPEG:

            flags = CV_IMREAD_FLAGS_SUPPORTS_EXIF_REORIENTATION

        else:

            flags = CV_IMREAD_FLAGS_SUPPORTS_ALPHA

        numpy_image = cv2.imread(path, flags=flags)

        if numpy_image is None:  # doesn't support some random stuff

            if HG.media_load_report_mode:

                HydrusData.ShowText('OpenCV Failed, loading with PIL')

            pil_image = GeneratePILImage(path)

            numpy_image = GenerateNumPyImageFromPILImage(pil_image)

        else:

            numpy_image = DequantizeNumPyImage(numpy_image)

    return numpy_image
Exemple #27
0
    def _WorkOnFiles(self, page_key):

        file_seed = self._file_seed_cache.GetNextFileSeed(CC.STATUS_UNKNOWN)

        if file_seed is None:

            return

        did_substantial_work = False

        path = file_seed.file_seed_data

        with self._lock:

            self._current_action = 'importing'

        def status_hook(text):

            with self._lock:

                if len(text) > 0:

                    text = text.splitlines()[0]

                self._current_action = text

        file_seed.ImportPath(self._file_seed_cache,
                             self._file_import_options,
                             status_hook=status_hook)

        did_substantial_work = True

        if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:

            if file_seed.ShouldPresent(self._file_import_options):

                file_seed.PresentToPage(page_key)

                did_substantial_work = True

            if self._delete_after_success:

                try:

                    ClientPaths.DeletePath(path)

                except Exception as e:

                    HydrusData.ShowText('While attempting to delete ' + path +
                                        ', the following error occurred:')
                    HydrusData.ShowException(e)

                txt_path = path + '.txt'

                if os.path.exists(txt_path):

                    try:

                        ClientPaths.DeletePath(txt_path)

                    except Exception as e:

                        HydrusData.ShowText('While attempting to delete ' +
                                            txt_path +
                                            ', the following error occurred:')
                        HydrusData.ShowException(e)

        with self._lock:

            self._current_action = ''

        if did_substantial_work:

            time.sleep(
                ClientImporting.DID_SUBSTANTIAL_FILE_WORK_MINIMUM_SLEEP_TIME)
Exemple #28
0
    def RepopulateMissingSubtags(self, file_service_id, tag_service_id):

        tags_table_name = self.GetTagsTableName(file_service_id,
                                                tag_service_id)
        subtags_fts4_table_name = self.GetSubtagsFTS4TableName(
            file_service_id, tag_service_id)
        subtags_searchable_map_table_name = self.GetSubtagsSearchableMapTableName(
            file_service_id, tag_service_id)
        integer_subtags_table_name = self.GetIntegerSubtagsTableName(
            file_service_id, tag_service_id)

        missing_subtag_ids = self._STS(
            self._Execute(
                'SELECT subtag_id FROM {} EXCEPT SELECT docid FROM {};'.format(
                    tags_table_name, subtags_fts4_table_name)))

        for subtag_id in missing_subtag_ids:

            result = self._Execute(
                'SELECT subtag FROM subtags WHERE subtag_id = ?;',
                (subtag_id, )).fetchone()

            if result is None:

                continue

            (subtag, ) = result

            searchable_subtag = ClientSearch.ConvertSubtagToSearchable(subtag)

            if searchable_subtag != subtag:

                searchable_subtag_id = self.modules_tags.GetSubtagId(
                    searchable_subtag)

                self._Execute(
                    'INSERT OR IGNORE INTO {} ( subtag_id, searchable_subtag_id ) VALUES ( ?, ? );'
                    .format(subtags_searchable_map_table_name),
                    (subtag_id, searchable_subtag_id))

            #

            self._Execute(
                'INSERT OR IGNORE INTO {} ( docid, subtag ) VALUES ( ?, ? );'.
                format(subtags_fts4_table_name),
                (subtag_id, searchable_subtag))

            if subtag.isdecimal():

                try:

                    integer_subtag = int(subtag)

                    if CanCacheInteger(integer_subtag):

                        self._Execute(
                            'INSERT OR IGNORE INTO {} ( subtag_id, integer_subtag ) VALUES ( ?, ? );'
                            .format(integer_subtags_table_name),
                            (subtag_id, integer_subtag))

                except ValueError:

                    pass

        if len(missing_subtag_ids) > 0:

            HydrusData.ShowText(
                'Repopulated {} missing subtags for {}_{}.'.format(
                    HydrusData.ToHumanInt(len(missing_subtag_ids)),
                    file_service_id, tag_service_id))
Exemple #29
0
    def _ImportFiles(self, job_key):

        did_work = False

        time_to_save = HydrusData.GetNow() + 600

        num_files_imported = 0
        presentation_hashes = []
        presentation_hashes_fast = set()

        i = 0

        num_total = len(self._file_seed_cache)
        num_total_unknown = self._file_seed_cache.GetFileSeedCount(
            CC.STATUS_UNKNOWN)
        num_total_done = num_total - num_total_unknown

        while True:

            file_seed = self._file_seed_cache.GetNextFileSeed(
                CC.STATUS_UNKNOWN)

            p1 = HC.options['pause_import_folders_sync'] or self._paused
            p2 = HydrusThreading.IsThreadShuttingDown()
            p3 = job_key.IsCancelled()

            if file_seed is None or p1 or p2 or p3:

                break

            did_work = True

            if HydrusData.TimeHasPassed(time_to_save):

                HG.client_controller.WriteSynchronous('serialisable', self)

                time_to_save = HydrusData.GetNow() + 600

            gauge_num_done = num_total_done + num_files_imported + 1

            job_key.SetVariable(
                'popup_text_1',
                'importing file ' + HydrusData.ConvertValueRangeToPrettyString(
                    gauge_num_done, num_total))
            job_key.SetVariable('popup_gauge_1', (gauge_num_done, num_total))

            path = file_seed.file_seed_data

            file_seed.ImportPath(self._file_seed_cache,
                                 self._file_import_options,
                                 limited_mimes=self._mimes)

            if file_seed.status in CC.SUCCESSFUL_IMPORT_STATES:

                if file_seed.HasHash():

                    hash = file_seed.GetHash()

                    if self._tag_import_options.HasAdditionalTags():

                        media_result = HG.client_controller.Read(
                            'media_result', hash)

                        downloaded_tags = []

                        service_keys_to_content_updates = self._tag_import_options.GetServiceKeysToContentUpdates(
                            file_seed.status, media_result,
                            downloaded_tags)  # additional tags

                        if len(service_keys_to_content_updates) > 0:

                            HG.client_controller.WriteSynchronous(
                                'content_updates',
                                service_keys_to_content_updates)

                    service_keys_to_tags = ClientTags.ServiceKeysToTags()

                    for (tag_service_key, filename_tagging_options) in list(
                            self._tag_service_keys_to_filename_tagging_options.
                            items()):

                        if not HG.client_controller.services_manager.ServiceExists(
                                tag_service_key):

                            continue

                        try:

                            tags = filename_tagging_options.GetTags(
                                tag_service_key, path)

                            if len(tags) > 0:

                                service_keys_to_tags[tag_service_key] = tags

                        except Exception as e:

                            HydrusData.ShowText(
                                'Trying to parse filename tags in the import folder "'
                                + self._name + '" threw an error!')

                            HydrusData.ShowException(e)

                    if len(service_keys_to_tags) > 0:

                        service_keys_to_content_updates = ClientData.ConvertServiceKeysToTagsToServiceKeysToContentUpdates(
                            {hash}, service_keys_to_tags)

                        HG.client_controller.WriteSynchronous(
                            'content_updates', service_keys_to_content_updates)

                num_files_imported += 1

                if hash not in presentation_hashes_fast:

                    if file_seed.ShouldPresent(self._file_import_options):

                        presentation_hashes.append(hash)

                        presentation_hashes_fast.add(hash)

            elif file_seed.status == CC.STATUS_ERROR:

                HydrusData.Print(
                    'A file failed to import from import folder ' +
                    self._name + ':' + path)

            i += 1

            if i % 10 == 0:

                self._ActionPaths()

        if num_files_imported > 0:

            HydrusData.Print('Import folder ' + self._name + ' imported ' +
                             HydrusData.ToHumanInt(num_files_imported) +
                             ' files.')

            if len(presentation_hashes) > 0:

                ClientImporting.PublishPresentationHashes(
                    self._name, presentation_hashes,
                    self._publish_files_to_popup_button,
                    self._publish_files_to_page)

        self._ActionPaths()

        return did_work
Exemple #30
0
def GetFFMPEGInfoLines(path,
                       count_frames_manually=False,
                       only_first_second=False):

    # open the file in a pipe, provoke an error, read output

    cmd = [FFMPEG_PATH, "-i", path]

    if only_first_second:

        cmd.insert(1, '-t')
        cmd.insert(2, '1')

    if count_frames_manually:

        # added -an here to remove audio component, which was sometimes causing convert fails on single-frame music webms

        if HC.PLATFORM_WINDOWS:

            cmd += ["-vf", "scale=-2:120", "-an", "-f", "null", "NUL"]

        else:

            cmd += ["-vf", "scale=-2:120", "-an", "-f", "null", "/dev/null"]

    sbp_kwargs = HydrusData.GetSubprocessKWArgs()

    HydrusData.CheckProgramIsNotShuttingDown()

    try:

        process = subprocess.Popen(cmd,
                                   bufsize=10**5,
                                   stdin=subprocess.PIPE,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE,
                                   **sbp_kwargs)

    except FileNotFoundError as e:

        global FFMPEG_MISSING_ERROR_PUBBED

        if not FFMPEG_MISSING_ERROR_PUBBED:

            message = 'FFMPEG, which hydrus uses to parse and render video, was not found! This may be due to it not being available on your system, or hydrus being unable to find it.'
            message += os.linesep * 2

            if HC.PLATFORM_WINDOWS:

                message += 'You are on Windows, so there should be a copy of ffmpeg.exe in your install_dir/bin folder. If not, please check if your anti-virus has removed it and restore it through a new install.'

            else:

                message += 'If you are certain that FFMPEG is installed on your OS and accessible in your PATH, please let hydrus_dev know, as this problem is likely due to an environment problem. You may be able to solve this problem immediately by putting a static build of the ffmpeg executable in your install_dir/bin folder.'

            message += os.linesep * 2
            message += 'You can check your current FFMPEG status through help->about.'

            HydrusData.ShowText(message)

            FFMPEG_MISSING_ERROR_PUBBED = True

        raise FileNotFoundError(
            'Cannot interact with video because FFMPEG not found--are you sure it is installed? Full error: '
            + str(e))

    (stdout, stderr) = HydrusThreading.SubprocessCommunicate(process)

    data_bytes = stderr

    if len(data_bytes) == 0:

        global FFMPEG_NO_CONTENT_ERROR_PUBBED

        if not FFMPEG_NO_CONTENT_ERROR_PUBBED:

            message = 'FFMPEG, which hydrus uses to parse and render video, did not return any data on a recent file metadata check! More debug info has been written to the log.'
            message += os.linesep * 2
            message += 'You can check this info again through help->about.'

            HydrusData.ShowText(message)

            message += os.linesep * 2
            message += str(sbp_kwargs)
            message += os.linesep * 2
            message += str(os.environ)
            message += os.linesep * 2
            message += 'STDOUT Response: {}'.format(stdout)
            message += os.linesep * 2
            message += 'STDERR Response: {}'.format(stderr)

            HydrusData.DebugPrint(message)

            FFMPEG_NO_CONTENT_ERROR_PUBBED = True

        raise HydrusExceptions.DataMissing(
            'Cannot interact with video because FFMPEG did not return any content.'
        )

    del process

    (text, encoding) = HydrusText.NonFailingUnicodeDecode(data_bytes, 'utf-8')

    lines = text.splitlines()

    CheckFFMPEGError(lines)

    return lines