示例#1
0
 def test_deprecation(self):
     reusables.find_all_files_generator(test_root)
     reusables.find_all_files(test_root)
     reusables.count_all_files(test_root)
     reusables.archive_all("data", name="tested.zip")
     try:
         reusables.extract_all("tested.zip", "new_dir")
     except Exception:
         pass
     try:
         os.unlink("tested.zip")
         shutil.rmtree("new_dir", True)
     except OSError:
         pass
     reusables.dup_finder_generator(test_root)
示例#2
0
    def add_images(self, directory, series=""):
        """Go through a directory for all image files and ingest them"""
        for file in reusables.find_all_files_generator(directory, ext=reusables.exts.pictures):
            sha256, ext, size = self.file_info(file)
            if not self.config.ignore_duplicates and self.already_ingested(sha256):
                logger.warning("file {0} already ingested".format(file))
                continue

            self.config.image_file_inc += 1
            if self.config.image_file_inc > self.config.folder_limit:
                self.config.image_file_inc = 0
                self.config.image_dir_inc += 1
                self.session.commit()
                self.save_config()

            ingest_folder = self.config.dir_names.format(increment=self.config.image_dir_inc)

            ingest_path = os.path.join(
                ingest_folder,
                self.config.image_names.format(increment=self.config.image_file_inc, ext=ext, hash=sha256, size=size),
            )

            try:
                self.ingest(file, ingest_path, sha256, file_type="image", series=series)
            except Exception as err:
                self.save_config()
                self.session.commit()
                raise err

        self.session.commit()
        self.save_config()
示例#3
0
    def add_images(self, directory: str, tags: tuple=()):
        """
        Go through a directory for all image files and ingest them.

        :param breaker:
        :param directory:
        :param tags:
        :return:
        """

        total = 0
        # I don't use enumerate because I have to return the number at the end
        for file in reusables.find_all_files_generator(
                directory, ext=reusables.exts.pictures):
            total += 1
            if total % 20 == 0.0:
                logger.info("Ingested {0} images so far.".format(total))

            sha256, ext, size = self.file_info(file)
            if (not self.config.ignore_duplicates and
                    self.already_ingested(sha256)):
                logger.warning("file {0} already ingested".format(file))
                continue

            self.config.file_inc += 1
            if self.config.file_inc > self.config.folder_limit:
                self.config.file_inc = 0
                self.config.dir_inc += 1
                self.session.commit()
                self.save_config()

            ingest_folder = self.config.dir_names.format(
                                increment=self.config.dir_inc)

            ingest_path = os.path.join(ingest_folder,
                                       self.config.file_names.format(
                                           increment=self.config.file_inc,
                                           ext=ext,
                                           hash=sha256,
                                           size=size))

            try:
                self._ingest(file, ingest_path, sha256, tags=tags)
            except Exception as err:
                self.save_config()
                self.session.commit()
                raise err

        self.session.commit()
        self.save_config()
        return total
示例#4
0
    def add_images(self, directory, series=""):
        """Go through a directory for all image files and ingest them"""
        for file in reusables.find_all_files_generator(
                directory, ext=reusables.exts.pictures):
            sha256, ext, size = self.file_info(file)
            if not self.config.ignore_duplicates and self.already_ingested(
                    sha256):
                logger.warning("file {0} already ingested".format(file))
                continue

            self.config.image_file_inc += 1
            if self.config.image_file_inc > self.config.folder_limit:
                self.config.image_file_inc = 0
                self.config.image_dir_inc += 1
                self.session.commit()
                self.save_config()

            ingest_folder = self.config.dir_names.format(
                increment=self.config.image_dir_inc)

            ingest_path = os.path.join(
                ingest_folder,
                self.config.image_names.format(
                    increment=self.config.image_file_inc,
                    ext=ext,
                    hash=sha256,
                    size=size))

            try:
                self.ingest(file,
                            ingest_path,
                            sha256,
                            file_type="image",
                            series=series)
            except Exception as err:
                self.save_config()
                self.session.commit()
                raise err

        self.session.commit()
        self.save_config()