def __init__(self, directory: str, required: bool = False, **kwargs: Any) -> None: super().__init__(**kwargs) # Store configuration self.directory = validate_input_path( directory, file_ok=False, directory_ok=True ) self.required = required self.infiles = { get_name(f): f for f in [ validate_input_path(f, default_directory=self.directory) for f in listdir(self.directory) if f != ".DS_Store" ] if isfile(f) } # Prepare description desc = f"{self.name} {self.__class__.__name__} ({self.directory})" if self.downstream_stages is not None: if len(self.downstream_stages) >= 2: for stage in self.downstream_stages[:-1]: desc += f"\n ├─ {stage}" desc += f"\n └─ {self.downstream_stages[-1]}" self.desc = desc
def process_file_from_cl(cls, infile: str, outfile: str, **kwargs): infile = validate_input_path(infile) outfile = validate_output_path(outfile) model_infile = validate_input_path(kwargs.get("model_infile")) device = kwargs.get("device") upscaler = ESRGANProcessor.RRDBNetUpscaler(model_infile, torch.device(device)) cls.process_file(infile, outfile, upscaler=upscaler, **kwargs)
def process_file_from_cl(cls, infile: str, outfile: str, **kwargs: Any) -> None: infile = validate_input_path(infile) outfile = validate_output_path(outfile) workflow = validate_input_path(kwargs.pop("workflow"), file_ok=False, directory_ok=True) cls.process_file(infile, outfile, workflow=workflow, **kwargs)
def __init__( self, pipeline: Pipeline, directory: str, downstream_stages: Optional[Union[List[str], str]] = None, **kwargs: Any, ) -> None: # Prepare attributes self.pipeline = pipeline self.directory = validate_output_path(directory, file_ok=False, directory_ok=True) if isinstance(downstream_stages, str): downstream_stages = [downstream_stages] self.downstream_stages = downstream_stages self.infiles = [ validate_input_path(f, default_directory=self.directory) for f in listdir(self.directory) if f != ".DS_Store" ] self.infiles.sort(key=self.sort) # Prepare name and description self.name = self.__class__.__name__.lower() desc = f"source {self.__class__.__name__} ({self.directory})" if self.downstream_stages is not None: if len(self.downstream_stages) >= 2: for stage in self.downstream_stages[:-1]: desc += f"\n ├─ {stage}" desc += f"\n └─ {self.downstream_stages[-1]}" self.desc = desc
def __init__( self, wip_directory: str, source: Dict[str, Dict[str, Any]], stages: Dict[str, Dict[str, Any]], verbosity: int = 1, ) -> None: # Store configuration self.wip_directory = validate_output_path(wip_directory, file_ok=False, directory_ok=True) self.verbosity = validate_int(verbosity, min_value=0) # Load configuration sources_module = import_module("pipescaler.sources") stage_modules = [ import_module(f"pipescaler.{package}") for package in ["mergers", "processors", "sorters", "splitters"] ] # Configure source source_cls_name = list(source.keys())[0] source_args = list(source.values())[0] source_cls = getattr(sources_module, source_cls_name) self.source = source_cls(pipeline=self, **source_args) print(repr(self.source)) # Configure stages self.stages: Dict[str, Stage] = {} for stage_name, stage_conf in stages.items(): if "module" in stage_conf: module_path = validate_input_path(stage_conf.pop("module")) else: module_path = None stage_cls_name = list(stage_conf.keys())[0] stage_args = list(stage_conf.values())[0] if stage_args is None: stage_args = {} if module_path is not None: spec = spec_from_file_location( splitext(basename(module_path))[0], module_path) module = module_from_spec(spec) spec.loader.exec_module(module) stage_cls = getattr(module, stage_cls_name) else: stage_cls = None for module in stage_modules: try: stage_cls = getattr(module, stage_cls_name) except AttributeError: continue if stage_cls is None: raise AttributeError(f"Class {stage_cls_name} not found") stage = stage_cls(pipeline=self, name=stage_name, **stage_args) print(repr(stage)) self.stages[stage_name] = stage() next(self.stages[stage_name])
def process_file_in_pipeline(self, image: PipeImage) -> None: infile = validate_input_path(image.last) outfile = validate_output_path( self.pipeline.get_outfile(image, self.suffix)) if not isfile(outfile): self.process_file(infile, outfile, verbosity=self.pipeline.verbosity) image.log(self.name, outfile)
def process_file_in_pipeline(self, image: PipeImage) -> None: infile = validate_input_path(image.last) outfile = validate_output_path( f"{self.output_directory}/{image.name}.{get_ext(image.last)}") if not isfile(outfile): self.process_file(infile, outfile, verbosity=self.pipeline.verbosity) image.log(self.name, outfile)
def __init__(self, infile: str) -> None: self.infile = validate_input_path(infile) self.name = get_name(self.infile) self.ext = get_ext(self.infile) image = Image.open(self.infile) self.mode: str = image.mode self.shape: Tuple[int] = image.size self.history = []
def __init__(self, conf_file: str, **kwargs: Any) -> None: """ Initializes. Args: conf_file (str): file from which to load configuration """ super().__init__(**kwargs) # Input with open(validate_input_path(conf_file), "r") as f: conf = yaml.load(f, Loader=yaml.SafeLoader) self.pipeline = Pipeline(verbosity=self.verbosity, **conf)
def __init__(self, workflow: str, **kwargs: Any) -> None: super().__init__(**kwargs) # Store configuration self.workflow = validate_input_path( workflow if workflow.endswith(".workflow") else f"{workflow}.workflow", file_ok=False, directory_ok=True, default_directory=join(*split(package_root), "data", "workflows"), ) # Prepare description desc = f"{self.name} {self.__class__.__name__} ({self.workflow})" if self.downstream_stages is not None: if len(self.downstream_stages) >= 2: for stage in self.downstream_stages[:-1]: desc += f"\n ├─ {stage}" desc += f"\n └─ {self.downstream_stages[-1]}" self.desc = desc
def process_file_from_cl(cls, infile: str, outfile: str, **kwargs: Any) -> None: infile = validate_input_path(infile) outfile = validate_output_path(outfile) cls.process_file(infile, outfile, **kwargs)
def __init__(self, forks: Dict[str, Dict[str, Any]], **kwargs: Any) -> None: super().__init__(**kwargs) # Store configuration desc = f"{self.name} {self.__class__.__name__}" # Organize downstream forks forks_by_filename = {} default_fork_name = None default_downstream_stages = None for fork_name, fork_conf in forks.items(): if fork_conf is None: fork_conf = {} # Parse filenames for this fork input_filenames = fork_conf.get("filenames") if input_filenames is None: if default_fork_name is not None: raise ValueError( "At most one configuration may omit 'filenames' and will be " "used as the default fork; two or more have been provided." ) default_fork_name = fork_name downstream_stages = fork_conf.get("downstream_stages") if isinstance(downstream_stages, str): downstream_stages = [downstream_stages] default_downstream_stages = downstream_stages continue if isinstance(input_filenames, str): input_filenames = [input_filenames] filenames = set() for input_filename in input_filenames: try: input_filename = validate_input_path( input_filename, file_ok=True, directory_ok=True ) if isdir(input_filename): filenames |= {get_name(f) for f in listdir(input_filename)} else: with open(input_filename, "r") as f: filenames |= { get_name(f) for f in yaml.load(f, Loader=yaml.SafeLoader) } except FileNotFoundError: filenames |= {get_name(input_filename)} filenames.discard(".DS_Store") desc += f"\n ├─ {fork_name} ({len(filenames)} filenames)" # Parse downstream stages for this class downstream_stages = fork_conf.get("downstream_stages") if isinstance(downstream_stages, str): downstream_stages = [downstream_stages] for filename in filenames: forks_by_filename[filename] = downstream_stages if downstream_stages is not None: if len(downstream_stages) >= 2: for stage in downstream_stages: desc += f"\n │ ├─ {stage}" desc += f"\n │ └─ {downstream_stages[-1]}" else: desc += f"\n │ └─" # Add description for default fork if default_fork_name is None: default_fork_name = "default" desc += f"\n └─ {default_fork_name}" if default_downstream_stages is not None: if len(default_downstream_stages) >= 2: for stage in default_downstream_stages[:-1]: desc += f"\n ├─ {stage}:" desc += f"\n └─ {default_downstream_stages[-1]}" else: desc += f"\n └─" # Store results self.desc = desc self.forks_by_filename = forks_by_filename self.default_downstream_stages = default_downstream_stages print(self.forks_by_filename)
def model_infile(self, value: str) -> None: self._model_infile = validate_input_path(value)