def __init__(self): self.fh = None self.d = Directories() self.console_log_level = logging.ERROR self.file_log_level = logging.INFO self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') self.logger = logging.getLogger(self.d.program_name) self.logger.setLevel(logging.DEBUG) self.init_console()
def main(): if COMPILE: dirs = Directories( base_dir=os.path.dirname(os.path.realpath(sys.argv[0]))) else: dirs = Directories( base_dir=os.path.dirname(os.path.realpath(__file__))) make_logger(dirs) mod.QuestionEngine(dirs=dirs)
class ArchiveLogger(object): """docstring for ArchiveLogger""" def __init__(self): self.fh = None self.d = Directories() self.console_log_level = logging.ERROR self.file_log_level = logging.INFO self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') self.logger = logging.getLogger(self.d.program_name) self.logger.setLevel(logging.DEBUG) self.init_console() def init_file_logging(self): if not self.d.check_myarchive(): raise Exception, "My Archive Does Not Exist" self.fh = logging.FileHandler(self.d.logger) self.fh.setLevel(self.file_log_level) self.fh.setFormatter(self.formatter) self.logger.addHandler(self.fh) self.logger.info('Log started') def init_console(self): self.ch = logging.StreamHandler() self.ch.setLevel(self.console_log_level) self.ch.setFormatter(logging.Formatter('%(message)s')) self.logger.addHandler(self.ch) def print_logger(self): self.check_fh() if self.d.check_log_exists(): f = open(self.d.logger) s = f.read() print s def check_fh(self): if self.fh is None: self.init_file_logging() def log_debug(self,msg): self.check_fh() self.logger.debug(msg) def log_info(self,msg): self.check_fh() self.logger.info(msg) def log_error(self,msg): self.check_fh() self.logger.error(msg)
def restore(destPath=None): d = Directories() paths = d.paths_in_index() for path in paths.keys(): sourcePath = os.path.join(d.objects, paths[path]) # print sourcePath if destPath is None: dest = path elif (os.path.exists(destPath)): dest = os.path.join(destPath, path) else: print "Invalid Path" break directory = os.path.join(*os.path.split(dest)[0:-1]) restoreFiles(dest, sourcePath, directory)
class Task(): def __init__(self): self._dirs = Directories() self._init_data_loader() self._init_network() def _update_dirs(self, log_folder): current_dir = os.path.dirname(os.path.realpath(__file__)) network_dir = Network.directory() + log_folder self._dirs.update_for_task(current_dir, network_dir) def _init_network(self): self.__network = Network(data_loader=self.__data_loader, dirs=self._dirs) def _init_data_loader(self): self.__data_loader = DataLoader(dirs=self._dirs, total_num_examples=22872) def _init_trainer(self): self.__trainer = Trainer(data_loader=self.__data_loader, network=self.__network, dirs=self._dirs, ld_chkpt=True, save_freq=10000, log_freq=200, vis_freq=5000) return self.__trainer def run(self, train): self._init_trainer() log_folder = '/logged_data' self._update_dirs(log_folder) self.__network.chkpt_dir = self._dirs.chkpt_dir if train: self.__trainer.run_trainer() else: self.__trainer.run_visualiser() self.__trainer.sess.close()
def __init__(self, **kwargs): debug_prefix = "[Main.__init__]" # Argv from shell self.argv = kwargs["argv"] # Empty list of flags and kflags self.flags = [] self.kflags = {} if self.argv is not None: # Iterate in all args for arg in self.argv[1:]: # Is a kwarg if "=" in arg: arg = arg.split("=") self.kflags[arg[0]] = arg[1] # Is a flag else: self.flags.append(arg) # Print user flags from the terminal print(debug_prefix, "Flags are:", self.flags) print(debug_prefix, "Keyword flags are:", self.kflags) # Create classes self.utils = Utils(self) self.subprocess_utils = SubprocessUtils(self) self.pacman = Pacman(self) self.directories = Directories(self) self.download = Download(self) self.actions = Actions(self) # Runs every action self.actions.run()
def init(): #Define Archive Location d = Directories() # check if myArchive directory exists, create it if it doesn't if os.path.exists(d.my_archive) and os.path.isdir(d.my_archive): print "Archive Directory EXISTS\t %s" % d.my_archive else: print "Archive Directory CREATED\t %s" % d.my_archive os.makedirs(d.my_archive) # check if objects directory exists, create it if it doesn't if os.path.exists(d.objects) and os.path.isdir(d.objects): print "Objects Directory EXISTS\t %s" % d.objects else: print "Objects Directory CREATED\t %s" % d.objects os.makedirs(d.objects) # check if INDEX file exists, create it if it doesn't if os.path.exists(d.index) and os.path.isfile(d.index): print "INDEX file EXISTS\t\t %s" % d.index else: print "INDEX file CREATED\t\t %s" % d.index open(d.index,'w').close()
import h5py from directories import Directories import os def add_axes_refs(obj_with_ref, obj_x_axis, obj_y_axis): obj_with_ref.attrs['x_axis'] = obj_x_axis.ref obj_with_ref.attrs['y_axis'] = obj_y_axis.ref def add_refs_to_all_images(filename): with h5py.File(filename, 'a') as f: group = f['q_interpolated/PEN04'] for key in list(group.keys()): current_group = group[key] x_axis = current_group['q_xy'] y_axis = current_group['q_z'] for k in list(current_group.keys()): if k.startswith('zaptime'): current_group[k].attrs['x_axis'] = x_axis.ref current_group[k].attrs['y_axis'] = y_axis.ref if __name__ == '__main__': filename = os.path.join(Directories.get_dir_to_save_images(), 'whole_data.h5') add_refs_to_all_images(filename)
from directories import Directories d = Directories() # show what files are in the archive def listfiles(searchArg=None): if searchArg == None: print "All Files" else: print "Files Matching %s:" % searchArg paths = d.paths_in_index() for x in paths: if searchArg == None or searchArg in x: print x
parser.add_argument("--model", help="Set model to VAE/CVAE/TDCVAE/TDCVAE2 (required)", required=True) parser.add_argument("--dataset", help="Set dataset to MNIST/LFW/FF/LungScans accordingly (required, not case sensitive)", required=True) parser.add_argument("--save_files", help="Determine if files (samples etc.) should be saved (optional, default: False)", required=False, action='store_true') parser.add_argument("--save_model_state", help="Determine if state of model should be saved after each epoch during training (optional, default: False)",\ required=False, action='store_true') parser.add_argument('--scales', help="Enables scaling of the model as specified in model_params", default=None, action='store_true') parser.add_argument('--thetas', help="Enables rotations of the model as specified in model_params", default=None, action='store_true') args = vars(parser.parse_args()) model_arg = args["model"] dataset_arg = args["dataset"] save_files = args["save_files"] save_model_state = args["save_model_state"] if model_arg.lower() == "vae": data = get_model_data_vae(dataset_arg) directories = Directories(model_arg.lower(), dataset_arg, data["z_dim"],\ make_dirs=save_files) data_loader = DataLoader(directories, data["batch_size"], dataset_arg) model = Vae(data_loader.input_dim, data["hidden_dim"],\ data["z_dim"], data["beta"], data["batch_norm"]) solver = Solver(model, data_loader, data["optimizer"],\ data["epochs"], data["optim_config"],\ step_config=data["step_config"], lr_scheduler=data["lr_scheduler"],\ save_model_state=save_model_state) elif model_arg.lower() == "cvae": data = get_model_data_cvae(dataset_arg) directories = Directories(model_arg.lower(), dataset_arg, data["z_dim"],\ make_dirs=save_files) data_loader = DataLoader(directories, data["batch_size"], dataset_arg) model = Cvae(data_loader.input_dim, data["hidden_dim"], data["z_dim"],\ data["beta"], data_loader.n_classes, data["batch_norm"]) solver = Solver(model, data_loader, data["optimizer"], data["epochs"],\
def __init__(self): self._dirs = Directories() self._init_data_loader() self._init_network()
def __init__(self, utils, config, failsafe): debug_prefix = "[Context.__init__]" # When logging isn't a simple line but something like a list or dictionary self.indentation = "ยทยทยท | " # Set the (static) rootfolder substitution for changing paths session folders, so we can move the Dandere2x folder and be able to resume (?) self.rootfolder_substitution = "//ROOTFOLDER//" # Context needs Utils for logging and works based on a config self.utils = utils self.config = config # For absolute-reffering self.ROOT = self.utils.ROOT # Failsafe module self.failsafe = failsafe self.utils.log( color, 3, debug_prefix, "Rootfolder substitution is [%s] on Context and context_vars file" % self.rootfolder_substitution) if "resume_session_context_vars_file" in self.config: self.load_vars_from_file( self.config["resume_session_context_vars_file"]) return # Session self.force = self.config["danger_zone"]["force"] # Get the operating system we're working with self.os = self.utils.get_os() self.utils.log(colors["warning"], 1, debug_prefix, "Got operating system: " + self.os) # Loglevel self.loglevel = self.config["developer"]["loglevel"] self.utils.log(colors["warning"], 1, debug_prefix, "LOGLEVEL: [%s]" % self.loglevel) # # Load variables # # # # See settings.yaml for a guide on the settings and what they are # # # # # Video I/O self.input_file = self.config["basic"]["input_file"] self.output_file = self.config["basic"]["output_file"] # Session, "auto" is the input filename self.session_name = self.config["basic"]["session_name"] self.input_filename = self.utils.get_basename(self.input_file) # Windows uses ugly back slashes, failsafe if self.os == "windows": self.input_file = self.input_file.replace("/", "\\") # Block matching related self.block_size = self.config["block_matching"]["block_size"] self.bleed = self.config["block_matching"]["bleed"] self.dark_threshold = self.config["block_matching"]["dark_threshold"] self.bright_threshold = self.config["block_matching"][ "bright_threshold"] self.upscale_full_frame_threshold = self.config["block_matching"][ "upscale_full_frame_threshold"] # Upscaler settings self.upscaler_type = self.config["upscaler"]["type"] self.upscale_ratio = self.config["upscaler"]["upscale_ratio"] self.denoise_level = self.config["upscaler"]["denoise_level"] self.tile_size = self.config["upscaler"]["tile_size"] self.upscaler_model = self.config["upscaler"]["model"] self.load_proc_save = self.config["upscaler"]["load:proc:save"] self.w2x_converter_cpp_jobs = self.config["upscaler"][ "w2x_converter_cpp_jobs"] # realsr-ncnn-vulkan must have upscale_ratio=4 as it's the only option if self.upscaler_type == "realsr-ncnn-vulkan": self.utils.log( color, 0, debug_prefix, "[WARNING] USING REALSR UPSCALER, FORCING UPSCALE_RATIO=4 FOR CONVENIENCE" ) self.upscale_ratio = 4 # See if selected upscaler type and ratio / denoise are compatible self.failsafe.compatible_utype_uratio(self.upscaler_type, self.upscale_ratio) self.failsafe.compatible_upscaler_denoise(self.upscaler_type, self.denoise_level) # If the user did not sent us a absolute path self.input_file = os.path.abspath(self.input_file) # Output file can be auto, that is, append $UPSCALE_RATIO$x_$UPSCALER_TYPE$ at the start of the filename if self.output_file == "auto": # This is already absolute path as we just set input_file to one self.output_file = self.utils.auto_output_file( self.input_file, self.upscale_ratio, self.config["upscaler"]["type"]) self.utils.log( color, 1, debug_prefix, "Output file set to \"auto\", assigning: [%s]" % self.output_file) else: # Else if it was manually set, get the absolute path for it self.output_file = os.path.abspath(self.output_file) # Get the new output filename if it was set to auto self.output_filename = self.utils.get_basename(self.output_file) # Check if input is file and output file directory exist, if not create it self.failsafe.check_input_output(self.input_file, self.output_file) # # The special case where the session name is "auto", # so we set it according to the input file "a.mkv" -> "a" if self.session_name == "auto": self.session_name = self.utils.get_auto_session_name( self.input_file) # Stats settings self.average_last_N_frames = self.config["stats"][ "average_last_N_frames"] self.show_stats = self.config["stats"]["show_stats"] # Create default variables self.resolution = [] self.valid_resolution = [] self.frame_rate = None self.frame_count = None self.frame_rate = None self.resume = False self.last_processing_frame = 0 self.zero_padding = 8 # We change this later on based on the frame_count self.total_upscale_time = 0 # Video related variables self.get_video_info_method = self.config["video"][ "get_video_info_method"] self.get_frame_count_method = self.config["video"][ "get_frame_count_method"] self.get_frame_rate_method = self.config["video"][ "get_frame_rate_method"] self.get_resolution_method = self.config["video"][ "get_resolution_method"] # # FFmpeg / FFprobe related self.deblock_filter = self.config["ffmpeg"]["deblock_filter"] self.encode_codec = self.config["ffmpeg"]["encode_codec"] # x264 encoding self.x264_preset = self.config["ffmpeg"]["x264"]["preset"] self.x264_tune = self.config["ffmpeg"]["x264"]["tune"] self.x264_crf = self.config["ffmpeg"]["x264"]["crf"] # Dandere2x C++ specific self.mindisk = self.config["dandere2x_cpp"]["mindisk"] self.write_debug_video = int( self.config["dandere2x_cpp"]["write_debug_video"]) self.show_debug_video_realtime = int( self.config["dandere2x_cpp"]["show_debug_video_realtime"]) self.show_block_matching_stats = int( self.config["dandere2x_cpp"]["show_block_matching_stats"]) self.only_run_dandere2x_cpp = int( self.config["dandere2x_cpp"]["only_run_dandere2x_cpp"]) # # Developer variables # How much time in seconds to wait for waiting operations like until_exist() self.wait_time = self.config["developer"]["wait_time_exists"] self.upscaler_wait_for_residuals = self.config["developer"][ "upscaler_wait_for_residuals"] # The range we'll delete the residuals self.safety_ruthless_residual_eliminator_range = self.config[ "developer"]["safety_ruthless_residual_eliminator_range"] # Will we be writing logs? self.write_log = self.config["developer"]["write_log"] # Vapoursynth settings self.use_vapoursynth = self.config["vapoursynth"]["enabled"] self.vapoursynth_pre = self.config["vapoursynth"]["pre"] self.vapoursynth_pos = self.config["vapoursynth"]["pos"] # This might sound dumb but it's good to debug as upscaler doesn't upscale and mindisk remove stuff self.enable_upscaler = self.config["debug"]["enable_upscaler"] # # # # Session directories / files # # # # self.utils.log(color, 4, debug_prefix, "Configuring context.* directories and static files") # Directories will handle for us a few tweaks we can do to the Dandere2x runtime self.directories = Directories() self.directories.input_filename = self.input_filename self.directories.session_name = self.session_name self.directories.generate_dirs() self.plain_dirs = self.directories.plain_dirs self.plain_files = self.directories.plain_files # We declare these as none just for annoying errors on this # dynamic variable setting workaround and for autocompleting self.residual = self.directories.get("residual") self.externals = self.directories.get("externals") self.upscaled = self.directories.get("upscaled") self.partial = self.directories.get("partial") self.processing = self.directories.get("processing") self.merged = self.directories.get("merged") self.session = self.directories.get("session") self.sessions_folder = self.directories.get("sessions_folder") self.upscaled_video = self.directories.get("upscaled_video") self.resume_video_frame = self.directories.get("resume_video_frame") self.debug_video = self.directories.get("debug_video") self.context_vars = self.directories.get("context_vars") self.temp_vpy_script = self.directories.get("temp_vpy_script") self.vapoursynth_processing = self.directories.get( "vapoursynth_processing") self.joined_audio = self.directories.get("joined_audio") self.partial_video = self.directories.get("partial_video") self.logfile = self.directories.get("logfile") self.logfile_last_session = self.directories.get( "logfile_last_session")