def load_S3_SL_1_RBT_zip(s3_file_name: str) -> Dataset: from tempfile import mkdtemp ds_list = [] with fs.open_fs(s3_file_name) as zip_fs: metadata = read_metadata(zip_fs) tmp_dir = mkdtemp() copy.copy_dir(zip_fs, '/', OSFS('/'), tmp_dir) matches = glob(f'{tmp_dir}/**/*.nc') for match in matches: with xr.load_dataset(match, decode_times=False, decode_cf=False, decode_coords=False, mask_and_scale=False, decode_timedelta=False, chunks={}) as array: source = s3_band_name(path.basename(match)) ds = rename_dimensions_and_variables(array, source) if source == "viscal": ds['viscal.calibration_time'] = xr.apply_ufunc( convert_date, ds['viscal.calibration_time']) ds['viscal.ANX_time'] = xr.apply_ufunc( convert_date, ds['viscal.ANX_time']) ds_list.append(ds) ds = xr.merge(ds_list) ds = ds.chunk(chunks='auto') shutil.rmtree(tmp_dir) return Dataset(data=ds, metadata=metadata)
def build_static_directory(self, obj): """ Builds an object's static subdirectory. """ # The location of static files in the dynamic page directory source_dir = os.path.join(obj.page_directory_path, 'static') # The location in the build directory where we want to copy them target_dir = path.join( bigbuild.get_build_directory(), obj.get_static_url().lstrip("/") ) # An internal django-bakery trick to gzip them if we need to if settings.BAKERY_GZIP: cmd = Build() cmd.set_options() cmd.copytree_and_gzip( source_dir, target_dir ) else: # Or a more vanilla way of copying the files with Python logger.debug("Copying {}{} to {}{}".format("osfs://", source_dir, self.fs_name, target_dir)) copy.copy_dir("osfs:///", smart_text(source_dir), self.fs, smart_text(target_dir))
def load_model(filesystem: S3FS, config: PredictionConfig) -> ResUnetA: """ Copy the model locally if not existing and load it """ if not os.path.exists(f'{config.temp_model_path}/{config.model_name}'): if not filesystem.exists( f'{config.model_path}/{config.model_name}/checkpoints/'): filesystem.makedirs( f'{config.model_path}/{config.model_name}/checkpoints/') copy_dir(filesystem, f'{config.model_path}/{config.model_name}/checkpoints/', f'{config.temp_model_path}/{config.model_name}', 'checkpoints') copy_file(filesystem, f'{config.model_path}/{config.model_name}/model_cfg.json', f'{config.temp_model_path}/{config.model_name}', 'model_cfg.json') input_shape = dict( features=[None, config.height, config.width, config.n_channels]) with open(f'{config.temp_model_path}/{config.model_name}/model_cfg.json', 'r') as jfile: model_cfg = json.load(jfile) # initialise model from config, build, compile and load trained weights model = ResUnetA(model_cfg) model.build(input_shape) model.net.compile() model.net.load_weights( f'{config.temp_model_path}/{config.model_name}/checkpoints/model.ckpt') return model
def build_object(self, obj): """ Build the provided object """ # If it is a Page object ... if isinstance(obj, Page): # ... use the default bakery build super(PageDetailView, self).build_object(obj) # ... and build the static directory as well. self.build_static_directory(obj) # If it is an ArchivedPage object ... elif isinstance(obj, ArchivedPage): # ... do a copy and paste from the archive to the build directory target = obj.build_directory_path if settings.BAKERY_GZIP: cmd = Build() cmd.set_options() cmd.copytree_and_gzip( obj.archive_static_directory_path, target ) else: logger.debug("Copying {}{} to {}{}".format( "osfs://", obj.archive_static_directory_path, self.fs_name, target )) copy.copy_dir( "osfs:///", smart_text(obj.archive_static_directory_path), self.fs, smart_text(target) )
def export_jsonlite(self, url: str): self.connection.commit() self.connection.close() remote_fs = open_fs(url) copy.copy_dir(self.local_fs, ".", remote_fs, ".") self.connection = sqlite3.connect(self.db_file) self.connection.row_factory = sqlite3.Row
def _try_copy_to_filesystem(self, source: str, target: str, filesystem: Optional[Filesystem]) -> None: other_filesystem = cast(PyFilesystemBased, filesystem).internal_fs if self.internal_fs.isdir(source): fscp.copy_dir(self.internal_fs, source, other_filesystem, target) return fscp.copy_file(self.internal_fs, source, other_filesystem, target)
def build_static(self, *args, **options): """ Builds the static files directory as well as robots.txt and favicon.ico """ logger.debug("Building static directory") if self.verbosity > 1: self.stdout.write("Building static directory") management.call_command("collectstatic", interactive=False, verbosity=0) # Set the target directory inside the filesystem. target_dir = path.join(self.build_dir, settings.STATIC_URL.lstrip('/')) target_dir = smart_text(target_dir) exclude_dirs = getattr(settings, 'BAKERY_STATIC_EXCLUDE_DIRS', None) if not exclude_dirs: # explicitly set to None to make sure we don't get an empty list/tuple exclude_dirs = None if os.path.exists(self.static_root) and settings.STATIC_URL: if getattr(settings, 'BAKERY_GZIP', False): self.copytree_and_gzip(self.static_root, target_dir, exclude_dirs) # if gzip isn't enabled, just copy the tree straight over else: logger.debug("Copying {}{} to {}{}".format( "osfs://", self.static_root, self.fs_name, target_dir)) walker = Walker(exclude_dirs=exclude_dirs) copy.copy_dir("osfs:///", self.static_root, self.fs, target_dir, walker=walker) # If they exist in the static directory, copy the robots.txt # and favicon.ico files down to the root so they will work # on the live website. robots_src = path.join(target_dir, 'robots.txt') if self.fs.exists(robots_src): robots_target = path.join(self.build_dir, 'robots.txt') logger.debug("Copying {}{} to {}{}".format(self.fs_name, robots_src, self.fs_name, robots_target)) self.fs.copy(robots_src, robots_target) favicon_src = path.join(target_dir, 'favicon.ico') if self.fs.exists(favicon_src): favicon_target = path.join(self.build_dir, 'favicon.ico') logger.debug("Copying {}{} to {}{}".format(self.fs_name, favicon_src, self.fs_name, favicon_target)) self.fs.copy(favicon_src, favicon_target)
def build_media(self): """ Build the media files. """ logger.debug("Building media directory") if self.verbosity > 1: self.stdout.write("Building media directory") if os.path.exists(self.media_root) and settings.MEDIA_URL: target_dir = path.join(self.fs_name, self.build_dir, settings.MEDIA_URL.lstrip('/')) logger.debug("Copying {}{} to {}{}".format("osfs://", self.media_root, self.fs_name, target_dir)) copy.copy_dir("osfs:///", smart_text(self.media_root), self.fs, smart_text(target_dir))
def add(self, object_path): """Add pre-constructed object from object_path.""" self.open_root_fs() self.check_root_structure() # Sanity check o = Object() o.open_fs(object_path) inventory = o.parse_inventory() identifier = inventory['id'] # Now copy path = self.object_path(identifier) logging.info("Copying from %s to %s", object_path, fs.path.join(self.root, path)) try: copy_dir(o.obj_fs, '/', self.root_fs, path) logging.info("Copied") except Exception as e: logging.error("Copy failed: %s", str(e)) raise StoreException("Add object failed!")
def handle(self, *args, **options): """ Make it happen. """ # Loop through the slugs for slug in options['slug']: # Pull the object try: p = PageList()[slug] except Exception: raise CommandError("Slug provided (%s) does not exist" % slug) if not isinstance(p, Page): raise CommandError("Slug (%s) is not a Page object" % slug) # Build it PageArchiveView().build_object(p) # If the retired directory exists, kill it if os.path.exists(p.archive_static_directory_path): shutil.rmtree(p.archive_static_directory_path) # pragma: no cover # Save that directory to the retired folder copy.copy_dir( self.fs, smart_text(p.build_directory_path), "osfs:///", smart_text(p.archive_static_directory_path) ) # Save the metadata to the retired folder frontmatter_path = os.path.join(p.archive_static_directory_path, 'metadata.md') p.write_frontmatter(frontmatter_path) # Copy the dynamic page folder to its archival location shutil.copytree(p.page_directory_path, p.archive_dynamic_directory_path) # Delete the page folder if not options['keep_page']: p.delete() # Update the cache call_command("cachepages")
def build_static(self, *args, **options): """ Builds the static files directory as well as robots.txt and favicon.ico """ logger.debug("Building static directory") if self.verbosity > 1: self.stdout.write("Building static directory") management.call_command( "collectstatic", interactive=False, verbosity=0 ) # Set the target directory inside the filesystem. target_dir = path.join( self.build_dir, settings.STATIC_URL.lstrip('/') ) target_dir = smart_text(target_dir) if os.path.exists(self.static_root) and settings.STATIC_URL: if getattr(settings, 'BAKERY_GZIP', False): self.copytree_and_gzip(self.static_root, target_dir) # if gzip isn't enabled, just copy the tree straight over else: logger.debug("Copying {}{} to {}{}".format("osfs://", self.static_root, self.fs_name, target_dir)) copy.copy_dir("osfs:///", self.static_root, self.fs, target_dir) # If they exist in the static directory, copy the robots.txt # and favicon.ico files down to the root so they will work # on the live website. robots_src = path.join(target_dir, 'robots.txt') if self.fs.exists(robots_src): robots_target = path.join(self.build_dir, 'robots.txt') logger.debug("Copying {}{} to {}{}".format(self.fs_name, robots_src, self.fs_name, robots_target)) self.fs.copy(robots_src, robots_target) favicon_src = path.join(target_dir, 'favicon.ico') if self.fs.exists(favicon_src): favicon_target = path.join(self.build_dir, 'favicon.ico') logger.debug("Copying {}{} to {}{}".format(self.fs_name, favicon_src, self.fs_name, favicon_target)) self.fs.copy(favicon_src, favicon_target)
def backup_scenario(self, confirm=True): """Copy scenario data to backup disk and remove original :param bool confirm: prompt before deleting each batch of files """ src_fs = dst_fs = get_ssh_fs() items = [ (self._join(*server_setup.INPUT_DIR), f"{self.scenario_id}_*"), (self._join(*server_setup.OUTPUT_DIR), f"{self.scenario_id}_*"), (self._data_access.tmp_folder(self.scenario_id), "**"), ] for folder, pattern in items: print(f"--> Moving files matching {pattern} from {folder}") src_path = self._join(server_setup.DATA_ROOT_DIR, folder) dst_path = self._join(server_setup.BACKUP_DATA_ROOT_DIR, folder) walker = Walker(filter=[pattern]) try: copy_dir(src_fs, src_path, dst_fs, dst_path, walker=walker) except FSError as e: print(f"Operation failed: {e}") self._data_access.remove(self._join(folder, pattern), confirm=confirm)
def install_from_fs(self, name: str, source_fs: FS, path: str = ".", output_name: str = None): """ Copy a module directory from where it is located to the installation directory. :param name: Name of the module :param source_fs: FS object pointing to the source location :param path: Path to the module directory from the source location root :param output_name: Name to give to the module's directory at installation """ path_to_module = join(path, name) if name not in source_fs.listdir(path): raise InvalidModuleError( name, "Module directory not found - Given path should be the parent directory" ) if "__manifest__.py" not in source_fs.listdir(path_to_module): raise InvalidModuleError( name, "Manifest not found - Given path should be the parent directory" ) self.pretty_print(output_name, "Copying from {}".format( source_fs.desc(path_to_module)), level=VERBOSE_FULL) copy_dir(source_fs, path_to_module, self.install_dir, output_name or name) self.pretty_print(output_name, "Installed and up to date.", status=LOG_STATUS_OK, level=VERBOSE_NORMAL)
def run_vectorisation(config: VectorisationConfig) -> List[str]: """ Run vectorisation process on entire AOI for the given time intervals """ filesystem = prepare_filesystem(config) LOGGER.info(f'Copy tiff files locally to {config.predictions_dir}') for time_interval in config.time_intervals: if not os.path.exists(f'{config.predictions_dir}/{time_interval}'): if not filesystem.exists( f'{config.tiffs_folder}/{time_interval}/'): filesystem.makedirs(f'{config.tiffs_folder}/{time_interval}/') copy_dir(filesystem, f'{config.tiffs_folder}/{time_interval}/', f'{config.predictions_dir}/', f'{time_interval}') LOGGER.info(f'Move files to utm folders') for time_interval in config.time_intervals: for utm in config.utms: utm_dir = f'{config.predictions_dir}/{time_interval}/utm{utm}' os.makedirs(utm_dir, exist_ok=True) tiffs_to_move = glob( f'{config.predictions_dir}/{time_interval}/*-{utm}.tiff') for tiff in tiffs_to_move: tiff_name = os.path.basename(tiff) os.rename(tiff, f'{utm_dir}/{tiff_name}') LOGGER.info(f'Create weights file {config.weights_file}') with rasterio.open(config.weights_file, 'w', driver='gTIFF', width=config.shape[0], height=config.shape[1], count=1, dtype=np.float32) as dst: dst.write_band(1, get_weights(config.shape, config.buffer)) rows = [] for time_interval in config.time_intervals: for utm in config.utms: start = time.time() LOGGER.info(f'Running contours for {time_interval}/{utm}!') contours_dir = f'{config.contours_dir}/{time_interval}/utm{utm}/' LOGGER.info(f'Create contour folder {contours_dir}') os.makedirs(contours_dir, exist_ok=True) predictions_dir = f'{config.predictions_dir}/{time_interval}/utm{utm}/' tifs = glob(f'{predictions_dir}*.tiff') output_vrt = f'{config.vrt_dir}/vrt_{time_interval}_{utm}.vrt' write_vrt(tifs, config.weights_file, output_vrt) results = process_rows(output_vrt, contours_dir, max_workers=config.max_workers, size=config.chunk_size, buffer=config.chunk_overlap, threshold=config.threshold, cleanup=config.cleanup, skip_existing=config.skip_existing) failed = [(file, excp) for file, finished, excp in results if not finished] if len(failed): LOGGER.warning('Some rows failed:') LOGGER.warning('\n'.join( [f'{file}: {excp}' for file, excp in failed])) # raise Exception(f'{len(failed)} rows failed! ') LOGGER.warning(f'{len(failed)} rows failed! ') rows.append({ 'time_interval': time_interval, 'utm': utm, 'vrt_file': output_vrt, 'rows': [file for file, finished, _ in results if finished], 'chunk_size': config.chunk_size, 'chunk_overlap': config.chunk_overlap, 'contours_dir': config.contours_dir }) LOGGER.info( f'Row contours processing for {time_interval}/{utm} done in {(time.time() - start) / 60} min!\n\n' ) list_of_merged_files = multiprocess(merging_rows, rows, max_workers=config.max_workers) return list_of_merged_files
if "renpy.zip" not in cwdfs.listdir("/"): puts("Downloading Ren'Py") r = requests.get( "https://www.renpy.org/dl/6.99.12.4/renpy-6.99.12.4-sdk.zip", stream=True) r.raise_for_status() with cwdfs.open("renpy.zip", 'wb') as fd: total_length = int(r.headers.get('content-length')) for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1): fd.write(chunk) puts("Extracting Ren'Py") with ZipFS("./renpy.zip") as zipfs: fscopy.copy_dir(zipfs, "renpy-6.99.12.4-sdk", tempfs, "renpy") cwdfs.remove("renpy.zip") puts("ModTemplate setup") with indent(2): if "modtemplate.zip" not in cwdfs.listdir("/"): puts("Downloading ModTemplate") r = requests.get( "https://github.com/Monika-After-Story/DDLCModTemplate/releases/download/v1.1.0/DDLCModTemplate_1.1.0.zip", stream=True) r.raise_for_status() with cwdfs.open("modtemplate.zip", 'wb') as fd: total_length = int(r.headers.get('content-length')) for chunk in progress.bar(r.iter_content(chunk_size=1024), expected_size=(total_length / 1024) + 1):
def run(self): args = self.args application = WSGIApplication( self.location, self.get_settings(), args.server, disable_autoreload=True, master_settings=self.master_settings, ) archive = application.archive filesystems = archive.filesystems fs = None if args.fs: try: fs = filesystems[args.fs] except KeyError: self.console.error("No filesystem called '%s'" % args.fs) return -1 if args.tree is not None: if fs is None: self.console.error("Filesystem required") return -1 with fs.opendir(args.tree or "/") as tree_fs: tree.render(tree_fs, max_levels=None) return if args.listdir: if fs is None: self.console.error("Filesystem required") return -1 dir_fs = fs.opendir(args.listdir) file_paths = [] dir_paths = [] for info in dir_fs.scandir("/"): if info.is_dir: dir_paths.append(info.name) else: file_paths.append(info.name) _ls(self.console, file_paths, dir_paths) elif args.cat: if fs is None: self.console.error("Filesystem required") return -1 contents = fs.gettext(args.cat) self.console.cat(contents, args.cat) elif args.open: if fs is None: self.console.error("Filesystem required") return -1 try: filepath = fs.getsyspath(args.open) except NoSysPath: self.console.error( "No system path for '%s' in filesystem '%s'" % (args.open, args.fs) ) return -1 import subprocess system = sys.platform if system == "darwin": subprocess.call(("open", filepath)) elif system == "win32": subprocess.call(("start", filepath), shell=True) elif system == "linux2": subprocess.call(("xdg-open", filepath)) else: self.console.error( "Moya doesn't know how to open files on this platform (%s)" % os.name ) elif args.syspath: if fs is None: self.console.error("Filesystem required") return -1 if not fs.exists(args.syspath): self.console.error( "No file called '%s' found in filesystem '%s'" % (args.syspath, args.fs) ) return -1 try: syspath = fs.getsyspath(args.syspath) except NoSysPath: self.console.error( "No system path for '%s' in filesystem '%s'" % (args.syspath, args.fs) ) else: self.console(syspath).nl() elif args.copy: if fs is None: self.console.error("Filesystem required") return -1 if len(args.copy) == 1: src = "/" dst = args.copy[0] elif len(args.copy) == 2: src, dst = args.copy else: self.console.error("--copy requires 1 or 2 arguments") return -1 if fs.isdir(src): src_fs = fs.opendir(src) from fs.copy import copy_dir with open_fs(dst, create=True) as dst_fs: if not args.force and not dst_fs.isempty("/"): response = raw_input( "'%s' is not empty. Copying may overwrite directory contents. Continue? " % dst ) if response.lower() not in ("y", "yes"): return 0 copy_dir(src_fs, "/", dst_fs, "/") else: with fs.open(src, "rb") as read_f: if os.path.isdir(dst): dst = os.path.join(dst, os.path.basename(src)) try: os.makedirs(dst) with open(dst, "wb") as write_f: while 1: chunk = read_f.read(16384) if not chunk: break write_f.write(chunk) except IOError as e: self.error("unable to write to {}".format(dst)) elif args.extract: if fs is None: self.console.error("Filesystem required") return -1 src_path, dst_dir_path = args.extract src_fs = fs dst_fs = open_fs(dst_dir_path, create=True) if not args.force and dst_fs.exists(src_path): response = raw_input( "'%s' exists. Do you want to overwrite? " % src_path ) if response.lower() not in ("y", "yes"): return 0 dst_fs.makedirs(dirname(src_path), recreate=True) with src_fs.open(src_path, "rb") as read_file: dst_fs.setfile(src_path, read_file) elif args.serve: from .serve import Serve Serve.run_server( args.host, args.port, fs, show_access=True, develop=False, debug=True ) else: table = [ [ Cell("Name", bold=True), Cell("Type", bold=True), Cell("Location", bold=True), ] ] if fs is None: list_filesystems = filesystems.items() else: list_filesystems = [(args.fs, fs)] def get_type_name(name): name = type(fs).__name__ return name[:-2].lower() if name.endswith("FS") else name.lower() for name, fs in sorted(list_filesystems): if isinstance(fs, MultiFS): location = "\n".join( mount_fs.desc("/") for name, mount_fs in fs.iterate_fs() ) fg = "yellow" elif isinstance(fs, MountFS): mount_desc = [] for path, dirmount in fs.mount_tree.items(): mount_desc.append("%s->%s" % (path, dirmount.fs.desc("/"))) location = "\n".join(mount_desc) fg = "magenta" else: try: syspath = fs.getsyspath("/") except NoSysPath: location = syspath fg = "green" else: try: location = fs.desc("/") except FSError as e: location = text_type(e) fg = "red" else: fg = "blue" table.append( [ Cell(name), Cell(get_type_name(fs)), Cell(location, bold=True, fg=fg), ] ) self.console.table(table, header=True)