def download_if_needed(uri, download_dir, fs=None): """Download a file into a directory if it's remote. If uri is local, there is no need to download the file. Args: uri: (string) URI of file download_dir: (string) local directory to download file into fs: Optional FileSystem to use. Returns: (string) path to local file Raises: NotReadableError if URI cannot be read from """ if uri is None: return None if not fs: fs = FileSystem.get_file_system(uri, 'r') path = get_local_path(uri, download_dir, fs=fs) make_dir(path, use_dirname=True) if path != uri: log.debug('Downloading {} to {}'.format(uri, path)) fs.copy_from(uri, path) return path
def set_tmp_dir(tmp_dir=None): """Set RVConfig.tmp_dir to well-known value. This static method sets the value of RVConfig.tmp_dir to some well-known value. The value is chosen from one of the following (in order of preference): an explicit value (presumably from the command line) is considered first, then values from the environment are considered, then the current value of RVConfig.tmp_dir is considered, then a directory from tempfile.TemporaryDirectory() is considered. Args: tmp_dir: Either a string or None. """ DEFAULT_DIR = '/opt/data/tmp/' # Check the various possibilities in order of priority. tmp_dir_array = [tmp_dir] env_array = [ os.environ.get(k) for k in ['TMPDIR', 'TEMP', 'TMP'] if k in os.environ ] current_array = [RVConfig.tmp_dir] it = tmp_dir_array + env_array + current_array it = list(filter(lambda p: p is not None, it)) if it: explicit_tmp_dir = it[0] else: explicit_tmp_dir = tempfile.TemporaryDirectory().name try: # Try to create directory if not os.path.exists(explicit_tmp_dir): os.makedirs(explicit_tmp_dir, exist_ok=True) # Check that it is actually a directory if not os.path.isdir(explicit_tmp_dir): raise Exception( '{} is not a directory.'.format(explicit_tmp_dir)) # Can we interact with directory? Path.touch(Path(os.path.join(explicit_tmp_dir, '.can_touch'))) # All checks have passed by this point RVConfig.tmp_dir = explicit_tmp_dir # If directory cannot be made and/or cannot be interacted # with, fall back to default. except Exception as e: log.warning( 'Root temporary directory cannot be used: {}. Using root: {}'. format(explicit_tmp_dir, DEFAULT_DIR)) RVConfig.tmp_dir = DEFAULT_DIR finally: make_dir(RVConfig.tmp_dir) log.debug('Temporary directory is: {}'.format(RVConfig.tmp_dir))
def zipdir(dir, zip_path): """Save a zip file with contents of directory. Contents of directory will be at root of zip file. Args: dir: (str) directory to zip zip_path: (str) path to zip file to create """ make_dir(zip_path, use_dirname=True) with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as ziph: for dirpath, dirnames, filenames in os.walk(dir): for fn in filenames: ziph.write(join(dirpath, fn), join(dirpath[len(dir):], fn))
def unzip(zip_path, target_dir): """Unzip contents of zip file at zip_path into target_dir""" make_dir(target_dir) zip_ref = zipfile.ZipFile(zip_path, 'r') zip_ref.extractall(target_dir) zip_ref.close()