def _save_file(self, path): md5 = self.state.update(path) cache = self.get(md5) if self._changed(md5): move(path, cache) self.state.update(cache) self._checkout(path, md5)
def download( self, from_infos, to_infos, names=None, no_progress_bar=False, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme != "local": raise NotImplementedError logger.debug("Downloading '{}' to '{}'".format(from_info, to_info)) tmp_file = tmp_fname(to_info) if not name: name = to_info.name cb = None if no_progress_bar else Callback(name) makedirs(fspath_py35(to_info.parent), exist_ok=True) try: self.oss_service.get_object_to_file(from_info.path, tmp_file, progress_callback=cb) except Exception: logger.warning("failed to download '{}'".format(from_info)) else: move(tmp_file, fspath_py35(to_info)) finally: if not no_progress_bar: progress.finish_target(name)
def _move(self, inp, outp): # moving in two stages to make the whole operation atomic in # case inp and outp are in different filesystems and actual # physical copying of data is happening tmp = "{}.{}".format(outp, str(uuid.uuid4())) move(inp, tmp) move(tmp, outp)
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != "s3": raise NotImplementedError if to_info["scheme"] == "s3": self.copy(from_info, to_info, s3=s3) continue if to_info["scheme"] != "local": raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format( from_info["bucket"], from_info["path"], to_info["path"] ) logger.debug(msg) tmp_file = tmp_fname(to_info["path"]) if not name: name = os.path.basename(to_info["path"]) makedirs(os.path.dirname(to_info["path"]), exist_ok=True) try: if no_progress_bar: cb = None else: total = s3.head_object( Bucket=from_info["bucket"], Key=from_info["path"] )["ContentLength"] cb = Callback(name, total) s3.download_file( from_info["bucket"], from_info["path"], tmp_file, Callback=cb, ) except Exception: msg = "failed to download '{}/{}'".format( from_info["bucket"], from_info["path"] ) logger.error(msg) continue move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name)
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): if not hasattr(self, "_download"): raise RemoteActionNotImplemented("download", self.scheme) names = self._verify_path_args(from_infos, to_infos, names) with self.transfer_context() as ctx: for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme == self.scheme != "local": self.copy(from_info, to_info, ctx=ctx) continue if to_info.scheme != "local": raise NotImplementedError msg = "Downloading '{}' to '{}'".format(from_info, to_info) logger.debug(msg) tmp_file = tmp_fname(to_info) if not name: name = to_info.name if not no_progress_bar: # real progress is not always available, # lets at least show start and finish progress.update_target(name, 0, None) makedirs(fspath_py35(to_info.parent), exist_ok=True) try: self._download( from_info, tmp_file, name=name, ctx=ctx, resume=resume, no_progress_bar=no_progress_bar, ) except Exception: msg = "failed to download '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) continue move(tmp_file, fspath_py35(to_info)) if not no_progress_bar: progress.finish_target(name)
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) gs = self.gs for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != "gs": raise NotImplementedError if to_info["scheme"] == "gs": self.copy(from_info, to_info, gs=gs) continue if to_info["scheme"] != "local": raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format( from_info["bucket"], from_info["path"], to_info["path"] ) logger.debug(msg) tmp_file = tmp_fname(to_info["path"]) if not name: name = os.path.basename(to_info["path"]) if not no_progress_bar: # percent_cb is not available for download_to_filename, so # lets at least update progress at pathpoints(start, finish) progress.update_target(name, 0, None) makedirs(os.path.dirname(to_info["path"]), exist_ok=True) try: bucket = gs.bucket(from_info["bucket"]) blob = bucket.get_blob(from_info["path"]) blob.download_to_filename(tmp_file) except Exception: msg = "failed to download '{}/{}' to '{}'" logger.exception( msg.format( from_info["bucket"], from_info["path"], to_info["path"] ) ) continue move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name)
def move(self, from_info, to_info): if from_info.scheme != "local" or to_info.scheme != "local": raise NotImplementedError self.makedirs(to_info.parent) if self.isfile(from_info): mode = self._file_mode else: mode = self._dir_mode move(from_info, to_info, mode=mode)
def _download_to(self, url, target_file, callback=None, resume=False): request = self._request("GET", url, stream=True) partial_file = target_file + ".part" mode, transferred_bytes = self._determine_mode_get_transferred_bytes( partial_file, resume) self._validate_existing_file_size(transferred_bytes, partial_file) self._write_request_content(mode, partial_file, request, transferred_bytes, callback) move(partial_file, target_file)
def move(self, from_info, to_info): if from_info.scheme != "local" or to_info.scheme != "local": raise NotImplementedError inp = from_info.fspath outp = to_info.fspath # moving in two stages to make the whole operation atomic in # case inp and outp are in different filesystems and actual # physical copying of data is happening tmp = "{}.{}".format(outp, str(uuid.uuid4())) move(inp, tmp) move(tmp, outp)
def download( self, from_info, to_info, name=None, no_progress_bar=False, file_mode=None, dir_mode=None, ): if not hasattr(self, "_download"): raise RemoteActionNotImplemented("download", self.scheme) if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme == self.scheme != "local": self.copy(from_info, to_info) return 0 if to_info.scheme != "local": raise NotImplementedError logger.debug("Downloading '{}' to '{}'".format(from_info, to_info)) name = name or to_info.name if not no_progress_bar: # real progress is not always available, # lets at least show start and finish progress.update_target(name, 0, None) makedirs(to_info.parent, exist_ok=True, mode=dir_mode) tmp_file = tmp_fname(to_info) try: self._download(from_info, tmp_file, name=name, no_progress_bar=no_progress_bar) except Exception: msg = "failed to download '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) return 1 # 1 fail move(tmp_file, to_info, mode=file_mode) if not no_progress_bar: progress.finish_target(name) return 0
def _unprotect_file(self, path): import stat import uuid from dvc.utils import copyfile, move, remove self.logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def _unprotect_file(path): if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def dump_dir_cache(self, md5, dir_info): path = self.get(md5) dname = os.path.dirname(path) assert self.is_dir_cache(path) assert isinstance(dir_info, list) if not os.path.isdir(dname): os.makedirs(dname) # NOTE: Writing first and renaming after that # to make sure that the operation is atomic. tmp = '{}.{}'.format(path, str(uuid.uuid4())) with open(tmp, 'w+') as fd: json.dump(dir_info, fd, sort_keys=True) move(tmp, path)
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != self.scheme: raise NotImplementedError if to_info["scheme"] != "local": raise NotImplementedError bucket = from_info["bucket"] path = from_info["path"] logger.debug( "Downloading '{}/{}' to '{}'".format( bucket, path, to_info["path"] ) ) tmp_file = tmp_fname(to_info["path"]) if not name: name = os.path.basename(to_info["path"]) cb = None if no_progress_bar else Callback(name) makedirs(os.path.dirname(to_info["path"]), exist_ok=True) try: self.blob_service.get_blob_to_path( bucket, path, tmp_file, progress_callback=cb ) except Exception: msg = "failed to download '{}/{}'".format(bucket, path) logger.warning(msg) else: move(tmp_file, to_info["path"]) if not no_progress_bar: progress.finish_target(name)
def download( self, from_infos, to_infos, no_progress_bar=False, names=None, resume=False, ): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info["scheme"] != "local": raise NotImplementedError if to_info["scheme"] != "local": raise NotImplementedError logger.debug( "Downloading '{}' to '{}'".format( from_info["path"], to_info["path"] ) ) if not name: name = os.path.basename(to_info["path"]) makedirs(os.path.dirname(to_info["path"]), exist_ok=True) tmp_file = tmp_fname(to_info["path"]) try: copyfile( from_info["path"], tmp_file, no_progress_bar=no_progress_bar, name=name, ) move(tmp_file, to_info["path"]) except Exception: logger.exception( "failed to download '{}' to '{}'".format( from_info["path"], to_info["path"] ) ) continue
def download( self, from_info, to_info, name=None, no_progress_bar=False, file_mode=None, dir_mode=None, ): if not hasattr(self, "_download"): raise RemoteActionNotImplemented("download", self.scheme) if from_info.scheme != self.scheme: raise NotImplementedError if to_info.scheme == self.scheme != "local": self.copy(from_info, to_info) return 0 if to_info.scheme != "local": raise NotImplementedError logger.debug("Downloading '{}' to '{}'".format(from_info, to_info)) name = name or to_info.name makedirs(to_info.parent, exist_ok=True, mode=dir_mode) tmp_file = tmp_fname(to_info) try: self._download(from_info, tmp_file, name=name, no_progress_bar=no_progress_bar) except Exception: msg = "failed to download '{}' to '{}'" logger.exception(msg.format(from_info, to_info)) return 1 # 1 fail move(tmp_file, to_info, mode=file_mode) return 0
def _unprotect_file(self, path): import stat import uuid from dvc.system import System from dvc.utils import copyfile, move, remove if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def move(self, from_info, to_info): if from_info['scheme'] != 'local' or to_info['scheme'] != 'local': raise NotImplementedError move(from_info['path'], to_info['path'])
def _move(self, inp, outp): # moving in two stages to make last the move atomic in # case inp and outp are in different filesystems tmp = '{}.{}'.format(outp, str(uuid.uuid4())) move(inp, tmp) move(tmp, outp)
def move(self, from_info, to_info): if from_info["scheme"] != "local" or to_info["scheme"] != "local": raise NotImplementedError move(from_info["path"], to_info["path"])