def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != 'local': raise NotImplementedError if to_info['scheme'] != 'local': raise NotImplementedError logger.debug("Downloading '{}' to '{}'".format( from_info['path'], to_info['path'])) if not name: name = os.path.basename(to_info['path']) self._makedirs(to_info['path']) tmp_file = self.tmp_file(to_info['path']) try: copyfile(from_info['path'], tmp_file, no_progress_bar=no_progress_bar, name=name) except Exception as exc: msg = "Failed to download '{}' to '{}'" logger.warn(msg.format(from_info['path'], to_info['path']), exc) continue os.rename(tmp_file, to_info['path'])
def _is_enabled(cmd=None): from dvc.config import Config from dvc.project import Project from dvc.exceptions import NotDvcProjectError from dvc.command.daemon import CmdDaemonBase if os.getenv('DVC_TEST'): return False if isinstance(cmd, CmdDaemonBase): return False if cmd is None or not hasattr(cmd, 'config'): try: dvc_dir = Project._find_dvc_dir() config = Config(dvc_dir) assert config is not None except NotDvcProjectError: config = Config(validate=False) assert config is not None else: config = cmd.config assert config is not None core = config._config.get(Config.SECTION_CORE, {}) enabled = core.get(Config.SECTION_CORE_ANALYTICS, True) logger.debug( "Analytics is {}abled.".format('en' if enabled else 'dis')) return enabled
def _with_lock(self, func, action): try: with self.lock: func() except LockError: msg = "Failed to acquire '{}' before {} updates" logger.debug(msg.format(self.lock.lock_file, action))
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info['scheme'] != self.scheme: raise NotImplementedError if from_info['scheme'] != 'local': raise NotImplementedError bucket = to_info['bucket'] key = to_info['key'] logger.debug("Uploading '{}' to '{}/{}'".format( from_info['path'], bucket, key)) if not name: name = os.path.basename(from_info['path']) cb = Callback(name) try: self.blob_service.create_blob_from_path(bucket, key, from_info['path'], progress_callback=cb) except Exception as ex: msg = "Failed to upload '{}'".format(from_info['path']) logger.warn(msg, ex) else: progress.finish_target(name)
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) s3 = self.s3 for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info['scheme'] != 's3': raise NotImplementedError if from_info['scheme'] != 'local': raise NotImplementedError logger.debug("Uploading '{}' to '{}/{}'".format( from_info['path'], to_info['bucket'], to_info['key'])) if not name: name = os.path.basename(from_info['path']) total = os.path.getsize(from_info['path']) cb = Callback(name, total) try: s3.upload_file(from_info['path'], to_info['bucket'], to_info['key'], Callback=cb) except Exception as exc: msg = "Failed to upload '{}'".format(from_info['path']) logger.warn(msg, exc) continue progress.finish_target(name)
def remove_unused_links(self, used): unused = [] self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE)) for row in self.c: p, i, m = row i = self._from_sqlite(i) path = os.path.join(self.root_dir, p) if path in used: continue if not os.path.exists(path): continue inode = self.inode(path) mtime = self.mtime(path) if i == inode and m == mtime: logger.debug('Removing \'{}\' as unused link.'.format(path)) remove(path) unused.append(p) for p in unused: cmd = 'DELETE FROM {} WHERE path = "{}"' self._execute(cmd.format(self.LINK_STATE_TABLE, p))
def save(self, config=None): if config is not None: clist = [config] else: clist = [ self._system_config, self._global_config, self._project_config, self._local_config ] for conf in clist: if conf.filename is None: continue try: logger.debug("Writing '{}'.".format(conf.filename)) dname = os.path.dirname(os.path.abspath(conf.filename)) try: os.makedirs(dname) except OSError as exc: if exc.errno != errno.EEXIST: raise conf.write() except Exception as exc: msg = "Failed to write config '{}'".format(conf.filename) raise ConfigError(msg, exc)
def remove(self, path_info): if path_info['scheme'] != self.scheme: raise NotImplementedError logger.debug('Removing azure://{}/{}'.format(path_info['bucket'], path_info['key'])) self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
def remove(self, path_info): if path_info['scheme'] != 's3': raise NotImplementedError logger.debug('Removing s3://{}/{}'.format(path_info['bucket'], path_info['key'])) self.s3.delete_object(Bucket=path_info['bucket'], Key=path_info['key'])
def remove(self, path_info): if path_info['scheme'] != 'hdfs': raise NotImplementedError assert path_info.get('url') logger.debug('Removing {}'.format(path_info['url'])) self.rm(path_info)
def changed(self, path, md5): actual = self.update(path) msg = "File '{}', md5 '{}', actual '{}'" logger.debug(msg.format(path, md5, actual)) if not md5 or not actual: return True return actual.split('.')[0] != md5.split('.')[0]
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != 's3': raise NotImplementedError if to_info['scheme'] == 's3': self._copy(from_info, to_info, s3=s3) continue if to_info['scheme'] != 'local': raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'], from_info['key'], to_info['path']) logger.debug(msg) tmp_file = self.tmp_file(to_info['path']) if not name: name = os.path.basename(to_info['path']) self._makedirs(to_info['path']) try: if no_progress_bar: cb = None else: total = s3.head_object( Bucket=from_info['bucket'], Key=from_info['key'])['ContentLength'] cb = Callback(name, total) s3.download_file(from_info['bucket'], from_info['key'], tmp_file, Callback=cb) except Exception as exc: msg = "Failed to download '{}/{}'".format( from_info['bucket'], from_info['key']) logger.warn(msg, exc) continue os.rename(tmp_file, to_info['path']) if not no_progress_bar: progress.finish_target(name)
def _read_user_id(self): if not os.path.exists(self.user_id_file): return None with open(self.user_id_file, 'r') as fobj: try: info = json.load(fobj) return info[self.PARAM_USER_ID] except json.JSONDecodeError as exc: logger.debug("Failed to load user_id: {}".format(exc)) return None
def _get_user_id(self): from dvc.lock import LockError try: with self.user_id_file_lock: user_id = self._read_user_id() if user_id is None: user_id = self._write_user_id() return user_id except LockError: msg = "Failed to acquire '{}'" logger.debug(msg.format(self.user_id_file_lock.lock_file))
def remove(self, path_info): if path_info['scheme'] != 'ssh': raise NotImplementedError logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'], path_info['host'], path_info['path'])) ssh = self.ssh(host=path_info['host'], user=path_info['user'], port=path_info['port']) ssh.open_sftp().remove(path_info['path']) ssh.close()
def _get_latest_version(self): import json try: r = requests.get(self.URL, timeout=self.TIMEOUT_GET) info = r.json() except requests.exceptions.RequestException as exc: msg = "Failed to retrieve latest version: {}" logger.debug(msg.format(exc)) return with open(self.updater_file, 'w+') as fobj: json.dump(info, fobj)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) ssh = self.ssh(host=from_infos[0]['host'], user=from_infos[0]['user'], port=from_infos[0]['port']) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != 'ssh': raise NotImplementedError if to_info['scheme'] == 'ssh': assert from_info['host'] == to_info['host'] assert from_info['port'] == to_info['port'] assert from_info['user'] == to_info['user'] self.cp(from_info, to_info, ssh=ssh) continue if to_info['scheme'] != 'local': raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format(from_info['host'], from_info['path'], to_info['path']) logger.debug(msg) if not name: name = os.path.basename(to_info['path']) self._makedirs(to_info['path']) tmp_file = self.tmp_file(to_info['path']) try: ssh.open_sftp().get(from_info['path'], tmp_file, callback=create_cb(name)) except Exception as exc: msg = "Failed to download '{}/{}' to '{}'" logger.warn( msg.format(from_info['host'], from_info['path'], to_info['path']), exc) continue os.rename(tmp_file, to_info['path']) progress.finish_target(name) ssh.close()
def _do_update(self, path): """ Search for the current inode on the table or update it with gathered information. """ if not os.path.exists(path): return (None, None) mtime = self.mtime(path) inode = self.inode(path) cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE, self._to_sqlite(inode)) self._execute(cmd) ret = self._fetchall() if len(ret) == 0: md5, info = self._collect(path) cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \ 'VALUES ({}, "{}", "{}", "{}")' self._execute( cmd.format(self.STATE_TABLE, self._to_sqlite(inode), mtime, md5, int(nanotime.timestamp(time.time())))) self.inserts += 1 else: assert len(ret) == 1 assert len(ret[0]) == 4 i, m, md5, timestamp = ret[0] i = self._from_sqlite(i) assert i == inode msg = "Inode '{}', mtime '{}', actual mtime '{}'." logger.debug(msg.format(i, m, mtime)) if mtime != m: md5, info = self._collect(path) cmd = 'UPDATE {} SET ' \ 'mtime = "{}", md5 = "{}", timestamp = "{}" ' \ 'WHERE inode = {}' self._execute( cmd.format(self.STATE_TABLE, mtime, md5, int(nanotime.timestamp(time.time())), self._to_sqlite(inode))) else: info = None cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}' self._execute( cmd.format(self.STATE_TABLE, int(nanotime.timestamp(time.time())), self._to_sqlite(inode))) return (md5, info)
def send(self): import requests if not self._is_enabled(): return self.collect() logger.debug("Sending analytics: {}".format(self.info)) try: requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST) except requests.exceptions.RequestException as exc: logger.debug("Failed to send analytics: {}".format(str(exc)))
def remove(path): if not os.path.exists(path): return logger.debug(u'Removing \'{}\''.format(os.path.relpath(path))) def _chmod(func, p, excinfo): perm = os.stat(p).st_mode perm |= stat.S_IWRITE os.chmod(p, perm) func(p) if os.path.isfile(path): _chmod(os.unlink, path, None) else: shutil.rmtree(path, onerror=_chmod)
def __call__(self, args): from dvc.utils import is_binary cmd = [sys.executable] if not is_binary(): cmd += ['-m', 'dvc'] cmd += ['daemon', '-q'] + args logger.debug("Trying to spawn '{}'".format(cmd)) if os.name == 'nt': self._spawn_windows(cmd) elif os.name == 'posix': self._spawn_posix(cmd) else: raise NotImplementedError logger.debug("Spawned '{}'".format(cmd))
def _check(self): if not os.path.exists(self.updater_file) or self._is_outdated_file(): self.fetch() return with open(self.updater_file, 'r') as fobj: import json try: info = json.load(fobj) self.latest = info['version'] except Exception as exc: msg = "'{}' is not a valid json: {}" logger.debug(msg.format(self.updater_file, exc)) self.fetch() return if self._is_outdated(): self._notify()
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(to_infos, from_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] not in ['http', 'https']: raise NotImplementedError if to_info['scheme'] != 'local': raise NotImplementedError msg = "Downloading '{}' to '{}'".format(from_info['url'], to_info['path']) logger.debug(msg) tmp_file = self.tmp_file(to_info['path']) if not name: name = os.path.basename(to_info['path']) self._makedirs(to_info['path']) total = self._content_length(from_info['url']) if no_progress_bar or not total: cb = None else: cb = ProgressBarCallback(name, total) try: self._download_to(from_info['url'], tmp_file, callback=cb) except Exception as exc: msg = "Failed to download '{}'".format(from_info['url']) logger.warn(msg, exc) continue os.rename(tmp_file, to_info['path']) if not no_progress_bar: progress.finish_target(name)
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != self.scheme: raise NotImplementedError if to_info['scheme'] != 'local': raise NotImplementedError bucket = from_info['bucket'] key = from_info['key'] logger.debug("Downloading '{}/{}' to '{}'".format( bucket, key, to_info['path'])) tmp_file = self.tmp_file(to_info['path']) if not name: name = os.path.basename(to_info['path']) cb = None if no_progress_bar else Callback(name) self._makedirs(to_info['path']) try: self.blob_service.get_blob_to_path(bucket, key, tmp_file, progress_callback=cb) except Exception as exc: msg = "Failed to download '{}/{}'".format(bucket, key) logger.warn(msg, exc) else: os.rename(tmp_file, to_info['path']) if not no_progress_bar: progress.finish_target(name)
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) ssh = self.ssh(host=to_infos[0]['host'], user=to_infos[0]['user'], port=to_infos[0]['port']) sftp = ssh.open_sftp() for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info['scheme'] != 'ssh': raise NotImplementedError if from_info['scheme'] != 'local': raise NotImplementedError logger.debug("Uploading '{}' to '{}/{}'".format( from_info['path'], to_info['host'], to_info['path'])) if not name: name = os.path.basename(from_info['path']) dname = posixpath.dirname(to_info['path']) self._exec(ssh, 'mkdir -p {}'.format(dname)) try: sftp.put(from_info['path'], to_info['path'], callback=create_cb(name)) except Exception as exc: msg = "Failed to upload '{}' to '{}/{}'" logger.warn( msg.format(from_info['path'], to_info['host'], to_info['path'], exc)) continue progress.finish_target(name) sftp.close() ssh.close()
def ssh(self, host=None, user=None, port=None): msg = "Establishing ssh connection with '{}' " \ "through port '{}' as user '{}'" logger.debug(msg.format(host, port, user)) ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) if self.ask_password and self.password is None: msg = ("Enter a private key passphrase or a password for " "host '{}' port '{}' user '{}'").format(host, port, user) self.password = self.project.prompt.prompt_password(msg) ssh.connect(host, username=user, port=port, key_filename=self.keyfile, timeout=self.timeout, password=self.password) return ssh
def upload(self, from_infos, to_infos, names=None): names = self._verify_path_args(to_infos, from_infos, names) for from_info, to_info, name in zip(from_infos, to_infos, names): if to_info['scheme'] != 'local': raise NotImplementedError if from_info['scheme'] != 'local': raise NotImplementedError logger.debug("Uploading '{}' to '{}'".format( from_info['path'], to_info['path'])) if not name: name = os.path.basename(from_info['path']) self._makedirs(to_info['path']) try: copyfile(from_info['path'], to_info['path'], name=name) except Exception as exc: msg = "Failed to upload '{}' tp '{}'" logger.warn(msg.format(from_info['path'], to_info['path']), exc)
def link(self, cache, path): assert os.path.isfile(cache) dname = os.path.dirname(path) if not os.path.exists(dname): os.makedirs(dname) # NOTE: just create an empty file for an empty cache if os.path.getsize(cache) == 0: open(path, 'w+').close() msg = "Created empty file: {} -> {}".format(cache, path) logger.debug(msg) return i = len(self.cache_types) while i > 0: try: self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path) if self.protected: os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH) msg = "Created {}'{}': {} -> {}".format( 'protected ' if self.protected else '', self.cache_types[0], cache, path) logger.debug(msg) return except DvcException as exc: msg = 'Cache type \'{}\' is not supported: {}' logger.debug(msg.format(self.cache_types[0], str(exc))) del self.cache_types[0] i -= 1 raise DvcException('No possible cache types left to try out.')
def percent_cb(name, complete, total): """ Callback for updating target progress """ logger.debug('{}: {} transferred out of {}'.format(name, sizeof_fmt(complete), sizeof_fmt(total))) progress.update_target(name, complete, total)
def _execute(self, cmd): logger.debug(cmd) return self.c.execute(cmd)