def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != 'local':
raise NotImplementedError
if to_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(
from_info['path'], to_info['path']))
if not name:
name = os.path.basename(to_info['path'])
self._makedirs(to_info['path'])
tmp_file = self.tmp_file(to_info['path'])
try:
copyfile(from_info['path'],
tmp_file,
no_progress_bar=no_progress_bar,
name=name)
except Exception as exc:
msg = "Failed to download '{}' to '{}'"
logger.warn(msg.format(from_info['path'], to_info['path']),
exc)
continue
os.rename(tmp_file, to_info['path'])
def _is_enabled(cmd=None):
from dvc.config import Config
from dvc.project import Project
from dvc.exceptions import NotDvcProjectError
from dvc.command.daemon import CmdDaemonBase
if os.getenv('DVC_TEST'):
return False
if isinstance(cmd, CmdDaemonBase):
return False
if cmd is None or not hasattr(cmd, 'config'):
try:
dvc_dir = Project._find_dvc_dir()
config = Config(dvc_dir)
assert config is not None
except NotDvcProjectError:
config = Config(validate=False)
assert config is not None
else:
config = cmd.config
assert config is not None
core = config._config.get(Config.SECTION_CORE, {})
enabled = core.get(Config.SECTION_CORE_ANALYTICS, True)
logger.debug(
"Analytics is {}abled.".format('en' if enabled else 'dis'))
return enabled
def _with_lock(self, func, action):
try:
with self.lock:
func()
except LockError:
msg = "Failed to acquire '{}' before {} updates"
logger.debug(msg.format(self.lock.lock_file, action))
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info['scheme'] != self.scheme:
raise NotImplementedError
if from_info['scheme'] != 'local':
raise NotImplementedError
bucket = to_info['bucket']
key = to_info['key']
logger.debug("Uploading '{}' to '{}/{}'".format(
from_info['path'], bucket, key))
if not name:
name = os.path.basename(from_info['path'])
cb = Callback(name)
try:
self.blob_service.create_blob_from_path(bucket,
key,
from_info['path'],
progress_callback=cb)
except Exception as ex:
msg = "Failed to upload '{}'".format(from_info['path'])
logger.warn(msg, ex)
else:
progress.finish_target(name)
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
s3 = self.s3
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info['scheme'] != 's3':
raise NotImplementedError
if from_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Uploading '{}' to '{}/{}'".format(
from_info['path'], to_info['bucket'], to_info['key']))
if not name:
name = os.path.basename(from_info['path'])
total = os.path.getsize(from_info['path'])
cb = Callback(name, total)
try:
s3.upload_file(from_info['path'],
to_info['bucket'],
to_info['key'],
Callback=cb)
except Exception as exc:
msg = "Failed to upload '{}'".format(from_info['path'])
logger.warn(msg, exc)
continue
progress.finish_target(name)
def remove_unused_links(self, used):
unused = []
self._execute('SELECT * FROM {}'.format(self.LINK_STATE_TABLE))
for row in self.c:
p, i, m = row
i = self._from_sqlite(i)
path = os.path.join(self.root_dir, p)
if path in used:
continue
if not os.path.exists(path):
continue
inode = self.inode(path)
mtime = self.mtime(path)
if i == inode and m == mtime:
logger.debug('Removing \'{}\' as unused link.'.format(path))
remove(path)
unused.append(p)
for p in unused:
cmd = 'DELETE FROM {} WHERE path = "{}"'
self._execute(cmd.format(self.LINK_STATE_TABLE, p))
def save(self, config=None):
if config is not None:
clist = [config]
else:
clist = [
self._system_config, self._global_config, self._project_config,
self._local_config
]
for conf in clist:
if conf.filename is None:
continue
try:
logger.debug("Writing '{}'.".format(conf.filename))
dname = os.path.dirname(os.path.abspath(conf.filename))
try:
os.makedirs(dname)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
conf.write()
except Exception as exc:
msg = "Failed to write config '{}'".format(conf.filename)
raise ConfigError(msg, exc)
def remove(self, path_info):
if path_info['scheme'] != self.scheme:
raise NotImplementedError
logger.debug('Removing azure://{}/{}'.format(path_info['bucket'],
path_info['key']))
self.blob_service.delete_blob(path_info['bucket'], path_info['key'])
def remove(self, path_info):
if path_info['scheme'] != 's3':
raise NotImplementedError
logger.debug('Removing s3://{}/{}'.format(path_info['bucket'],
path_info['key']))
self.s3.delete_object(Bucket=path_info['bucket'], Key=path_info['key'])
def remove(self, path_info):
if path_info['scheme'] != 'hdfs':
raise NotImplementedError
assert path_info.get('url')
logger.debug('Removing {}'.format(path_info['url']))
self.rm(path_info)
def changed(self, path, md5):
actual = self.update(path)
msg = "File '{}', md5 '{}', actual '{}'"
logger.debug(msg.format(path, md5, actual))
if not md5 or not actual:
return True
return actual.split('.')[0] != md5.split('.')[0]
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != 's3':
raise NotImplementedError
if to_info['scheme'] == 's3':
self._copy(from_info, to_info, s3=s3)
continue
if to_info['scheme'] != 'local':
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
from_info['key'],
to_info['path'])
logger.debug(msg)
tmp_file = self.tmp_file(to_info['path'])
if not name:
name = os.path.basename(to_info['path'])
self._makedirs(to_info['path'])
try:
if no_progress_bar:
cb = None
else:
total = s3.head_object(
Bucket=from_info['bucket'],
Key=from_info['key'])['ContentLength']
cb = Callback(name, total)
s3.download_file(from_info['bucket'],
from_info['key'],
tmp_file,
Callback=cb)
except Exception as exc:
msg = "Failed to download '{}/{}'".format(
from_info['bucket'], from_info['key'])
logger.warn(msg, exc)
continue
os.rename(tmp_file, to_info['path'])
if not no_progress_bar:
progress.finish_target(name)
def _read_user_id(self):
if not os.path.exists(self.user_id_file):
return None
with open(self.user_id_file, 'r') as fobj:
try:
info = json.load(fobj)
return info[self.PARAM_USER_ID]
except json.JSONDecodeError as exc:
logger.debug("Failed to load user_id: {}".format(exc))
return None
def _get_user_id(self):
from dvc.lock import LockError
try:
with self.user_id_file_lock:
user_id = self._read_user_id()
if user_id is None:
user_id = self._write_user_id()
return user_id
except LockError:
msg = "Failed to acquire '{}'"
logger.debug(msg.format(self.user_id_file_lock.lock_file))
def remove(self, path_info):
if path_info['scheme'] != 'ssh':
raise NotImplementedError
logger.debug('Removing ssh://{}@{}/{}'.format(path_info['user'],
path_info['host'],
path_info['path']))
ssh = self.ssh(host=path_info['host'],
user=path_info['user'],
port=path_info['port'])
ssh.open_sftp().remove(path_info['path'])
ssh.close()
def _get_latest_version(self):
import json
try:
r = requests.get(self.URL, timeout=self.TIMEOUT_GET)
info = r.json()
except requests.exceptions.RequestException as exc:
msg = "Failed to retrieve latest version: {}"
logger.debug(msg.format(exc))
return
with open(self.updater_file, 'w+') as fobj:
json.dump(info, fobj)
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
ssh = self.ssh(host=from_infos[0]['host'],
user=from_infos[0]['user'],
port=from_infos[0]['port'])
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != 'ssh':
raise NotImplementedError
if to_info['scheme'] == 'ssh':
assert from_info['host'] == to_info['host']
assert from_info['port'] == to_info['port']
assert from_info['user'] == to_info['user']
self.cp(from_info, to_info, ssh=ssh)
continue
if to_info['scheme'] != 'local':
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(from_info['host'],
from_info['path'],
to_info['path'])
logger.debug(msg)
if not name:
name = os.path.basename(to_info['path'])
self._makedirs(to_info['path'])
tmp_file = self.tmp_file(to_info['path'])
try:
ssh.open_sftp().get(from_info['path'],
tmp_file,
callback=create_cb(name))
except Exception as exc:
msg = "Failed to download '{}/{}' to '{}'"
logger.warn(
msg.format(from_info['host'], from_info['path'],
to_info['path']), exc)
continue
os.rename(tmp_file, to_info['path'])
progress.finish_target(name)
ssh.close()
def _do_update(self, path):
"""
Search for the current inode on the table or update it with
gathered information.
"""
if not os.path.exists(path):
return (None, None)
mtime = self.mtime(path)
inode = self.inode(path)
cmd = 'SELECT * from {} WHERE inode={}'.format(self.STATE_TABLE,
self._to_sqlite(inode))
self._execute(cmd)
ret = self._fetchall()
if len(ret) == 0:
md5, info = self._collect(path)
cmd = 'INSERT INTO {}(inode, mtime, md5, timestamp) ' \
'VALUES ({}, "{}", "{}", "{}")'
self._execute(
cmd.format(self.STATE_TABLE, self._to_sqlite(inode), mtime,
md5, int(nanotime.timestamp(time.time()))))
self.inserts += 1
else:
assert len(ret) == 1
assert len(ret[0]) == 4
i, m, md5, timestamp = ret[0]
i = self._from_sqlite(i)
assert i == inode
msg = "Inode '{}', mtime '{}', actual mtime '{}'."
logger.debug(msg.format(i, m, mtime))
if mtime != m:
md5, info = self._collect(path)
cmd = 'UPDATE {} SET ' \
'mtime = "{}", md5 = "{}", timestamp = "{}" ' \
'WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE, mtime, md5,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
else:
info = None
cmd = 'UPDATE {} SET timestamp = "{}" WHERE inode = {}'
self._execute(
cmd.format(self.STATE_TABLE,
int(nanotime.timestamp(time.time())),
self._to_sqlite(inode)))
return (md5, info)
def send(self):
import requests
if not self._is_enabled():
return
self.collect()
logger.debug("Sending analytics: {}".format(self.info))
try:
requests.post(self.URL, json=self.info, timeout=self.TIMEOUT_POST)
except requests.exceptions.RequestException as exc:
logger.debug("Failed to send analytics: {}".format(str(exc)))
def remove(path):
if not os.path.exists(path):
return
logger.debug(u'Removing \'{}\''.format(os.path.relpath(path)))
def _chmod(func, p, excinfo):
perm = os.stat(p).st_mode
perm |= stat.S_IWRITE
os.chmod(p, perm)
func(p)
if os.path.isfile(path):
_chmod(os.unlink, path, None)
else:
shutil.rmtree(path, onerror=_chmod)
def __call__(self, args):
from dvc.utils import is_binary
cmd = [sys.executable]
if not is_binary():
cmd += ['-m', 'dvc']
cmd += ['daemon', '-q'] + args
logger.debug("Trying to spawn '{}'".format(cmd))
if os.name == 'nt':
self._spawn_windows(cmd)
elif os.name == 'posix':
self._spawn_posix(cmd)
else:
raise NotImplementedError
logger.debug("Spawned '{}'".format(cmd))
def _check(self):
if not os.path.exists(self.updater_file) or self._is_outdated_file():
self.fetch()
return
with open(self.updater_file, 'r') as fobj:
import json
try:
info = json.load(fobj)
self.latest = info['version']
except Exception as exc:
msg = "'{}' is not a valid json: {}"
logger.debug(msg.format(self.updater_file, exc))
self.fetch()
return
if self._is_outdated():
self._notify()
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] not in ['http', 'https']:
raise NotImplementedError
if to_info['scheme'] != 'local':
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(from_info['url'],
to_info['path'])
logger.debug(msg)
tmp_file = self.tmp_file(to_info['path'])
if not name:
name = os.path.basename(to_info['path'])
self._makedirs(to_info['path'])
total = self._content_length(from_info['url'])
if no_progress_bar or not total:
cb = None
else:
cb = ProgressBarCallback(name, total)
try:
self._download_to(from_info['url'], tmp_file, callback=cb)
except Exception as exc:
msg = "Failed to download '{}'".format(from_info['url'])
logger.warn(msg, exc)
continue
os.rename(tmp_file, to_info['path'])
if not no_progress_bar:
progress.finish_target(name)
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != self.scheme:
raise NotImplementedError
if to_info['scheme'] != 'local':
raise NotImplementedError
bucket = from_info['bucket']
key = from_info['key']
logger.debug("Downloading '{}/{}' to '{}'".format(
bucket, key, to_info['path']))
tmp_file = self.tmp_file(to_info['path'])
if not name:
name = os.path.basename(to_info['path'])
cb = None if no_progress_bar else Callback(name)
self._makedirs(to_info['path'])
try:
self.blob_service.get_blob_to_path(bucket,
key,
tmp_file,
progress_callback=cb)
except Exception as exc:
msg = "Failed to download '{}/{}'".format(bucket, key)
logger.warn(msg, exc)
else:
os.rename(tmp_file, to_info['path'])
if not no_progress_bar:
progress.finish_target(name)
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
ssh = self.ssh(host=to_infos[0]['host'],
user=to_infos[0]['user'],
port=to_infos[0]['port'])
sftp = ssh.open_sftp()
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info['scheme'] != 'ssh':
raise NotImplementedError
if from_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Uploading '{}' to '{}/{}'".format(
from_info['path'], to_info['host'], to_info['path']))
if not name:
name = os.path.basename(from_info['path'])
dname = posixpath.dirname(to_info['path'])
self._exec(ssh, 'mkdir -p {}'.format(dname))
try:
sftp.put(from_info['path'],
to_info['path'],
callback=create_cb(name))
except Exception as exc:
msg = "Failed to upload '{}' to '{}/{}'"
logger.warn(
msg.format(from_info['path'], to_info['host'],
to_info['path'], exc))
continue
progress.finish_target(name)
sftp.close()
ssh.close()
def ssh(self, host=None, user=None, port=None):
msg = "Establishing ssh connection with '{}' " \
"through port '{}' as user '{}'"
logger.debug(msg.format(host, port, user))
ssh = paramiko.SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
if self.ask_password and self.password is None:
msg = ("Enter a private key passphrase or a password for "
"host '{}' port '{}' user '{}'").format(host, port, user)
self.password = self.project.prompt.prompt_password(msg)
ssh.connect(host,
username=user,
port=port,
key_filename=self.keyfile,
timeout=self.timeout,
password=self.password)
return ssh
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info['scheme'] != 'local':
raise NotImplementedError
if from_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Uploading '{}' to '{}'".format(
from_info['path'], to_info['path']))
if not name:
name = os.path.basename(from_info['path'])
self._makedirs(to_info['path'])
try:
copyfile(from_info['path'], to_info['path'], name=name)
except Exception as exc:
msg = "Failed to upload '{}' tp '{}'"
logger.warn(msg.format(from_info['path'], to_info['path']),
exc)
def link(self, cache, path):
assert os.path.isfile(cache)
dname = os.path.dirname(path)
if not os.path.exists(dname):
os.makedirs(dname)
# NOTE: just create an empty file for an empty cache
if os.path.getsize(cache) == 0:
open(path, 'w+').close()
msg = "Created empty file: {} -> {}".format(cache, path)
logger.debug(msg)
return
i = len(self.cache_types)
while i > 0:
try:
self.CACHE_TYPE_MAP[self.cache_types[0]](cache, path)
if self.protected:
os.chmod(path, stat.S_IREAD | stat.S_IRGRP | stat.S_IROTH)
msg = "Created {}'{}': {} -> {}".format(
'protected ' if self.protected else '',
self.cache_types[0], cache, path)
logger.debug(msg)
return
except DvcException as exc:
msg = 'Cache type \'{}\' is not supported: {}'
logger.debug(msg.format(self.cache_types[0], str(exc)))
del self.cache_types[0]
i -= 1
raise DvcException('No possible cache types left to try out.')
def percent_cb(name, complete, total):
""" Callback for updating target progress """
logger.debug('{}: {} transferred out of {}'.format(name,
sizeof_fmt(complete),
sizeof_fmt(total)))
progress.update_target(name, complete, total)
def _execute(self, cmd):
logger.debug(cmd)
return self.c.execute(cmd)
