def _push(self,
source_target,
replicas=None,
forceon=[],
exclude=[],
**kwargs):
source, target = source_target
qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
('include', ','.join(forceon)),
('replicas', replicas)) if v])
urls = self._download('{0}/ddfs/new_blob/{1}?{2}'.format(
self.master, target, qs))
try:
return [
json.loads(bytes_to_str(url)) for url in self._upload(
urls, source, to_master=False, **kwargs)
]
except CommError as e:
scheme, (host, port), path = urlsplit(e.url)
if hasattr(source, "seek"):
source.seek(
0) # source will be read again; seek to the beginning
else:
print("{0} is not seekable, retrying".format(source))
return self._push((source, target),
replicas=replicas,
forceon=forceon,
exclude=exclude + [host],
**kwargs)
def _push(self, source_target, replicas=None, exclude=[], **kwargs):
source, target = source_target
qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
('replicas', replicas)) if v])
urls = self._download('{0}/ddfs/new_blob/{1}?{2}'
.format(self.master, target, qs))
try:
return [json.loads(bytes_to_str(url))
for url in self._upload(urls, source, to_master=False, **kwargs)]
except CommError as e:
scheme, (host, port), path = urlsplit(e.url)
return self._push((source, target),
replicas=replicas,
exclude=exclude + [host],
**kwargs)
def _push(self, source_target, replicas=None, exclude=[], **kwargs):
source, target = source_target
qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
('replicas', replicas)) if v])
urls = self._download('{0}/ddfs/new_blob/{1}?{2}'.format(
self.master, target, qs))
try:
return [
json.loads(bytes_to_str(url)) for url in self._upload(
urls, source, to_master=False, **kwargs)
]
except CommError as e:
scheme, (host, port), path = urlsplit(e.url)
return self._push((source, target),
replicas=replicas,
exclude=exclude + [host],
**kwargs)
def _push(self, source_target, replicas=None, forceon=[], exclude=[], **kwargs):
source, target = source_target
qs = urlencode(
[
(k, v)
for k, v in (("exclude", ",".join(exclude)), ("include", ",".join(forceon)), ("replicas", replicas))
if v
]
)
urls = self._download("{0}/ddfs/new_blob/{1}?{2}".format(self.master, target, qs))
try:
return [json.loads(bytes_to_str(url)) for url in self._upload(urls, source, to_master=False, **kwargs)]
except CommError as e:
scheme, (host, port), path = urlsplit(e.url)
if hasattr(source, "seek"):
source.seek(0) # source will be read again; seek to the beginning
else:
print("{0} is not seekable, retrying".format(source))
return self._push((source, target), replicas=replicas, forceon=forceon, exclude=exclude + [host], **kwargs)
