Esempio n. 1
0
 def _push(self,
           source_target,
           replicas=None,
           forceon=[],
           exclude=[],
           **kwargs):
     source, target = source_target
     qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
                                         ('include', ','.join(forceon)),
                                         ('replicas', replicas)) if v])
     urls = self._download('{0}/ddfs/new_blob/{1}?{2}'.format(
         self.master, target, qs))
     try:
         return [
             json.loads(bytes_to_str(url)) for url in self._upload(
                 urls, source, to_master=False, **kwargs)
         ]
     except CommError as e:
         scheme, (host, port), path = urlsplit(e.url)
         if hasattr(source, "seek"):
             source.seek(
                 0)  # source will be read again; seek to the beginning
         else:
             print("{0} is not seekable, retrying".format(source))
         return self._push((source, target),
                           replicas=replicas,
                           forceon=forceon,
                           exclude=exclude + [host],
                           **kwargs)
Esempio n. 2
0
File: ddfs.py Progetto: wquan/disco
 def _push(self, source_target, replicas=None, exclude=[], **kwargs):
     source, target = source_target
     qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
                                         ('replicas', replicas)) if v])
     urls = self._download('{0}/ddfs/new_blob/{1}?{2}'
                           .format(self.master, target, qs))
     try:
         return [json.loads(bytes_to_str(url))
                 for url in self._upload(urls, source, to_master=False, **kwargs)]
     except CommError as e:
         scheme, (host, port), path = urlsplit(e.url)
         return self._push((source, target),
                           replicas=replicas,
                           exclude=exclude + [host],
                           **kwargs)
Esempio n. 3
0
File: ddfs.py Progetto: yuj/disco
 def _push(self, source_target, replicas=None, exclude=[], **kwargs):
     source, target = source_target
     qs = urlencode([(k, v) for k, v in (('exclude', ','.join(exclude)),
                                         ('replicas', replicas)) if v])
     urls = self._download('{0}/ddfs/new_blob/{1}?{2}'.format(
         self.master, target, qs))
     try:
         return [
             json.loads(bytes_to_str(url)) for url in self._upload(
                 urls, source, to_master=False, **kwargs)
         ]
     except CommError as e:
         scheme, (host, port), path = urlsplit(e.url)
         return self._push((source, target),
                           replicas=replicas,
                           exclude=exclude + [host],
                           **kwargs)
Esempio n. 4
0
 def _push(self, source_target, replicas=None, forceon=[], exclude=[], **kwargs):
     source, target = source_target
     qs = urlencode(
         [
             (k, v)
             for k, v in (("exclude", ",".join(exclude)), ("include", ",".join(forceon)), ("replicas", replicas))
             if v
         ]
     )
     urls = self._download("{0}/ddfs/new_blob/{1}?{2}".format(self.master, target, qs))
     try:
         return [json.loads(bytes_to_str(url)) for url in self._upload(urls, source, to_master=False, **kwargs)]
     except CommError as e:
         scheme, (host, port), path = urlsplit(e.url)
         if hasattr(source, "seek"):
             source.seek(0)  # source will be read again; seek to the beginning
         else:
             print("{0} is not seekable, retrying".format(source))
         return self._push((source, target), replicas=replicas, forceon=forceon, exclude=exclude + [host], **kwargs)