def _download(self, url, data=None, token=None, method='GET'):
return json.loads(download(self._resolve(proxy_url(url,
proxy=self.proxy,
meth=method)),
data=data,
method=method,
token=self._token(url, token, method)))
def curl(replicas):
for replica in replicas:
try:
return download(proxy_url(urlresolve(replica, master=program.ddfs.master),
to_master=False))
except Exception, e:
sys.stderr.write("%s\n" % e)
def read_index(dir_url):
from disco.comm import download
scheme, netloc, path = urlsplit(dir_url)
url = proxy_url(path, netloc)
for line in download(url).splitlines():
id, url = line.split()
yield int(id), url
def curl(replicas):
for replica in replicas:
try:
return download(proxy_url(urlresolve(replica, master=program.ddfs.master),
to_master=False))
except Exception, e:
sys.stderr.write("%s\n" % e)
def request(self, url, data = None, redir = False, offset = 0):
try:
return comm.download(self.host + url, data = data, redir = redir, offset = offset)
except KeyboardInterrupt:
raise
except Exception, e:
raise DiscoError('Got %s, make sure disco master is running at %s' % (e, self.host))
def _download(self, url, data=None, token=None, method="GET", to_master=True):
byts = download(
self._resolve(proxy_url(url, proxy=self.proxy, meth=method, to_master=to_master)),
data=data,
method=method,
token=self._token(url, token, method),
)
return json.loads(bytes_to_str(byts))
def _download(self, url, data=None, token=None, method='GET', to_master=True):
return json.loads(download(self._resolve(proxy_url(url,
proxy=self.proxy,
meth=method,
to_master=to_master)),
data=data,
method=method,
token=self._token(url, token, method)))
def request(self, url, data=None, offset=0):
try:
return download('%s%s' % (self.master, url),
data=data,
offset=offset)
except CommError, e:
if e.code == None:
e.msg += " (is disco master running at %s?)" % self.master
raise
def request(self, url, data=None, offset=0):
try:
return download(proxy_url('%s%s' % (self.master, url), proxy=self.proxy),
data=data,
offset=offset)
except CommError, e:
if e.code == None:
e.msg += " (is disco master running at %s?)" % self.master
raise
def request(self, url, data=None, offset=0):
try:
byts = download(proxy_url('{0}{1}'.format(self.master, url), proxy=self.proxy),
data=data,
offset=offset)
return byts.decode('utf-8')
except CommError as e:
if e.code == None:
e.msg += " (is disco master running at {0}?)".format(self.master)
raise
def read_index(dir_url):
from disco.comm import download
scheme, netloc, path = urlsplit(dir_url)
url = proxy_url(path, netloc)
body = StringIO(download(url))
if url.endswith(".gz"):
body = gzip.GzipFile(fileobj = body)
for line in body:
id, url = line.split()
yield int(id), url
def curl(replicas):
for replica in replicas:
try:
return download(proxy_url(urlresolve(replica, master=program.ddfs.master),
to_master=False))
except Exception as e:
sys.stderr.write("{0}\n".format(e))
if not ignore_missing:
raise Exception("Failed downloading all replicas: {0}".format(replicas))
return ''
def curl(replicas):
for replica in replicas:
try:
return download(proxy_url(urlresolve(replica, master=program.ddfs.master),
to_master=False))
except Exception as e:
sys.stderr.write("{0}\n".format(e))
if not ignore_missing:
raise Exception("Failed downloading all replicas: {0}".format(replicas))
return ''
def parse_dir(dir_url, partid = None):
def parse_index(index):
return [url for id, url in (line.split() for line in index)
if partid is None or partid == int(id)]
settings = DiscoSettings()
scheme, netloc, path = urlsplit(dir_url)
if 'resultfs' in settings['DISCO_FLAGS']:
path = '%s/data/%s' % (settings['DISCO_ROOT'], path)
return parse_index(file(path))
url = proxy_url(path, netloc)
return parse_index(download(url).splitlines())
def load_oob(host, name, key):
settings = DiscoSettings()
params = {'name': name,
'key': key,
'proxy': '1' if settings['DISCO_PROXY'] else '0'}
url = '%s/disco/ctrl/oob_get?%s' % (host, urlencode(params))
if 'resultfs' in settings['DISCO_FLAGS']:
size, fd = open_remote(url, expect=302)
location = fd.getheader('location').split('/', 3)[-1]
path = '%s/data/%s' % (settings['DISCO_ROOT'], location)
return file(path).read()
return download(url, redir=True)
def put(self, tag, urls):
"""Put the list of ``urls`` to the tag ``tag``.
.. warning::
Generally speaking, concurrent applications should use
:meth:`DDFS.tag` instead.
"""
from comm_httplib import download
status, body = download('%s/ddfs/tag/%s' % (self.master, tagname(tag)),
data=json.dumps(urls),
method='PUT')
return json.loads(body)
def request(self, url, data=None, offset=0, as_bytes=False):
try:
byts = download(proxy_url('{0}{1}'.format(self.master, url),
proxy=self.proxy),
data=data,
offset=offset)
if as_bytes:
return byts
return byts.decode('utf-8')
except CommError as e:
if e.code == None:
e.msg += " (is disco master running at {0}?)".format(
self.master)
raise
def request(self, url, data=None, offset=0):
"""
Requests *url* at the master.
If a string *data* is specified, a POST request
is made with *data* as the request payload.
A string is returned that contains the reply for the request.
This method is mostly used by other methods in this class internally.
"""
try:
return download('%s%s' % (self.master, url),
data=data,
offset=offset)
except CommError, e:
e.msg += " (is disco master running at %s?)" % self.master
raise
def load_oob(host, name, key):
use_proxy = "DISCO_PROXY" in os.environ
url = "%s/disco/ctrl/oob_get?name=%s&key=%s&proxy=%d" %\
(host, name, key, use_proxy)
if resultfs_enabled:
sze, fd = open_remote(url, expect = 302)
loc = fd.getheader("location")
fname = "%s/data/%s" % (ROOT, "/".join(loc.split("/")[3:]))
try:
return file(fname).read()
except KeyboardInterrupt:
raise
except Exception:
raise DiscoError("OOB key (%s) not found at %s" %\
(key, fname))
else:
return download(url, redir = True)
def request(self, url, data=None, offset=0):
"""
Requests *url* at the master.
If a string *data* is specified, a POST request
is made with *data* as the request payload.
A string is returned that contains the reply for the request.
This method is mostly used by other methods in this class internally.
"""
try:
return download('%s%s' % (self.master, url),
data=data,
offset=offset)
except CommError, e:
if e.code == None:
e.msg += " (is disco master running at %s?)" % self.master
raise
def input_stream(fd, size, url, params):
import os
from disco import util
from disco.comm import download
from discodb import DiscoDB, Q
scheme, netloc, rest = util.urlsplit(url)
path, rest = rest.split('!', 1) if '!' in rest else (rest, '')
if netloc[0] == Task.netloc[0]:
discodb = DiscoDB.load(open(os.path.join(Task.root, path)))
else:
discodb = DiscoDB.loads(download('disco://%s/%s' % (netloc, path)))
if rest:
method_name, arg = rest.split('/', 1) if '/' in rest else (rest, None)
method = getattr(discodb, method_name)
if method_name in ('metaquery', 'query'):
return method(Q.urlscan(arg)), size, url
return method(*filter(None, arg)), size, url
return discodb, size, url
def parse_dir(dir_url, proxy = None, part_id = None):
x, x, host, name = dir_url.split("/", 3)
url = proxy_url(proxy, name, host)
if name.endswith(".txt"):
if resultfs_enabled:
r = file("%s/data/%s" % (ROOT, name)).readlines()
else:
r = download(url).splitlines()
else:
b, mmax = name.split("/")[-1].split(":")
fl = len(mmax)
base = b[:len(b) - fl]
t = "%s%%.%dd" % (base, fl)
if part_id != None:
r = [t % part_id]
else:
r = [t % i for i in range(int(mmax) + 1)]
p = "/".join(name.split("/")[:-1])
return ["disco://%s/%s/%s" % (host, p, x.strip()) for x in r]
def input_stream(fd, size, url, params):
import os
from disco import util
from disco.comm import download
from discodb import DiscoDB, Q
scheme, netloc, rest = util.urlsplit(url)
path, rest = rest.split('!', 1) if '!' in rest else (rest, '')
if netloc[0] == Task.netloc[0]:
discodb = DiscoDB.load(open(os.path.join(Task.root, path)))
else:
discodb = DiscoDB.loads(download('disco://%s/%s' % (netloc, path)))
if rest:
method_name, arg = rest.split('/', 1) if '/' in rest else (rest, None)
method = getattr(discodb, method_name)
if method_name in ('metaquery', 'query'):
return method(Q.urlscan(arg)), size, url
return method(*filter(None, arg)), size, url
return discodb, size, url
def parse_dir(dir_url, partid=None, numpartitions=None):
"""
Translates a directory URL to a list of normal URLs.
This function might be useful for other programs that need to parse
results returned by :meth:`disco.core.Disco.wait`, for instance.
:param dir_url: a directory url, such as ``dir://nx02/test_simple@12243344``
"""
from disco.comm import download
def parse_index(index):
# XXX: This should be fixed with dir://
# we shouldn't need to know the number of partitions here
lines = [line.split() for line in index]
if partid is not None and numpartitions != len(lines):
raise ValueError("Invalid number of partitions!")
return [url for id, url in lines
if partid is None or partid == int(id)]
settings = DiscoSettings()
scheme, netloc, path = urlsplit(dir_url)
url = proxy_url(path, netloc)
return parse_index(download(url).splitlines())
def _download(self, url, data=None, method='GET'):
response = download(self.master + url, data=data, method=method)
return json.loads(response)
def _download(self, url, data=None, token=None, method='GET'):
return json.loads(
download(self._resolve(url),
data=data,
method=method,
token=self._token(token, method)))
def curl(replicas):
for replica in replicas:
try:
return download(replica)
except Exception, e:
sys.stderr.write("%s\n" % e)
def _download(self, url, data=None, token=None, method='GET'):
return json.loads(download(self._resolve(url),
data=data,
method=method,
token=self._token(token, method)))
def curl(replicas):
for replica in replicas:
try:
return download(replica)
except Exception, e:
sys.stderr.write("%s\n" % e)
def request(self, url, data=None, redir=False, offset=0):
try:
return download(self.host + url, data=data, redir=redir, offset=offset)
except CommError, e:
raise DiscoError("Got %s, make sure disco master is running at %s" % (e, self.host))
def _request(self, url, data=None, method=None):
response = download(self.master + url, data=data, method=method)
return json.loads(response)