def index(self, refresh='false'):
"""Return the list of available modules urls"""
#server = urlparse.urljoin(config.get('bisque.server'), self.service_type)
server = urlparse.urljoin(tg.request.host_url, self.service_type)
if asbool(refresh):
self.refresh()
modules = [("%s/%s" % (server, k), k)
for k in sorted(self.module_by_name.keys())]
return dict(modules=modules)
def process_config(self, runner, **kw):
runner.load_section('docker', runner.bisque_cfg)
runner.load_section('docker', runner.module_cfg)
self.enabled = asbool(runner.config.get('docker.enabled', False))
#self.docker_hub = runner.config.get('docker.hub', '')
#self.docker_image = runner.config.get('docker.image', '')
#self.docker_user = runner.config.get ('docker.hub.user', '')
#self.docker_pass = runner.config.get('docker.hub.password', '')
#self.docker_email = runner.config.get('docker.hub.email', '')
#self.docker_login_tmpl = runner.config.get ('docker.login_tmpl', self.docker_login_tmpl)
self.docker_params = AttrDict()
for k in self.docker_keys:
self.docker_params[k.replace('.', '_')] = runner.config.get(k, '')
def setup(params, *args, **kw):
docker_params = read_config('runtime-bisque.cfg', "docker")
if not asbool(docker_params.get('docker.enabled', False)):
print "No Docker available... cannot set up module."
return 1
try:
# clone Dream.3D UCSB repo and build Docker image
tmp_dir = tempfile.mkdtemp()
print "Cloning code into %s..." % tmp_dir
p = Popen([
'git', 'clone', '[email protected]:wlenthe/UCSB_DREAM3D',
'%s/source' % tmp_dir
],
stdout=PIPE)
if p.wait() != 0:
print "Dream.3D repo could not be cloned... cannot set up module."
return 1
# build Docker image
print "Building Dream.3D docker image..."
p = Popen([
'timeout', '-s', 'SIGKILL', '120m', 'docker', 'build', '--tag',
'dream3d_ucsb',
'%s/source' % tmp_dir
],
stdout=PIPE)
while True:
line = p.stdout.readline()
if line == '':
break
m = re.search('\[[ ]*[0-9]+%\]', line)
if m:
print '\r\r\r\r\r\r',
print m.group(0),
sys.stdout.flush()
print
if p.wait() != 0:
print "Dream.3D image could not be built... cannot set up module."
return 1
python_setup('Dream3D.py', params=params)
docker_setup('bisque_dream3d_ucsb',
'Dream3D',
'dream3d_ucsb',
params=params)
finally:
# delete tmp_dir
shutil.rmtree(tmp_dir)
from bq.util.locks import Locks, FileLocked
from bq.util.timer import Timer
from bq.util.sizeoffmt import sizeof_fmt
from bq.util.converters import asbool
class S3Error(Exception):
pass
log = logging.getLogger('bq.blobs.storage.s3')
#def s3_parse_url(url):
# "Read an s3 url, return a bucket and key"
# pass
s3q=None
if asbool (config.get('bisque.s3queue')):
from rq import Queue #pylint: disable=import-error
from redis import Redis #pylint: disable=import-error
s3q = Queue ("S3", connection = Redis())
def rate_str (cache_filename, t):
size_bytes = os.path.getsize (cache_filename)
return "%s %s in %s (%s)/s" % ( cache_filename, sizeof_fmt(size_bytes), t.interval, sizeof_fmt (size_bytes/t.interval))
def s3_download(bucket, key, cache_filename, creds, blocking):
s3_client = boto3.client ('s3', **creds)
with Locks (None, cache_filename, failonexist=True) as l:
if l.locked is True:
with Timer () as t:
def docker_setup(image, command, base_image=None, params=None):
#print "PARAMS:", params
module_config = read_config('runtime-module.cfg', "command")
docker_params = read_config('runtime-bisque.cfg', "docker")
print "Docker ", docker_params
# Load and setup the the environment for storage in docker container
from bq.engine.controllers.command_run import BaseRunner
runner = BaseRunner()
runner.read_config() # loads runtime-module and runtime-bisuqe
runner.load_section("command", runner.module_cfg)
runner.init_runstate(['staging_path=.', 'mex_id='])
runner.create_environments()
runner.process_config()
runner.setup_environments(build=True)
files = runner.mexes[0].files
files = [s.strip() for s in files.split(',')] if isinstance(
files, basestring) else files
print "BUILDING", files
runner.mexes[0].files = files
docker_params = runner.config
command = runner.mexes[0].executable[1:]
if base_image is None:
import platform
base_image = ":".join(platform.linux_distribution()[:2])
# Must be lowercase
if not asbool(docker_params.get('docker.enabled', False)):
return
image = docker_params.get('docker.image')
docker_hub = docker_params.get('docker.hub', '')
docker_user = docker_params.get('docker.hub.user', '')
docker_pass = docker_params.get('docker.hub.password', '')
docker_email = docker_params.get('docker.hub.email', '')
docker_default_tag = docker_params.get('docker.default_tag', None)
if docker_default_tag and ':' not in image:
image = "{}:{}".format(image, docker_default_tag)
image = "/".join(
filter(
lambda x: x,
[docker_hub, docker_user, image.lower()]))
created_docker = False
if not os.path.exists('Dockerfile'):
#files = [ x.strip() for x in module_config.get ('files','').split (",") ]
files = runner.mexes[0].files
dirs = [x for x in files if os.path.isdir(x)]
files = [x for x in files if os.path.isfile(x)]
copies = []
print "FILES", files
if files:
copies.append("COPY %s /module/" % " ".join(files))
for dr in dirs:
copies.append("COPY %s /module/%s/ " % (dr, dr))
created_docker = True
with open('Dockerfile', 'w') as f:
maintainer = docker_params.get('docker.maintainer',
'*****@*****.**')
# check for configured base i.e docker.image.matlab_runtime
base = docker_params.get('docker.image.%s' % base_image,
base_image)
f.write(
string.Template(DOCKERFILE).safe_substitute(
base_image=base_image,
command=command,
maintainer=maintainer,
copy="\n".join(copies)))
print "Calling", " ".join(['docker', 'build', '-q', '-t', image, '.'])
check_call(['docker', 'build', '-t', image, '.'])
if docker_hub:
print "Pushing %s " % (image)
if docker_user and docker_pass:
check_call([
'docker', 'login', '-u', docker_user, '-p', docker_pass, '-e',
docker_email, docker_hub
])
check_call(['docker', 'push', image])
if created_docker:
os.unlink('Dockerfile')
def _default(self, *path, **kw):
request = tg.request
response = tg.response
path = list(path)
resource = None
if not hasattr(request, 'bisque'):
bisque = Bunch()
request.bisque = bisque
bisque = request.bisque
user_id = identity.get_user_id()
usecache = asbool(kw.pop('cache', True))
http_method = request.method.lower()
log.debug('Request "%s" with %s?%s', http_method, request.path,
str(kw))
#log.debug ('Request "%s" ', path)
#check the http method is supported.
try:
method_name = dict(get='get',
head='check',
post='append',
put='modify',
delete='delete')[http_method]
except KeyError:
abort(501)
if not path: #If the request path is to a collection.
#self.check_cache_header(http_method, resource)
if http_method == 'post':
#If the method is a post, we call self.create which returns
#a class which is passed into the self.new method.
resource = self.create(**kw)
assert resource is not None
method_name = 'new'
elif http_method == 'get':
resource = getattr(request.bisque, 'parent', None)
method_name = 'dir'
# if parent:
# self.check_cache_header (http_method, parent)
# #If the method is a get, call the self.index method, which
# #should list the contents of the collection.
# accept_header = headers = value = None
# if usecache:
# headers, value = self.server_cache.fetch(request.url, user=user_id)
# if headers:
# _, accept_header = find_formatter (accept_header=request.headers.get ('accept'))
# content_type = headers.get ('Content-Type')
# if value and accept_header == content_type:
# response.headers.update(headers) # cherrypy.response.headers.update (headers)
# else:
# #self.add_cache_header(None)
# value = self.dir(**kw)
# self.server_cache.save (request.url,
# response.headers,
# value, user=user_id)
# #self.add_cache_header(resource)
# return value
elif http_method == 'put':
resource = getattr(bisque, 'parent', None)
method_name = 'replace_all'
elif http_method == 'delete':
resource = getattr(bisque, 'parent', None)
method_name = 'delete_all'
elif http_method == 'head':
# Determine whether the collection has changed
resource = getattr(bisque, 'parent', None)
method_name = "check"
else:
#Any other methods get rejected.
abort(501)
if resource is None and method_name != 'dir':
#if we don't have a resource by now, (it wasn't created)
#then try and load one.
if path:
token = path.pop(0)
resource = self.load(token)
if resource is None:
#No resource found?
if user_id is None:
abort(401)
abort(404)
#if we have a path, check if the first token matches this
#classes children.
if path:
token = path.pop(0)
log.debug('Token: ' + str(token))
child = self.get_child_resource(token)
if child is not None:
bisque.parent = resource
log.debug("parent = %s", str(resource))
#call down into the child resource.
return child._default(*path, **kw)
# if http_method == 'get':
# #if this resource has children, make sure it has a '/'
# #on the end of the URL
# if getattr(self, 'children', None) is not None:
# if request.path[-1:] != '/':
# redirect(request.path + "/")
#resource = self.server_cache.force_load(resource)
self.check_cache_header(http_method, resource)
method = getattr(self, method_name)
#pylons.response.headers['Content-Type'] = 'text/xml'
log.debug("Dispatch for %s", method_name)
try:
if http_method in ('post', 'put'):
clen = int(request.headers.get('Content-Length', 0))
content_type = request.headers.get('Content-Type')
inputer = find_inputer(content_type)
if not inputer:
log.debug("Bad media type in post/put:%s", content_type)
abort(415, "Bad media type in post/put:%s" % content_type)
# xml arg is for backward compat
value = method(resource,
xml=inputer(request.body_file, clen),
**kw)
# if content.startswith('text/xml') or \
# content.startswith('application/xml'):
# data = request.body_file.read(clen)
# #log.debug('POST '+ data)
# #kw['xml_text'] = data
# value = method(resource, xml=data, **kw)
# elif content.startswith("application/json"):
# try:
# #data = request.body_file.read(clen)
# data = d2xml (json.load (request.body_file))
# value = method(resource, xml=data, **kw)
# except Exception as e:
# log.exception ("while reading json content")
# abort(415, "Bad media type in post/put:%s" % content )
# else:
# #response = method(resource, doc = None, **kw)
# # Raise illegal operation (you should provide XML)
# log.debug ("Bad media type in post/put:%s" , content)
# abort(415, "Bad media type in post/put:%s" % content )
# #self.server_cache.invalidate(request.url, user=user_id)
self.server_cache.invalidate_resource(resource, user=user_id)
elif http_method == 'delete':
self.server_cache.invalidate_resource(resource, user=user_id)
value = method(resource, **kw)
#self.server_cache.invalidate(request.url, user=user_id)
elif http_method == 'get':
accept_header = headers = value = None
if usecache:
headers, value = self.server_cache.fetch(request.url,
user=user_id)
if headers:
content_type = headers.get('Content-Type')
_, accept_header = find_formatter(
accept_header=request.headers.get('accept'))
if value and accept_header == content_type:
response.headers.update(headers)
return value
else:
#run the requested method, passing it the resource
value = method(resource, **kw)
# SET content length?
self.server_cache.save(request.url,
response.headers,
value,
user=user_id)
self.add_cache_header(resource)
else: # http_method == HEAD
value = method(resource, **kw)
#set the last modified date header for the response
return value
except identity.BQIdentityException:
response.status_int = 401
return "<response>FAIL</response>"
from tg.configuration import config
#from tg.controllers import CUSTOM_CONTENT_TYPE
from bq.core import identity
from bq.core.service import ServiceController
#from bq.exceptions import RequestError
from bq.util.paths import data_path
from bq.util.converters import asbool
#from bq.util.copylink import copy_link
from bq.util.io_misc import dolink
#from bq.util.xmldict import d2xml
from .formats import find_inputer, find_formatter
log = logging.getLogger("bq.data_service.resource")
CACHING = asbool(config.get('bisque.data_service.caching', True))
ETAGS = asbool(config.get('bisque.data_service.etags', True))
#SERVER_CACHE = asbool(config.get('bisque.data_service.server_cache', True))
CACHEDIR = config.get('bisque.data_service.server_cache',
data_path('server_cache'))
URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?")
def parse_uri(uri):
"""Parses a URI using the regex given in Appendix B of RFC 3986.
(scheme, authority, path, query, fragment) = parse_uri(uri)
"""
groups = URI.match(uri).groups()
return (groups[1], groups[3], groups[4], groups[6], groups[8])