def index(self, refresh='false'): """Return the list of available modules urls""" #server = urlparse.urljoin(config.get('bisque.server'), self.service_type) server = urlparse.urljoin(tg.request.host_url, self.service_type) if asbool(refresh): self.refresh() modules = [("%s/%s" % (server, k), k) for k in sorted(self.module_by_name.keys())] return dict(modules=modules)
def process_config(self, runner, **kw): runner.load_section('docker', runner.bisque_cfg) runner.load_section('docker', runner.module_cfg) self.enabled = asbool(runner.config.get('docker.enabled', False)) #self.docker_hub = runner.config.get('docker.hub', '') #self.docker_image = runner.config.get('docker.image', '') #self.docker_user = runner.config.get ('docker.hub.user', '') #self.docker_pass = runner.config.get('docker.hub.password', '') #self.docker_email = runner.config.get('docker.hub.email', '') #self.docker_login_tmpl = runner.config.get ('docker.login_tmpl', self.docker_login_tmpl) self.docker_params = AttrDict() for k in self.docker_keys: self.docker_params[k.replace('.', '_')] = runner.config.get(k, '')
def setup(params, *args, **kw): docker_params = read_config('runtime-bisque.cfg', "docker") if not asbool(docker_params.get('docker.enabled', False)): print "No Docker available... cannot set up module." return 1 try: # clone Dream.3D UCSB repo and build Docker image tmp_dir = tempfile.mkdtemp() print "Cloning code into %s..." % tmp_dir p = Popen([ 'git', 'clone', '[email protected]:wlenthe/UCSB_DREAM3D', '%s/source' % tmp_dir ], stdout=PIPE) if p.wait() != 0: print "Dream.3D repo could not be cloned... cannot set up module." return 1 # build Docker image print "Building Dream.3D docker image..." p = Popen([ 'timeout', '-s', 'SIGKILL', '120m', 'docker', 'build', '--tag', 'dream3d_ucsb', '%s/source' % tmp_dir ], stdout=PIPE) while True: line = p.stdout.readline() if line == '': break m = re.search('\[[ ]*[0-9]+%\]', line) if m: print '\r\r\r\r\r\r', print m.group(0), sys.stdout.flush() print if p.wait() != 0: print "Dream.3D image could not be built... cannot set up module." return 1 python_setup('Dream3D.py', params=params) docker_setup('bisque_dream3d_ucsb', 'Dream3D', 'dream3d_ucsb', params=params) finally: # delete tmp_dir shutil.rmtree(tmp_dir)
from bq.util.locks import Locks, FileLocked from bq.util.timer import Timer from bq.util.sizeoffmt import sizeof_fmt from bq.util.converters import asbool class S3Error(Exception): pass log = logging.getLogger('bq.blobs.storage.s3') #def s3_parse_url(url): # "Read an s3 url, return a bucket and key" # pass s3q=None if asbool (config.get('bisque.s3queue')): from rq import Queue #pylint: disable=import-error from redis import Redis #pylint: disable=import-error s3q = Queue ("S3", connection = Redis()) def rate_str (cache_filename, t): size_bytes = os.path.getsize (cache_filename) return "%s %s in %s (%s)/s" % ( cache_filename, sizeof_fmt(size_bytes), t.interval, sizeof_fmt (size_bytes/t.interval)) def s3_download(bucket, key, cache_filename, creds, blocking): s3_client = boto3.client ('s3', **creds) with Locks (None, cache_filename, failonexist=True) as l: if l.locked is True: with Timer () as t:
def docker_setup(image, command, base_image=None, params=None): #print "PARAMS:", params module_config = read_config('runtime-module.cfg', "command") docker_params = read_config('runtime-bisque.cfg', "docker") print "Docker ", docker_params # Load and setup the the environment for storage in docker container from bq.engine.controllers.command_run import BaseRunner runner = BaseRunner() runner.read_config() # loads runtime-module and runtime-bisuqe runner.load_section("command", runner.module_cfg) runner.init_runstate(['staging_path=.', 'mex_id=']) runner.create_environments() runner.process_config() runner.setup_environments(build=True) files = runner.mexes[0].files files = [s.strip() for s in files.split(',')] if isinstance( files, basestring) else files print "BUILDING", files runner.mexes[0].files = files docker_params = runner.config command = runner.mexes[0].executable[1:] if base_image is None: import platform base_image = ":".join(platform.linux_distribution()[:2]) # Must be lowercase if not asbool(docker_params.get('docker.enabled', False)): return image = docker_params.get('docker.image') docker_hub = docker_params.get('docker.hub', '') docker_user = docker_params.get('docker.hub.user', '') docker_pass = docker_params.get('docker.hub.password', '') docker_email = docker_params.get('docker.hub.email', '') docker_default_tag = docker_params.get('docker.default_tag', None) if docker_default_tag and ':' not in image: image = "{}:{}".format(image, docker_default_tag) image = "/".join( filter( lambda x: x, [docker_hub, docker_user, image.lower()])) created_docker = False if not os.path.exists('Dockerfile'): #files = [ x.strip() for x in module_config.get ('files','').split (",") ] files = runner.mexes[0].files dirs = [x for x in files if os.path.isdir(x)] files = [x for x in files if os.path.isfile(x)] copies = [] print "FILES", files if files: copies.append("COPY %s /module/" % " ".join(files)) for dr in dirs: copies.append("COPY %s /module/%s/ " % (dr, dr)) created_docker = True with open('Dockerfile', 'w') as f: maintainer = docker_params.get('docker.maintainer', '*****@*****.**') # check for configured base i.e docker.image.matlab_runtime base = docker_params.get('docker.image.%s' % base_image, base_image) f.write( string.Template(DOCKERFILE).safe_substitute( base_image=base_image, command=command, maintainer=maintainer, copy="\n".join(copies))) print "Calling", " ".join(['docker', 'build', '-q', '-t', image, '.']) check_call(['docker', 'build', '-t', image, '.']) if docker_hub: print "Pushing %s " % (image) if docker_user and docker_pass: check_call([ 'docker', 'login', '-u', docker_user, '-p', docker_pass, '-e', docker_email, docker_hub ]) check_call(['docker', 'push', image]) if created_docker: os.unlink('Dockerfile')
def _default(self, *path, **kw): request = tg.request response = tg.response path = list(path) resource = None if not hasattr(request, 'bisque'): bisque = Bunch() request.bisque = bisque bisque = request.bisque user_id = identity.get_user_id() usecache = asbool(kw.pop('cache', True)) http_method = request.method.lower() log.debug('Request "%s" with %s?%s', http_method, request.path, str(kw)) #log.debug ('Request "%s" ', path) #check the http method is supported. try: method_name = dict(get='get', head='check', post='append', put='modify', delete='delete')[http_method] except KeyError: abort(501) if not path: #If the request path is to a collection. #self.check_cache_header(http_method, resource) if http_method == 'post': #If the method is a post, we call self.create which returns #a class which is passed into the self.new method. resource = self.create(**kw) assert resource is not None method_name = 'new' elif http_method == 'get': resource = getattr(request.bisque, 'parent', None) method_name = 'dir' # if parent: # self.check_cache_header (http_method, parent) # #If the method is a get, call the self.index method, which # #should list the contents of the collection. # accept_header = headers = value = None # if usecache: # headers, value = self.server_cache.fetch(request.url, user=user_id) # if headers: # _, accept_header = find_formatter (accept_header=request.headers.get ('accept')) # content_type = headers.get ('Content-Type') # if value and accept_header == content_type: # response.headers.update(headers) # cherrypy.response.headers.update (headers) # else: # #self.add_cache_header(None) # value = self.dir(**kw) # self.server_cache.save (request.url, # response.headers, # value, user=user_id) # #self.add_cache_header(resource) # return value elif http_method == 'put': resource = getattr(bisque, 'parent', None) method_name = 'replace_all' elif http_method == 'delete': resource = getattr(bisque, 'parent', None) method_name = 'delete_all' elif http_method == 'head': # Determine whether the collection has changed resource = getattr(bisque, 'parent', None) method_name = "check" else: #Any other methods get rejected. abort(501) if resource is None and method_name != 'dir': #if we don't have a resource by now, (it wasn't created) #then try and load one. if path: token = path.pop(0) resource = self.load(token) if resource is None: #No resource found? if user_id is None: abort(401) abort(404) #if we have a path, check if the first token matches this #classes children. if path: token = path.pop(0) log.debug('Token: ' + str(token)) child = self.get_child_resource(token) if child is not None: bisque.parent = resource log.debug("parent = %s", str(resource)) #call down into the child resource. return child._default(*path, **kw) # if http_method == 'get': # #if this resource has children, make sure it has a '/' # #on the end of the URL # if getattr(self, 'children', None) is not None: # if request.path[-1:] != '/': # redirect(request.path + "/") #resource = self.server_cache.force_load(resource) self.check_cache_header(http_method, resource) method = getattr(self, method_name) #pylons.response.headers['Content-Type'] = 'text/xml' log.debug("Dispatch for %s", method_name) try: if http_method in ('post', 'put'): clen = int(request.headers.get('Content-Length', 0)) content_type = request.headers.get('Content-Type') inputer = find_inputer(content_type) if not inputer: log.debug("Bad media type in post/put:%s", content_type) abort(415, "Bad media type in post/put:%s" % content_type) # xml arg is for backward compat value = method(resource, xml=inputer(request.body_file, clen), **kw) # if content.startswith('text/xml') or \ # content.startswith('application/xml'): # data = request.body_file.read(clen) # #log.debug('POST '+ data) # #kw['xml_text'] = data # value = method(resource, xml=data, **kw) # elif content.startswith("application/json"): # try: # #data = request.body_file.read(clen) # data = d2xml (json.load (request.body_file)) # value = method(resource, xml=data, **kw) # except Exception as e: # log.exception ("while reading json content") # abort(415, "Bad media type in post/put:%s" % content ) # else: # #response = method(resource, doc = None, **kw) # # Raise illegal operation (you should provide XML) # log.debug ("Bad media type in post/put:%s" , content) # abort(415, "Bad media type in post/put:%s" % content ) # #self.server_cache.invalidate(request.url, user=user_id) self.server_cache.invalidate_resource(resource, user=user_id) elif http_method == 'delete': self.server_cache.invalidate_resource(resource, user=user_id) value = method(resource, **kw) #self.server_cache.invalidate(request.url, user=user_id) elif http_method == 'get': accept_header = headers = value = None if usecache: headers, value = self.server_cache.fetch(request.url, user=user_id) if headers: content_type = headers.get('Content-Type') _, accept_header = find_formatter( accept_header=request.headers.get('accept')) if value and accept_header == content_type: response.headers.update(headers) return value else: #run the requested method, passing it the resource value = method(resource, **kw) # SET content length? self.server_cache.save(request.url, response.headers, value, user=user_id) self.add_cache_header(resource) else: # http_method == HEAD value = method(resource, **kw) #set the last modified date header for the response return value except identity.BQIdentityException: response.status_int = 401 return "<response>FAIL</response>"
from tg.configuration import config #from tg.controllers import CUSTOM_CONTENT_TYPE from bq.core import identity from bq.core.service import ServiceController #from bq.exceptions import RequestError from bq.util.paths import data_path from bq.util.converters import asbool #from bq.util.copylink import copy_link from bq.util.io_misc import dolink #from bq.util.xmldict import d2xml from .formats import find_inputer, find_formatter log = logging.getLogger("bq.data_service.resource") CACHING = asbool(config.get('bisque.data_service.caching', True)) ETAGS = asbool(config.get('bisque.data_service.etags', True)) #SERVER_CACHE = asbool(config.get('bisque.data_service.server_cache', True)) CACHEDIR = config.get('bisque.data_service.server_cache', data_path('server_cache')) URI = re.compile(r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?") def parse_uri(uri): """Parses a URI using the regex given in Appendix B of RFC 3986. (scheme, authority, path, query, fragment) = parse_uri(uri) """ groups = URI.match(uri).groups() return (groups[1], groups[3], groups[4], groups[6], groups[8])