def __init__(self, mount_url=None, credentials = None, bucket_id=None, location=None, readonly = False, cache=None, **kw): """Create a iRods storage driver: :param path: s3:// url format_path for where to store files :param credentials : access_key, secret_key :param bucket_id: A unique bucket ID to store file :param location: The S3 location identifier (default is USWest) :param readonly: set repo readonly """ self.mount_url = posixpath.join (mount_url, '') if credentials: access_key, secret_key = credentials.split(':') self.creds = { 'aws_access_key_id': access_key, 'aws_secret_access_key' : secret_key, } else: log.error ('need credentials for S3 store') self.location = location or Location.USWest self.bucket_id = bucket_id #config.get('bisque.blob_service.s3.bucket_id') #self.bucket = None self.conn = None self.readonly = asbool(readonly) self.top = mount_url.split('$')[0] self.options = kw #self.mount () cache = cache or data_path ('s3_cache') self.cache = string.Template(cache).safe_substitute(datadir=data_url_path())
def __init__(self, mount_url, readonly=False, credentials=None, cache = None, **kw): """Create a iRods storage driver: :param path: irods:// url format_path for where to store files :param user: the irods users :param password: the irods password :param readonly: set repo readonly """ self.mount_url = posixpath.join (mount_url, '') datadir = data_url_path() for key, value in kw.items(): setattr(self, key, string.Template(value).safe_substitute(datadir=datadir)) if credentials: try: self.user, self.password = [ x.strip('"\'') for x in credentials.split(':') ] except ValueError: log.exception ('bad credentials for irods %s', credentials) #self.user = kw.pop('credentials.user',None) or kw.pop('user',None) #self.password = kw.pop('credentials.password', None) or kw.pop('password', None) self.readonly = asbool(readonly) self.options = kw self.user = self.user.strip ('"\'') self.password = self.password.strip ('"\'') cache = cache or data_path ('irods_cache') self.cache = string.Template(cache).safe_substitute(datadir=data_url_path()) log.debug('irods.user: %s irods.password: %s' , self.user, self.password) # Get the constant portion of the path log.debug("created irods store %s " , self.mount_url)
def __init__(self, server_url): super(ConnoisseurController, self).__init__(server_url) self.basepath = os.path.dirname(inspect.getfile( inspect.currentframe())) self.workdir = config.get('bisque.connoisseur.models', data_path('connoisseur')) #_mkdir (self.workdir) self.path_models = os.path.join(self.workdir, 'models') self.path_templates = os.path.join(self.workdir, 'templates') self.path_images = os.path.join(self.workdir, 'images') _mkdir(self.path_models) _mkdir(self.path_templates) _mkdir(self.path_images) self.adapters_gobs = PluginManager( 'adapters_gobs', os.path.join(self.basepath, 'adapters_gobs'), AdapterGobjectsBase) self.adapters_pixels = PluginManager( 'adapters_pixels', os.path.join(self.basepath, 'adapters_pixels'), AdapterPixelsBase) # frameworks available in the system self.frameworks = PluginManager( 'frameworks', os.path.join(self.basepath, 'frameworks'), FrameworkBase) # classifiers available to the system, requested with method argument self.classifiers = PluginManager( 'classifiers', os.path.join(self.basepath, 'classifiers'), ClassifierBase) # importers are matched by their input mime and output mime self.importers = PluginManager( 'importers', os.path.join(self.basepath, 'importers'), ImporterBase) self.importers_by_mime = { '%s;%s' % (e.mime_input, e.mime_type): e for k, e in self.importers.plugins.iteritems() } #log.debug('Importers: %s', self.importers_by_mime) # exporters are matched by their input mime and output mime self.exporters = PluginManager( 'exporters', os.path.join(self.basepath, 'exporters'), ExporterBase) self.exporters_by_mime = { '%s;%s' % (e.mime_input, e.mime_type): e for k, e in self.exporters.plugins.iteritems() } #log.debug('Exporters: %s', self.exporters_by_mime) # loaded models hashed by their ID self.models_loaded_max = 10 self.models = {} log.info('Connoisseur service started, using temp space at: %s' % self.workdir)
def finish(cleanup_cache=False): cachedir = config.get('bisque.http_client.cache_dir', data_path('client_cache')) if async_request_pool.isrunning(): async_request_pool.stop_pool_handler() if cleanup_cache: import os for filename in os.listdir(cachedir): os.remove(filename)
def start(need_async=False): cachedir = config.get('bisque.http_client.cache_dir', data_path('client_cache')) global local_client log.debug("starting http") if not os.path.exists(cachedir): os.makedirs(cachedir) local_client = Http(cachedir, disable_ssl_certificate_validation=True) if need_async: async_request_pool.start_pool_handler()
def __init__(self, server_url): super(ImageServiceController, self).__init__(server_url) workdir = config.get('bisque.image_service.work_dir', data_path('workdir')) rundir = config.get('bisque.paths.run', os.getcwd()) _mkdir(workdir) log.info('ROOT=%s work=%s run = ', config.get('bisque.root'), workdir) self.user_map = {} # users = data_service.query('user', wpublic=1) # for u in users.xpath('user'): # self.user_map[u.get('uri')] = u.get('name') self.srv = ImageServer(work_dir=workdir, run_dir=rundir)
def clearcache(self): log.info("CLEARING CACHE") def clearfiles(folder): for the_file in os.listdir(folder): file_path = os.path.join(folder, the_file) try: if os.path.isfile(file_path): os.unlink(file_path) except OSError as e: log.debug("unlinking failed: %s", file_path) except Exception as e: log.exception('while removing %s' % file_path) server_cache = data_path('server_cache') clearfiles(server_cache) log.info("CLEARED CACHE") return '<resource name="cache_clear" value="finished">'
def savefile(self, **kw): log.info("savefile request " + str(tg.request)) username = get_username() # check the user identity here and return 401 if fails if anonymous(): response.status_int = 401 log.debug('Access denied') return 'Access denied' # if requested test for uploaded hashes_str = kw.pop('hashes', None) if hashes_str != None: all_hashes = [fhash.strip() for fhash in hashes_str.split(',')] #for fhash in hashes_str.split(','): # all_hashes.append( fhash ) #found_hashes = blob_service.files_exist(all_hashes) TODO found_hashes = [] found_html = ",".join([str(h) for h in found_hashes]) return "Found: " + found_html # here user is authenticated - upload if not 'upload' in kw: response.status_int = 501 return "No file to be uploaded..." upload = kw['upload'] uploadroot = config.get('bisque.image_service.upload_dir', data_path('uploads')) upload_dir = uploadroot + '/' + str(username) _mkdir(upload_dir) if not upload.filename: return 'No file sent...' #patch for no copy file uploads - check for regular file or file like object uploadpath = upload_dir + '/' + upload.filename #KGK: note upload.file.name is not available for some uploads (attached files) #abs_path_src = os.path.abspath(upload.file.name) #if os.path.isfile(abs_path_src): # shutil.move(abs_path_src, uploadpath) #else: with open(uploadpath, 'wb') as trg: shutil.copyfileobj(upload.file, trg) return 'Upload done for: ' + upload.filename
from bq.exceptions import BadValue from bq.core import identity from bq.util.paths import data_path from bq.util.urlutil import strip_url_params from bq.util.hash import make_uniq_code, is_uniq_code from .bisquik_resource import BisquikResource, force_dbload, RESOURCE_HANDLERS from .resource_query import resource_query, resource_count, resource_load, resource_delete, resource_types, resource_permission from . import resource as resource_module from .resource_query import prepare_permissions, RESOURCE_READ, RESOURCE_EDIT from .resource_auth import resource_acl, check_access from .resource import BaseCache, HierarchicalCache from .formats import find_formatter #from .doc_resource import XMLDocumentResource cachedir = config.get('bisque.data_service.server_cache', data_path('server_cache')) log = logging.getLogger("bq.data_service") class DataServerController(ServiceController): service_type = "data_service" #doc = XMLDocumentResource () def __init__(self, url=None): super(DataServerController, self).__init__(url) self.children = {} self.server_cache = BaseCache() if resource_module.CACHING: self.server_cache = HierarchicalCache(cachedir)
import os import re import urlparse import logging import subprocess import urllib # THIS FILE IS NO LONGER USED # pylint: skip-file import irods from bq.util.mkdir import _mkdir from bq.util.paths import data_path IRODS_CACHE = data_path('irods_cache') CONNECTION_POOL = {} class IrodsError(Exception): pass log = logging.getLogger('bq.irods') PARSE_NET = re.compile( r'^((?P<user>[^:]+):(?P<password>[\w.#^!;]+)?@)?(?P<host>[^:]+)(?P<port>:\d+)?' ) irods_env, status = irods.getRodsEnv() # pylint: disable=no-member
# global variables import bq import os from bq.util.paths import data_path #features dirs FEATURES_CONTOLLERS_DIR = bq.features.controllers.__path__[0] EXTRACTOR_DIR = os.path.join(FEATURES_CONTOLLERS_DIR,'extractors') #data dirs FEATURES_STORAGE_FILE_DIR = data_path('features') FEATURES_TABLES_FILE_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR ,'feature_tables') FEATURES_WORK_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR , 'workdir') FEATURES_TEMP_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR, 'temp')
def clean_store(stores, options): from bq.data_service.model import Taggable, Image, DBSession tops = [] localfiles = set() for local in stores: if 'top' not in local: continue top = local['top'][7:] top = string.Template(top).safe_substitute(datadir=data_path()) tops.append(top) print "Scanning ", top #for root, dirs, files in local.walk(): for root, dirs, files in os.walk(top): for f in files: #if f.endswith('.info'): # continue filepath = os.path.join(root, f) localfiles.add(filepath) print "file count ", len(localfiles) dbfiles = [] resources_missing = [] locs = DBSession.query(Taggable).filter( or_(Taggable.resource_type == 'image', Taggable.resource_type == 'file', Taggable.resource_type == 'table')) for f in locs: if f.resource_value is None: # check for sub values for ref in f.values: if ref.valstr and ref.valstr.startswith('file://'): relpath = ref.valstr[7:] elif f.resource_value.startswith( 'irods') or f.resource_value.startswith('s3'): continue elif f.resource_value.startswith('file://'): relpath = f.resource_value[7:] else: relpath = f.resource_value for top in tops: filepath = posixpath.join(top, relpath) if os.path.exists(filepath): dbfiles.append(filepath) break if not dbfiles[-1].endswith(relpath): resources_missing.append((f.resource_uniq, relpath)) dbfiles = set(dbfiles) print "DB count", len(dbfiles) missing = localfiles - dbfiles print "deleting %s files" % len(missing) before = disk_usage(top) if not options.dryrun: for f in missing: os.remove(f) else: print "would delete %s" % list(missing)[:20] print "DBFILES:", list(dbfiles)[:20] print "LOCALFILES", list(localfiles)[:20] print "resource_missing in DB", resources_missing after = disk_usage(top) print "Reclaimed %s space" % sizeof_fmt(before.used - after.used)
def notify(self, **kw): ''' DN upload request using HTTP uploads this function is only created temporaly to provide backward compatibility for DN provide http uploads, where directory is not specified by DN but is known by the server and thus should be ignored here ''' username = get_username() #log.debug( 'notify - username: '******'notify needs credentialed user') abort(401) log.debug('notify - args: ' + str(kw)) bixfiles = kw.pop('bixfiles', []) imagefiles = kw.pop('imagefiles', []) upload_dir = kw.pop('uploaddir', None) if (upload_dir == None): uploadroot = config.get('bisque.image_service.upload_dir', data_path('uploads')) upload_dir = os.path.join(uploadroot, username) #upload_dir = uploadroot+'/'+ str(identity.current.user_name) remove_uploads = config.get('bisque.image_service.remove_uploads', False) #identity.set_current_identity(ident) importer = BIXImporter(upload_dir) images = [] for bix in bixfiles.split(':'): if bix != None and bix != '': try: name, uri = importer.process_bix(bix) except Exception: name = '%s [error importing]' % (bix) uri = '#' if name != '' and uri != '': images.append((name, uri)) if remove_uploads: try: os.unlink(os.path.join(upload_dir, bix)) os.unlink(os.path.join(upload_dir, name)) except Exception: log.debug( 'Error removing temp BIX and/or Image files') imageshtml = "<table>" imageshtml += "".join([ '<tr><td><a href="%s">%s</a></td></tr>' % (self.imagelink(u), n) for n, u in images ]) imageshtml += '</table>' if len(bixfiles) < len(imagefiles): inputimages = imagefiles.split(':') for n, u in images: inputimages.remove(n) if remove_uploads: for name in inputimages: os.unlink(os.path.join(upload_dir, name)) imageshtml += '<h2>Uploaded but not included as images %d (no meta-data file):</h2>' % ( len(inputimages)) imageshtml += "<table>".join( ['<tr><td>%s</td></tr>' % (n) for n in inputimages]) imageshtml += '</table>' return '<h2>Uploaded %d images:</h2>' % (len(images)) + imageshtml
def run(self): #Check for self.command in init.. for p in (data_path('.server_cache'), data_path('.client_cache')): if os.path.exists(p): shutil.rmtree(p) os.makedirs(p)
def data_url_path (*names): path = data_path(*names) #if len(path)>1 and path[1]==':': #file:// url requires / for drive lettered path like c: -> file:///c:/path # path = '/%s'%path return path