def __init__(self, mount_url=None, credentials = None,
bucket_id=None, location=None,
readonly = False, cache=None, **kw):
"""Create a iRods storage driver:
:param path: s3:// url format_path for where to store files
:param credentials : access_key, secret_key
:param bucket_id: A unique bucket ID to store file
:param location: The S3 location identifier (default is USWest)
:param readonly: set repo readonly
"""
self.mount_url = posixpath.join (mount_url, '')
if credentials:
access_key, secret_key = credentials.split(':')
self.creds = { 'aws_access_key_id': access_key,
'aws_secret_access_key' : secret_key,
}
else:
log.error ('need credentials for S3 store')
self.location = location or Location.USWest
self.bucket_id = bucket_id #config.get('bisque.blob_service.s3.bucket_id')
#self.bucket = None
self.conn = None
self.readonly = asbool(readonly)
self.top = mount_url.split('$')[0]
self.options = kw
#self.mount ()
cache = cache or data_path ('s3_cache')
self.cache = string.Template(cache).safe_substitute(datadir=data_url_path())
def __init__(self, mount_url, readonly=False, credentials=None, cache = None, **kw):
"""Create a iRods storage driver:
:param path: irods:// url format_path for where to store files
:param user: the irods users
:param password: the irods password
:param readonly: set repo readonly
"""
self.mount_url = posixpath.join (mount_url, '')
datadir = data_url_path()
for key, value in kw.items():
setattr(self, key, string.Template(value).safe_substitute(datadir=datadir))
if credentials:
try:
self.user, self.password = [ x.strip('"\'') for x in credentials.split(':') ]
except ValueError:
log.exception ('bad credentials for irods %s', credentials)
#self.user = kw.pop('credentials.user',None) or kw.pop('user',None)
#self.password = kw.pop('credentials.password', None) or kw.pop('password', None)
self.readonly = asbool(readonly)
self.options = kw
self.user = self.user.strip ('"\'')
self.password = self.password.strip ('"\'')
cache = cache or data_path ('irods_cache')
self.cache = string.Template(cache).safe_substitute(datadir=data_url_path())
log.debug('irods.user: %s irods.password: %s' , self.user, self.password)
# Get the constant portion of the path
log.debug("created irods store %s " , self.mount_url)
def __init__(self, server_url):
super(ConnoisseurController, self).__init__(server_url)
self.basepath = os.path.dirname(inspect.getfile(
inspect.currentframe()))
self.workdir = config.get('bisque.connoisseur.models',
data_path('connoisseur'))
#_mkdir (self.workdir)
self.path_models = os.path.join(self.workdir, 'models')
self.path_templates = os.path.join(self.workdir, 'templates')
self.path_images = os.path.join(self.workdir, 'images')
_mkdir(self.path_models)
_mkdir(self.path_templates)
_mkdir(self.path_images)
self.adapters_gobs = PluginManager(
'adapters_gobs', os.path.join(self.basepath, 'adapters_gobs'),
AdapterGobjectsBase)
self.adapters_pixels = PluginManager(
'adapters_pixels', os.path.join(self.basepath, 'adapters_pixels'),
AdapterPixelsBase)
# frameworks available in the system
self.frameworks = PluginManager(
'frameworks', os.path.join(self.basepath, 'frameworks'),
FrameworkBase)
# classifiers available to the system, requested with method argument
self.classifiers = PluginManager(
'classifiers', os.path.join(self.basepath, 'classifiers'),
ClassifierBase)
# importers are matched by their input mime and output mime
self.importers = PluginManager(
'importers', os.path.join(self.basepath, 'importers'),
ImporterBase)
self.importers_by_mime = {
'%s;%s' % (e.mime_input, e.mime_type): e
for k, e in self.importers.plugins.iteritems()
}
#log.debug('Importers: %s', self.importers_by_mime)
# exporters are matched by their input mime and output mime
self.exporters = PluginManager(
'exporters', os.path.join(self.basepath, 'exporters'),
ExporterBase)
self.exporters_by_mime = {
'%s;%s' % (e.mime_input, e.mime_type): e
for k, e in self.exporters.plugins.iteritems()
}
#log.debug('Exporters: %s', self.exporters_by_mime)
# loaded models hashed by their ID
self.models_loaded_max = 10
self.models = {}
log.info('Connoisseur service started, using temp space at: %s' %
self.workdir)
def finish(cleanup_cache=False):
cachedir = config.get('bisque.http_client.cache_dir',
data_path('client_cache'))
if async_request_pool.isrunning():
async_request_pool.stop_pool_handler()
if cleanup_cache:
import os
for filename in os.listdir(cachedir):
os.remove(filename)
def start(need_async=False):
cachedir = config.get('bisque.http_client.cache_dir',
data_path('client_cache'))
global local_client
log.debug("starting http")
if not os.path.exists(cachedir):
os.makedirs(cachedir)
local_client = Http(cachedir, disable_ssl_certificate_validation=True)
if need_async:
async_request_pool.start_pool_handler()
def __init__(self, server_url):
super(ImageServiceController, self).__init__(server_url)
workdir = config.get('bisque.image_service.work_dir',
data_path('workdir'))
rundir = config.get('bisque.paths.run', os.getcwd())
_mkdir(workdir)
log.info('ROOT=%s work=%s run = ', config.get('bisque.root'), workdir)
self.user_map = {}
# users = data_service.query('user', wpublic=1)
# for u in users.xpath('user'):
# self.user_map[u.get('uri')] = u.get('name')
self.srv = ImageServer(work_dir=workdir, run_dir=rundir)
def clearcache(self):
log.info("CLEARING CACHE")
def clearfiles(folder):
for the_file in os.listdir(folder):
file_path = os.path.join(folder, the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
except OSError as e:
log.debug("unlinking failed: %s", file_path)
except Exception as e:
log.exception('while removing %s' % file_path)
server_cache = data_path('server_cache')
clearfiles(server_cache)
log.info("CLEARED CACHE")
return '<resource name="cache_clear" value="finished">'
def savefile(self, **kw):
log.info("savefile request " + str(tg.request))
username = get_username()
# check the user identity here and return 401 if fails
if anonymous():
response.status_int = 401
log.debug('Access denied')
return 'Access denied'
# if requested test for uploaded
hashes_str = kw.pop('hashes', None)
if hashes_str != None:
all_hashes = [fhash.strip() for fhash in hashes_str.split(',')]
#for fhash in hashes_str.split(','):
# all_hashes.append( fhash )
#found_hashes = blob_service.files_exist(all_hashes) TODO
found_hashes = []
found_html = ",".join([str(h) for h in found_hashes])
return "Found: " + found_html
# here user is authenticated - upload
if not 'upload' in kw:
response.status_int = 501
return "No file to be uploaded..."
upload = kw['upload']
uploadroot = config.get('bisque.image_service.upload_dir',
data_path('uploads'))
upload_dir = uploadroot + '/' + str(username)
_mkdir(upload_dir)
if not upload.filename:
return 'No file sent...'
#patch for no copy file uploads - check for regular file or file like object
uploadpath = upload_dir + '/' + upload.filename
#KGK: note upload.file.name is not available for some uploads (attached files)
#abs_path_src = os.path.abspath(upload.file.name)
#if os.path.isfile(abs_path_src):
# shutil.move(abs_path_src, uploadpath)
#else:
with open(uploadpath, 'wb') as trg:
shutil.copyfileobj(upload.file, trg)
return 'Upload done for: ' + upload.filename
from bq.exceptions import BadValue
from bq.core import identity
from bq.util.paths import data_path
from bq.util.urlutil import strip_url_params
from bq.util.hash import make_uniq_code, is_uniq_code
from .bisquik_resource import BisquikResource, force_dbload, RESOURCE_HANDLERS
from .resource_query import resource_query, resource_count, resource_load, resource_delete, resource_types, resource_permission
from . import resource as resource_module
from .resource_query import prepare_permissions, RESOURCE_READ, RESOURCE_EDIT
from .resource_auth import resource_acl, check_access
from .resource import BaseCache, HierarchicalCache
from .formats import find_formatter
#from .doc_resource import XMLDocumentResource
cachedir = config.get('bisque.data_service.server_cache',
data_path('server_cache'))
log = logging.getLogger("bq.data_service")
class DataServerController(ServiceController):
service_type = "data_service"
#doc = XMLDocumentResource ()
def __init__(self, url=None):
super(DataServerController, self).__init__(url)
self.children = {}
self.server_cache = BaseCache()
if resource_module.CACHING:
self.server_cache = HierarchicalCache(cachedir)
import os
import re
import urlparse
import logging
import subprocess
import urllib
# THIS FILE IS NO LONGER USED
# pylint: skip-file
import irods
from bq.util.mkdir import _mkdir
from bq.util.paths import data_path
IRODS_CACHE = data_path('irods_cache')
CONNECTION_POOL = {}
class IrodsError(Exception):
pass
log = logging.getLogger('bq.irods')
PARSE_NET = re.compile(
r'^((?P<user>[^:]+):(?P<password>[\w.#^!;]+)?@)?(?P<host>[^:]+)(?P<port>:\d+)?'
)
irods_env, status = irods.getRodsEnv() # pylint: disable=no-member
# global variables
import bq
import os
from bq.util.paths import data_path
#features dirs
FEATURES_CONTOLLERS_DIR = bq.features.controllers.__path__[0]
EXTRACTOR_DIR = os.path.join(FEATURES_CONTOLLERS_DIR,'extractors')
#data dirs
FEATURES_STORAGE_FILE_DIR = data_path('features')
FEATURES_TABLES_FILE_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR ,'feature_tables')
FEATURES_WORK_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR , 'workdir')
FEATURES_TEMP_DIR = os.path.join(FEATURES_STORAGE_FILE_DIR, 'temp')
def clean_store(stores, options):
from bq.data_service.model import Taggable, Image, DBSession
tops = []
localfiles = set()
for local in stores:
if 'top' not in local:
continue
top = local['top'][7:]
top = string.Template(top).safe_substitute(datadir=data_path())
tops.append(top)
print "Scanning ", top
#for root, dirs, files in local.walk():
for root, dirs, files in os.walk(top):
for f in files:
#if f.endswith('.info'):
# continue
filepath = os.path.join(root, f)
localfiles.add(filepath)
print "file count ", len(localfiles)
dbfiles = []
resources_missing = []
locs = DBSession.query(Taggable).filter(
or_(Taggable.resource_type == 'image',
Taggable.resource_type == 'file',
Taggable.resource_type == 'table'))
for f in locs:
if f.resource_value is None:
# check for sub values
for ref in f.values:
if ref.valstr and ref.valstr.startswith('file://'):
relpath = ref.valstr[7:]
elif f.resource_value.startswith(
'irods') or f.resource_value.startswith('s3'):
continue
elif f.resource_value.startswith('file://'):
relpath = f.resource_value[7:]
else:
relpath = f.resource_value
for top in tops:
filepath = posixpath.join(top, relpath)
if os.path.exists(filepath):
dbfiles.append(filepath)
break
if not dbfiles[-1].endswith(relpath):
resources_missing.append((f.resource_uniq, relpath))
dbfiles = set(dbfiles)
print "DB count", len(dbfiles)
missing = localfiles - dbfiles
print "deleting %s files" % len(missing)
before = disk_usage(top)
if not options.dryrun:
for f in missing:
os.remove(f)
else:
print "would delete %s" % list(missing)[:20]
print "DBFILES:", list(dbfiles)[:20]
print "LOCALFILES", list(localfiles)[:20]
print "resource_missing in DB", resources_missing
after = disk_usage(top)
print "Reclaimed %s space" % sizeof_fmt(before.used - after.used)
def notify(self, **kw):
''' DN upload request using HTTP uploads
this function is only created temporaly to provide backward compatibility
for DN provide http uploads, where directory is not specified by DN
but is known by the server and thus should be ignored here
'''
username = get_username()
#log.debug( 'notify - username: '******'notify needs credentialed user')
abort(401)
log.debug('notify - args: ' + str(kw))
bixfiles = kw.pop('bixfiles', [])
imagefiles = kw.pop('imagefiles', [])
upload_dir = kw.pop('uploaddir', None)
if (upload_dir == None):
uploadroot = config.get('bisque.image_service.upload_dir',
data_path('uploads'))
upload_dir = os.path.join(uploadroot, username)
#upload_dir = uploadroot+'/'+ str(identity.current.user_name)
remove_uploads = config.get('bisque.image_service.remove_uploads',
False)
#identity.set_current_identity(ident)
importer = BIXImporter(upload_dir)
images = []
for bix in bixfiles.split(':'):
if bix != None and bix != '':
try:
name, uri = importer.process_bix(bix)
except Exception:
name = '%s [error importing]' % (bix)
uri = '#'
if name != '' and uri != '':
images.append((name, uri))
if remove_uploads:
try:
os.unlink(os.path.join(upload_dir, bix))
os.unlink(os.path.join(upload_dir, name))
except Exception:
log.debug(
'Error removing temp BIX and/or Image files')
imageshtml = "<table>"
imageshtml += "".join([
'<tr><td><a href="%s">%s</a></td></tr>' % (self.imagelink(u), n)
for n, u in images
])
imageshtml += '</table>'
if len(bixfiles) < len(imagefiles):
inputimages = imagefiles.split(':')
for n, u in images:
inputimages.remove(n)
if remove_uploads:
for name in inputimages:
os.unlink(os.path.join(upload_dir, name))
imageshtml += '<h2>Uploaded but not included as images %d (no meta-data file):</h2>' % (
len(inputimages))
imageshtml += "<table>".join(
['<tr><td>%s</td></tr>' % (n) for n in inputimages])
imageshtml += '</table>'
return '<h2>Uploaded %d images:</h2>' % (len(images)) + imageshtml
def run(self):
#Check for self.command in init..
for p in (data_path('.server_cache'), data_path('.client_cache')):
if os.path.exists(p):
shutil.rmtree(p)
os.makedirs(p)
def data_url_path (*names):
path = data_path(*names)
#if len(path)>1 and path[1]==':': #file:// url requires / for drive lettered path like c: -> file:///c:/path
# path = '/%s'%path
return path
