def push(self, fp, storeurl, uniq=None):
"Push a local file (file pointer) to the store"
log.debug('local.push: url=%s', storeurl)
origpath = localpath = url2localpath(storeurl)
fpath,ext = os.path.splitext(origpath)
_mkdir (os.path.dirname(localpath))
uniq = uniq or make_uniq_code()
for x in xrange(len(uniq)-7):
#for x in range(100):
if not os.path.exists (localpath):
log.debug('local.write: %s -> %s' , tounicode(storeurl), tounicode(localpath))
#patch for no copy file uploads - check for regular file or file like object
try:
move_file (fp, localpath)
except OSError as e:
if not os.path.exists (localpath):
log.exception ("Problem moving file to%s ", localpath)
else:
log.error ("Problem moving file, but it seems to be there.. check permissions on store")
#log.debug ("local.push: top = %s path= %s",self.top_path, localpath )
ident = localpath[len(self.top_path):]
#if ident[0] == '/':
# ident = ident[1:]
ident = localpath2url(ident)
log.info('local push blob_id: %s -> %s', tounicode(ident), tounicode(localpath))
return ident, localpath
localpath = "%s-%s%s" % (fpath , uniq[3:7+x] , ext)
#localpath = "%s-%04d%s" % (fpath , x , ext)
log.warn ("local.write: File exists... trying %s", tounicode(localpath))
raise DuplicateFile(localpath)
def valid(self, storeurl):
#log.debug('valid ident %s top %s', ident, self.top)
#log.debug('valid local ident %s local top %s', url2localpath(ident), url2localpath(self.top))
if store_compare(storeurl, self.mount_url):
return storeurl
# It might be a shorted
storeurl,_ = split_subpath(storeurl)
scheme = urlparse.urlparse(storeurl).scheme
if not scheme and storeurl[0] != '/':
storeurl = urlparse.urljoin (self.top, storeurl)
# OLD STYLE : may have written %encoded values to file system
path = posixpath.normpath(urlparse.urlparse(storeurl).path)
path = force_filesys (path)
log.debug ("checking unquoted %s", tounicode(path))
if os.path.exists (path):
return path # not returning an actual URL ..
elif storeurl.startswith('file:///'):
#should have matched earlier
return None
elif storeurl.startswith('file://'):
storeurl = urlparse.urljoin(self.top, storeurl[7:])
else:
return None
localpath = url2localpath (storeurl)
log.debug ("checking %s", tounicode(localpath))
return os.path.exists(localpath) and localpath2url(localpath)
def _save_store(self,
store,
storepath,
resource,
fileobj=None,
rooturl=None):
'store the file to the named store'
resource_name, sub = split_subpath(resource.get('name'))
# Force a move into the store
if fileobj:
with self._get_driver(store) as driver:
if driver.readonly:
raise IllegalOperation('readonly store')
storeurl = posixpath.join(driver.mount_url, storepath)
log.debug('_save_store: %s from %s %s', storeurl,
driver.mount_url, storepath)
storeurl, localpath = driver.push(
fileobj, storeurl, resource.get('resource_uniq'))
# Store URL may be changed during driver push by disambiguation code.
# revert change from changeset:2297 to always use storeurl
#name = os.path.basename(resource_name) or tounicode(url2localpath(os.path.basename(storeurl)))
name = tounicode(url2localpath(os.path.basename(storeurl)))
resource.set('name', join_subpath(name, sub))
resource.set('value', join_subpath(storeurl, sub))
log.debug('_save_store: %s', etree.tostring(resource))
else:
# Try to reference the data in place (if on safe store)
storeurl, localpath = self._save_storerefs(store, storepath,
resource, rooturl)
#Store url may have changed due to conflict
if resource.get('value') is None: # This is multifile
name = os.path.basename(resource_name)
else:
name = tounicode(url2localpath(os.path.basename(storeurl)))
resource.set('name', join_subpath(name, sub))
# Update the store path reference to similar to the storeurl
storepath = storepath.split('/')
storepath[-1] = os.path.basename(storeurl)
log.debug('_save_store: %s %s %s', storeurl, localpath,
etree.tostring(resource))
return storeurl, storepath, localpath
def call_imgcnvlib(command):
arr = (ctypes.c_wchar_p * len(command))()
arr[:] = [misc.tounicode(i) for i in command]
res = ctypes.pointer(ctypes.c_char_p())
try:
rw.acquire_write('libimgcnv')
r = imgcnvlib.imgcnv(len(command), arr, res)
rw.release_write('libimgcnv')
except Exception:
log.exception('Exception calling libbioimage')
return 100, None
out = res.contents.value
_ = imgcnvlib.imgcnv_clear(res)
return r, out
def run_read(cls, ifnm, command):
with Locks(ifnm, failonread=(not block_reads)) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
#command, tmp = misc.start_nounicode_win(ifnm, command)
log.debug('run_read dylib command: %s', misc.tounicode(command))
#out = cls.run_command( command )
#misc.end_nounicode_win(tmp)
retcode, out = call_imgcnvlib(command)
if retcode == 100 or retcode == 101: # some error in libbioimage, retry once
log.error('Libioimage retcode %s: retry once: %s', retcode,
command)
retcode, out = call_imgcnvlib(command)
#log.debug('Retcode: %s', retcode)
#log.debug('out: %s', out)
return out
def run(cls, ifnm, ofnm, args, **kw):
'''converts input filename into output using exact arguments as provided in args'''
if not cls.installed:
return None
failonread = kw.get('failonread') or (not block_reads)
tmp = None
with Locks(ifnm, ofnm, failonexist=True) as l:
if l.locked: # the file is not being currently written by another process
command = [cls.CONVERTERCOMMAND]
command.extend(args)
log.debug('Run dylib command: %s', misc.tounicode(command))
proceed = True
if ofnm is not None and os.path.exists(
ofnm) and os.path.getsize(ofnm) > 16:
if kw.get('nooverwrite', False) is True:
proceed = False
log.warning(
'Run: output exists before command [%s], skipping',
misc.tounicode(ofnm))
else:
log.warning(
'Run: output exists before command [%s], overwriting',
misc.tounicode(ofnm))
if proceed is True:
#command, tmp = misc.start_nounicode_win(ifnm, command)
retcode, out = call_imgcnvlib(command)
#misc.end_nounicode_win(tmp)
if retcode == 100 or retcode == 101: # some error in libbioimage, retry once
log.error('Libioimage retcode %s: retry once: %s',
retcode, command)
retcode, out = call_imgcnvlib(command)
if retcode == 99:
# in case of a timeout
log.info('Run: timed-out for [%s]',
misc.tounicode(command))
if ofnm is not None and os.path.exists(ofnm):
os.remove(ofnm)
raise ImageServiceException(
412, 'Requested timeout reached')
if retcode != 0:
log.info('Run: returned [%s] for [%s]', retcode,
misc.tounicode(command))
return None
if ofnm is None:
return str(retcode)
# output file does not exist for some operations, like tiles
# tile command does not produce a file with this filename
# if not os.path.exists(ofnm):
# log.error ('Run: output does not exist after command [%s]', ofnm)
# return None
elif l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# make sure the write of the output file have finished
if ofnm is not None and os.path.exists(ofnm):
with Locks(ofnm, failonread=failonread) as l:
if l.locked is False: # dima: never wait, respond immediately
raise ImageServiceFuture((1, 15))
# safeguard for incorrectly converted files, sometimes only the tiff header can be written
# empty lock files are automatically removed before by lock code
if os.path.exists(
ofnm) and os.path.getsize(ofnm) < cls.MINIMUM_FILE_SIZE:
log.error(
'Run: output file is smaller than %s bytes, probably an error, removing [%s]',
cls.MINIMUM_FILE_SIZE, ofnm)
os.remove(ofnm)
return None
return ofnm