def push(self, fp, storeurl, uniq=None): "Push a local file (file pointer) to the store" log.debug('local.push: url=%s', storeurl) origpath = localpath = url2localpath(storeurl) fpath,ext = os.path.splitext(origpath) _mkdir (os.path.dirname(localpath)) uniq = uniq or make_uniq_code() for x in xrange(len(uniq)-7): #for x in range(100): if not os.path.exists (localpath): log.debug('local.write: %s -> %s' , tounicode(storeurl), tounicode(localpath)) #patch for no copy file uploads - check for regular file or file like object try: move_file (fp, localpath) except OSError as e: if not os.path.exists (localpath): log.exception ("Problem moving file to%s ", localpath) else: log.error ("Problem moving file, but it seems to be there.. check permissions on store") #log.debug ("local.push: top = %s path= %s",self.top_path, localpath ) ident = localpath[len(self.top_path):] #if ident[0] == '/': # ident = ident[1:] ident = localpath2url(ident) log.info('local push blob_id: %s -> %s', tounicode(ident), tounicode(localpath)) return ident, localpath localpath = "%s-%s%s" % (fpath , uniq[3:7+x] , ext) #localpath = "%s-%04d%s" % (fpath , x , ext) log.warn ("local.write: File exists... trying %s", tounicode(localpath)) raise DuplicateFile(localpath)
def valid(self, storeurl): #log.debug('valid ident %s top %s', ident, self.top) #log.debug('valid local ident %s local top %s', url2localpath(ident), url2localpath(self.top)) if store_compare(storeurl, self.mount_url): return storeurl # It might be a shorted storeurl,_ = split_subpath(storeurl) scheme = urlparse.urlparse(storeurl).scheme if not scheme and storeurl[0] != '/': storeurl = urlparse.urljoin (self.top, storeurl) # OLD STYLE : may have written %encoded values to file system path = posixpath.normpath(urlparse.urlparse(storeurl).path) path = force_filesys (path) log.debug ("checking unquoted %s", tounicode(path)) if os.path.exists (path): return path # not returning an actual URL .. elif storeurl.startswith('file:///'): #should have matched earlier return None elif storeurl.startswith('file://'): storeurl = urlparse.urljoin(self.top, storeurl[7:]) else: return None localpath = url2localpath (storeurl) log.debug ("checking %s", tounicode(localpath)) return os.path.exists(localpath) and localpath2url(localpath)
def _save_store(self, store, storepath, resource, fileobj=None, rooturl=None): 'store the file to the named store' resource_name, sub = split_subpath(resource.get('name')) # Force a move into the store if fileobj: with self._get_driver(store) as driver: if driver.readonly: raise IllegalOperation('readonly store') storeurl = posixpath.join(driver.mount_url, storepath) log.debug('_save_store: %s from %s %s', storeurl, driver.mount_url, storepath) storeurl, localpath = driver.push( fileobj, storeurl, resource.get('resource_uniq')) # Store URL may be changed during driver push by disambiguation code. # revert change from changeset:2297 to always use storeurl #name = os.path.basename(resource_name) or tounicode(url2localpath(os.path.basename(storeurl))) name = tounicode(url2localpath(os.path.basename(storeurl))) resource.set('name', join_subpath(name, sub)) resource.set('value', join_subpath(storeurl, sub)) log.debug('_save_store: %s', etree.tostring(resource)) else: # Try to reference the data in place (if on safe store) storeurl, localpath = self._save_storerefs(store, storepath, resource, rooturl) #Store url may have changed due to conflict if resource.get('value') is None: # This is multifile name = os.path.basename(resource_name) else: name = tounicode(url2localpath(os.path.basename(storeurl))) resource.set('name', join_subpath(name, sub)) # Update the store path reference to similar to the storeurl storepath = storepath.split('/') storepath[-1] = os.path.basename(storeurl) log.debug('_save_store: %s %s %s', storeurl, localpath, etree.tostring(resource)) return storeurl, storepath, localpath
def call_imgcnvlib(command): arr = (ctypes.c_wchar_p * len(command))() arr[:] = [misc.tounicode(i) for i in command] res = ctypes.pointer(ctypes.c_char_p()) try: rw.acquire_write('libimgcnv') r = imgcnvlib.imgcnv(len(command), arr, res) rw.release_write('libimgcnv') except Exception: log.exception('Exception calling libbioimage') return 100, None out = res.contents.value _ = imgcnvlib.imgcnv_clear(res) return r, out
def run_read(cls, ifnm, command): with Locks(ifnm, failonread=(not block_reads)) as l: if l.locked is False: # dima: never wait, respond immediately raise ImageServiceFuture((1, 15)) #command, tmp = misc.start_nounicode_win(ifnm, command) log.debug('run_read dylib command: %s', misc.tounicode(command)) #out = cls.run_command( command ) #misc.end_nounicode_win(tmp) retcode, out = call_imgcnvlib(command) if retcode == 100 or retcode == 101: # some error in libbioimage, retry once log.error('Libioimage retcode %s: retry once: %s', retcode, command) retcode, out = call_imgcnvlib(command) #log.debug('Retcode: %s', retcode) #log.debug('out: %s', out) return out
def run(cls, ifnm, ofnm, args, **kw): '''converts input filename into output using exact arguments as provided in args''' if not cls.installed: return None failonread = kw.get('failonread') or (not block_reads) tmp = None with Locks(ifnm, ofnm, failonexist=True) as l: if l.locked: # the file is not being currently written by another process command = [cls.CONVERTERCOMMAND] command.extend(args) log.debug('Run dylib command: %s', misc.tounicode(command)) proceed = True if ofnm is not None and os.path.exists( ofnm) and os.path.getsize(ofnm) > 16: if kw.get('nooverwrite', False) is True: proceed = False log.warning( 'Run: output exists before command [%s], skipping', misc.tounicode(ofnm)) else: log.warning( 'Run: output exists before command [%s], overwriting', misc.tounicode(ofnm)) if proceed is True: #command, tmp = misc.start_nounicode_win(ifnm, command) retcode, out = call_imgcnvlib(command) #misc.end_nounicode_win(tmp) if retcode == 100 or retcode == 101: # some error in libbioimage, retry once log.error('Libioimage retcode %s: retry once: %s', retcode, command) retcode, out = call_imgcnvlib(command) if retcode == 99: # in case of a timeout log.info('Run: timed-out for [%s]', misc.tounicode(command)) if ofnm is not None and os.path.exists(ofnm): os.remove(ofnm) raise ImageServiceException( 412, 'Requested timeout reached') if retcode != 0: log.info('Run: returned [%s] for [%s]', retcode, misc.tounicode(command)) return None if ofnm is None: return str(retcode) # output file does not exist for some operations, like tiles # tile command does not produce a file with this filename # if not os.path.exists(ofnm): # log.error ('Run: output does not exist after command [%s]', ofnm) # return None elif l.locked is False: # dima: never wait, respond immediately raise ImageServiceFuture((1, 15)) # make sure the write of the output file have finished if ofnm is not None and os.path.exists(ofnm): with Locks(ofnm, failonread=failonread) as l: if l.locked is False: # dima: never wait, respond immediately raise ImageServiceFuture((1, 15)) # safeguard for incorrectly converted files, sometimes only the tiff header can be written # empty lock files are automatically removed before by lock code if os.path.exists( ofnm) and os.path.getsize(ofnm) < cls.MINIMUM_FILE_SIZE: log.error( 'Run: output file is smaller than %s bytes, probably an error, removing [%s]', cls.MINIMUM_FILE_SIZE, ofnm) os.remove(ofnm) return None return ofnm