def decodeMessageData(message): data = None if 'data' not in message: raise RequestError('decodeMessageData: data field not in response') decodedData = b64decode(message['data']) if 'compressed' in message: data = zlib.decompress(decodedData) else: data = decodedData if 'hash' in message: dataHash = hashlib.md5(data).hexdigest() if dataHash != message['hash']: raise RequestError('decodeMessageData: Hash checksum mismatch {} {}'. format(dataHash, message['hash'])) return data
def watchFile(self, filename: str, timeout: int = 5) -> bytes: """Watches for a specific file to be created and returns the file data. InitWatch() must be called first, before watching for specific files. If filename includes the full path, the path must match that used in initWatch(). """ data = None if not self.initWatchSet: raise StateError( "DataInterface: watchFile() called without an initWatch()") # check filename dir matches initWatch dir fileDir, fileCheck = os.path.split(filename) if fileDir not in ('', None): if fileDir != self.watchDir: raise RequestError( "DataInterface: watchFile: filepath doesn't match " f"watch directory: {fileDir}, {self.watchDir}") self._checkAllowedDirs(fileDir) self._checkAllowedFileTypes(fileCheck) self.fileWatchLock.acquire() try: foundFilename = self.fileWatcher.waitForFile( filename, timeout=timeout, timeCheckIncrement=0.2) finally: self.fileWatchLock.release() if foundFilename is None: raise TimeoutError("WatchFile: Timeout {}s: {}".format( timeout, filename)) else: with open(foundFilename, 'rb') as fp: data = fp.read() return data
def clientSendCmd(commPipes, cmd): '''Send a request using named pipes to the projectInterface for handling. This allows a separate client process to make requests of the projectInterface process. It writes the request on fd_out and recieves the reply on fd_in. ''' data = None savedError = None incomplete = True while incomplete: commPipes.fd_out.write(json.dumps(cmd) + os.linesep) msg = commPipes.fd_in.readline() if len(msg) == 0: # fifo closed raise StateError('commPipe closed') response = json.loads(msg) status = response.get('status', -1) if status != 200: raise RequestError( 'clientSendCmd: Cmd: {} status {}: error {}'.format( cmd.get('cmd'), status, response.get('error'))) if 'data' in response: try: data = unpackDataMessage(response) except Exception as err: # The call may be incomplete, save the error and keep receiving as needed logging.error('clientSendCmd: {}'.format(err)) if savedError is None: savedError = err cmd['callId'] = response.get('callId', -1) # Check if need to continue to get more parts incomplete = response.get('incomplete', False) if savedError: raise RequestError('clientSendCmd: {}'.format(savedError)) retVals = StructDict() retVals.statusCode = response.get('status', -1) if 'filename' in response: retVals.filename = response['filename'] if 'fileList' in response: retVals.fileList = response['fileList'] if 'fileTypes' in response: retVals.fileTypes = response['fileTypes'] if data: retVals.data = data if retVals.filename is None: raise StateError('clientSendCmd: filename field is None') return retVals
def handleDataRequest(cmd): savedError = None incomplete = True while incomplete: response = Web.sendDataMsgFromThread(cmd, timeout=60) if response.get('status') != 200: raise RequestError('handleDataRequest: status not 200: {}'.format(response.get('status'))) try: data = unpackDataMessage(response) except Exception as err: logging.error('handleDataRequest: unpackDataMessage: {}'.format(err)) if savedError is None: savedError = err cmd['callId'] = response.get('callId', -1) incomplete = response.get('incomplete', False) if savedError: raise RequestError('handleDataRequest: unpackDataMessage: {}'.format(savedError)) return data
def handleRPCRequest(self, channelName, cmd, timeout=60): """Process RPC requests using websocket RequestHandler to send the request""" """Caller will catch exceptions""" handler = self.handlers[channelName] if handler is None: raise StateError(f'RPC Handler {channelName} not registered') savedError = None incomplete = True # print(f'handle request {cmd}') if cmd.get('cmd') == 'rpc': # if cmd is rpc, check and encode any byte args as base64 cmd = encodeByteTypeArgs(cmd) # Convert numpy arguments to native python types cmd['args'] = npToPy(cmd.get('args', ())) cmd['kwargs'] = npToPy(cmd.get('kwargs', {})) while incomplete: response = handler.doRequest(cmd, timeout) if response.get('status') != 200: errStr = 'handleDataRequest: status {}, err {}'.format( response.get('status'), response.get('error')) self.setError(errStr) raise RequestError(errStr) try: data = unpackDataMessage(response) except Exception as err: errStr = 'handleDataRequest: unpackDataMessage: {}'.format(err) logging.error(errStr) if savedError is None: savedError = errStr incomplete = response.get('incomplete', False) cmd['callId'] = response.get('callId', -1) cmd['incomplete'] = incomplete if savedError: self.setError(savedError) raise RequestError(savedError) serializationType = response.get('dataSerialization') if serializationType == 'json': if type(data) is bytes: data = data.decode() data = json.loads(data) elif serializationType == 'pickle': data = pickle.loads(data) return data
def decodeMessageData(message): """ Given a message encoded with encodeMessageData (above), decode that message. Validate and retrive orignal bytes. Args: message (dict): encoded message to decode Returns: The byte data of the original message from the sender """ data = None if 'data' not in message: raise RequestError('decodeMessageData: data field not in response') decodedData = b64decode(message['data']) if 'compressed' in message: data = zlib.decompress(decodedData) else: data = decodedData if 'hash' in message: dataHash = hashlib.md5(data).hexdigest() if dataHash != message['hash']: raise RequestError( 'decodeMessageData: Hash checksum mismatch {} {}'.format( dataHash, message['hash'])) return data
def listFiles(self, filepattern: str) -> List[str]: """Lists files matching the regex filePattern""" fileDir, fileCheck = os.path.split(filepattern) self._checkAllowedDirs(fileDir) self._checkAllowedFileTypes(fileCheck) if not os.path.isabs(filepattern): errStr = "listFiles must have an absolute path: {}".format( filepattern) raise RequestError(errStr) fileList = [] for filename in glob.iglob(filepattern, recursive=True): if os.path.isdir(filename): continue fileList.append(filename) fileList = self._filterFileList(fileList) return fileList
def __init__(self, dsAccessionNumber, **entities): """ Args: dsAccessionNumber: The OpenNeruo specific accession number for the dataset to stream. entities: BIDS entities (subject, session, task, run, suffix, datatype) that define the particular subject/run of the data to stream """ subject = entities.get('subject') run = entities.get('run') if subject is None or run is None: raise RequestError("OpenNeuroStream: Must specify subject and run number") # TODO - Use OpenNeuroService when it is available, to download # and access the dataset and get dataset entities # OpenNeuroService to provide path to dataset datasetPath = tmpDownloadOpenNeuro(dsAccessionNumber, subject, run) super().__init__(datasetPath, **entities)
def listFiles(self, filePattern): if self.local: if not os.path.isabs(filePattern): errStr = "listFiles must have an absolute path: {}".format(filePattern) raise RequestError(errStr) fileList = [] for filename in glob.iglob(filePattern, recursive=True): if os.path.isdir(filename): continue fileList.append(filename) else: listCmd = projUtils.listFilesReqStruct(filePattern) retVals = projUtils.clientSendCmd(self.commPipes, listCmd) fileList = retVals.get('fileList') if type(fileList) is not list: errStr = "Invalid fileList reponse type {}: expecting list".format(type(fileList)) raise StateError(errStr) return fileList
def _addResultValue(self, runId, trId, value): """Track classification result values, used to plot the results in the web browser.""" # This assume runIds starting at 1 (not zero based) if not isinstance(runId, numbers.Number) or runId <= 0: raise RequestError( f'addResultValue: runId must be number > 0: {runId}') x = trId y = value # Make sure dataPoints has at least as many arrays as runIds for i in range(len(self.dataPoints), runId): self.dataPoints.append([]) if not isinstance(x, numbers.Number): # clear plot for this runId self.dataPoints[runId - 1] = [] return runVals = self.dataPoints[runId - 1] for i, val in enumerate(runVals): if val['x'] == x: runVals[i] = {'x': x, 'y': y} return runVals.append({'x': x, 'y': y})
def runRemoteCall(self, callDict): # print(f'remoteCall {callDict}') className = callDict.get('class') attributeName = callDict.get('attribute') if None in (className, attributeName): raise RequestError(f'Malformed remote call struct: missing one of ' f'class {className}, attribute {attributeName}') classInstance = self.classInstanceDict.get(className) if classInstance is None: raise StateError( f'RemoteHandler: class {className} not registered') attributeInstance = getattr(classInstance, attributeName) if not callable(attributeInstance): return attributeInstance args = callDict.get('args', ()) if args is None: # Can happen if key 'args' exists and is set to None args = () kwargs = callDict.get('kwargs', {}) if kwargs is None: kwargs = {} res = attributeInstance(*args, **kwargs) return res
def decodeByteTypeArgs(cmd) -> dict: """ Decodes rpc args that were previously encoded with encodeByteTypeArgs. Args: cmd: a dictionary with encoded args Returns: cmd: a dictionary with decoded args """ byteArgIndices = cmd.get('encodedByteArgs') if byteArgIndices is not None: args = cmd.get('args', ()) args = list(args) for i in byteArgIndices: tag = 'encodedBytes_' + str(i) encdata = cmd.get(tag) if encdata is None or args[i] != tag: raise RequestError( f'Byte encoded data error: index {i} tag {tag}') decodedData = b64decode(encdata) args[i] = decodedData cmd.pop(tag, None) cmd.pop('encodedByteArgs') cmd['args'] = tuple(args) return cmd
def unpackDataMessage(msg): global multiPartDataCache try: if msg.get('status') != 200: # On error delete any partial transfers fileHash = msg.get('fileHash') if fileHash is not None and fileHash in multiPartDataCache: del multiPartDataCache[fileHash] raise RequestError('unpackDataMessage: {} {}'.format( msg.get('status'), msg.get('error'))) data = decodeMessageData(msg) multipart = msg.get('multipart', False) numParts = msg.get('numParts', 1) partId = msg.get('partId', 1) logging.debug('unpackDataMessage: callid {}, part {} of {}'.format( msg.get('callId'), partId, numParts)) if multipart is False or numParts == 1: # All data sent in a single message return data else: assert numParts > 1 assert multipart is True if partId > numParts: raise RequestError( 'unpackDataMessage: Inconsistent parts: partId {} exceeds numParts {}' .format(partId, numParts)) # get the data structure for this data fileHash = msg.get('fileHash') if partId > 1: partialDataStruct = multiPartDataCache.get(fileHash) if partialDataStruct is None: raise RequestError( 'unpackDataMessage: partialDataStruct not found') else: partialDataStruct = StructDict({ 'cachedDataParts': [None] * numParts, 'numCachedParts': 0 }) multiPartDataCache[fileHash] = partialDataStruct partialDataStruct.cachedDataParts[partId - 1] = data partialDataStruct.numCachedParts += 1 if partialDataStruct.numCachedParts == numParts: # All parts of the multipart transfer have been received # Concatenate the data into one bytearray data = bytearray() for i in range(numParts): dataPart = partialDataStruct.cachedDataParts[i] if dataPart is None: raise StateError( 'unpackDataMessage: missing dataPart {}'.format(i)) data.extend(dataPart) # Check fileHash and fileSize dataHash = hashlib.md5(data).hexdigest() dataSize = len(data) if dataHash != fileHash: raise RequestError( "unpackDataMessage: File checksum mismatch {} {}". format(dataHash, fileHash)) if dataSize != msg.get('fileSize', 0): raise RequestError( "unpackDataMessage: File size mismatch {} {}".format( dataSize, msg.get('fileSize', 0))) # delete the multipart data cache for this item del multiPartDataCache[fileHash] return data # Multi-part transfer not complete, nothing to return return None except Exception as err: # removed any cached data fileHash = msg.get('fileHash') if fileHash and fileHash in multiPartDataCache: del multiPartDataCache[fileHash] raise err
def initialize(cfg, args): """ purpose: load information and add to config """ if cfg.sessionId in (None, '') or cfg.useSessionTimestamp is True: cfg.useSessionTimestamp = True cfg.sessionId = utils.dateStr30(time.localtime()) else: cfg.useSessionTimestamp = False # MERGE WITH PARAMS if args.runs != '' and args.scans != '': # use the run and scan numbers passed in as parameters cfg.runNum = [int(x) for x in args.runs.split(',')] cfg.scanNum = [int(x) for x in args.scans.split(',')] else: # when you're not specifying on the command line it's already in a list cfg.runNum = [int(x) for x in cfg.runNum] cfg.scanNum = [int(x) for x in cfg.scanNum] # GET DICOM DIRECTORY if cfg.buildImgPath: imgDirDate = datetime.now() dateStr = cfg.date.lower() if dateStr != 'now' and dateStr != 'today': try: imgDirDate = parser.parse(cfg.date) except ValueError as err: raise RequestError('Unable to parse date string {} {}'.format( cfg.date, err)) datestr = imgDirDate.strftime("%Y%m%d") imgDirName = "{}.{}.{}".format(datestr, cfg.subjectName, cfg.subjectName) cfg.dicomDir = os.path.join(cfg.local.dicomDir, imgDirName) else: cfg.dicomDir = cfg.local.dicomDir # then the whole path was supplied ######## cfg.bids_id = 'sub-{0:03d}'.format(cfg.subjectNum) cfg.ses_id = 'ses-{0:02d}'.format(cfg.subjectDay) # specify local directories cfg.local.codeDir = os.path.join(cfg.local.rtcloudDir, 'projects', cfg.projectName) cfg.local.dataDir = os.path.join(cfg.local.codeDir, 'data') cfg.local.subject_full_day_path = os.path.join(cfg.local.dataDir, cfg.bids_id, cfg.ses_id) cfg.local.subject_reg_dir = os.path.join(cfg.local.subject_full_day_path, 'registration_outputs') cfg.local.wf_dir = os.path.join(cfg.local.dataDir, cfg.bids_id, 'ses-01', 'registration') cfg.local.maskDir = os.path.join(cfg.local.codeDir, 'ROI') cfg.subject_reg_dir = cfg.local.subject_reg_dir cfg.wf_dir = cfg.local.wf_dir cfg.n_masks = len(cfg.MASK) # Copy mask files to registration directory if not os.path.exists(cfg.local.subject_reg_dir): os.makedirs(cfg.local.subject_reg_dir) if not os.path.exists(cfg.local.wf_dir): os.makedirs(cfg.local.wf_dir) os.system(f'cp {cfg.local.maskDir}/* {cfg.local.subject_reg_dir}') if args.dataRemote: # here we will need to specify separate paths for processing cfg.server.codeDir = os.path.join(cfg.server.rtcloudDir, 'projects', cfg.projectName) cfg.server.dataDir = os.path.join(cfg.server.codeDir, cfg.server.serverDataDir) cfg.server.subject_full_day_path = os.path.join( cfg.server.dataDir, cfg.bids_id, cfg.ses_id) cfg.server.subject_reg_dir = os.path.join( cfg.server.subject_full_day_path, 'registration_outputs') cfg.server.wf_dir = os.path.join(cfg.server.dataDir, cfg.bids_id, 'ses-01', 'registration') cfg.server.maskDir = os.path.join(cfg.server.codeDir, 'ROI') cfg.subject_reg_dir = cfg.server.subject_reg_dir cfg.wf_dir = cfg.server.wf_dir cfg.ref_BOLD = os.path.join(cfg.wf_dir, 'ref_image.nii.gz') cfg.MNI_ref_filename = os.path.join(cfg.wf_dir, cfg.MNI_ref_BOLD) cfg.T1_to_BOLD = os.path.join(cfg.wf_dir, 'affine.txt') cfg.MNI_to_T1 = os.path.join(cfg.wf_dir, 'ants_t1_to_mniInverseComposite.h5') cfg.MASK_transformed = [''] * cfg.n_masks cfg.local_MASK_transformed = [''] * cfg.n_masks for m in np.arange(cfg.n_masks): mask_name = cfg.MASK[m].split('.')[0] + '_space-native.nii.gz' cfg.MASK_transformed[m] = os.path.join(cfg.subject_reg_dir, mask_name) cfg.local_MASK_transformed[m] = os.path.join(cfg.local.subject_reg_dir, mask_name) # get conversion to flip dicom to nifti files cfg.axesTransform = getTransform(('L', 'A', 'S'), ('P', 'L', 'S')) return cfg