def decodeMessageData(message):
data = None
if 'data' not in message:
raise RequestError('decodeMessageData: data field not in response')
decodedData = b64decode(message['data'])
if 'compressed' in message:
data = zlib.decompress(decodedData)
else:
data = decodedData
if 'hash' in message:
dataHash = hashlib.md5(data).hexdigest()
if dataHash != message['hash']:
raise RequestError('decodeMessageData: Hash checksum mismatch {} {}'.
format(dataHash, message['hash']))
return data
def watchFile(self, filename: str, timeout: int = 5) -> bytes:
"""Watches for a specific file to be created and returns the file data.
InitWatch() must be called first, before watching for specific files.
If filename includes the full path, the path must match that used in initWatch().
"""
data = None
if not self.initWatchSet:
raise StateError(
"DataInterface: watchFile() called without an initWatch()")
# check filename dir matches initWatch dir
fileDir, fileCheck = os.path.split(filename)
if fileDir not in ('', None):
if fileDir != self.watchDir:
raise RequestError(
"DataInterface: watchFile: filepath doesn't match "
f"watch directory: {fileDir}, {self.watchDir}")
self._checkAllowedDirs(fileDir)
self._checkAllowedFileTypes(fileCheck)
self.fileWatchLock.acquire()
try:
foundFilename = self.fileWatcher.waitForFile(
filename, timeout=timeout, timeCheckIncrement=0.2)
finally:
self.fileWatchLock.release()
if foundFilename is None:
raise TimeoutError("WatchFile: Timeout {}s: {}".format(
timeout, filename))
else:
with open(foundFilename, 'rb') as fp:
data = fp.read()
return data
def clientSendCmd(commPipes, cmd):
'''Send a request using named pipes to the projectInterface for handling.
This allows a separate client process to make requests of the projectInterface process.
It writes the request on fd_out and recieves the reply on fd_in.
'''
data = None
savedError = None
incomplete = True
while incomplete:
commPipes.fd_out.write(json.dumps(cmd) + os.linesep)
msg = commPipes.fd_in.readline()
if len(msg) == 0:
# fifo closed
raise StateError('commPipe closed')
response = json.loads(msg)
status = response.get('status', -1)
if status != 200:
raise RequestError(
'clientSendCmd: Cmd: {} status {}: error {}'.format(
cmd.get('cmd'), status, response.get('error')))
if 'data' in response:
try:
data = unpackDataMessage(response)
except Exception as err:
# The call may be incomplete, save the error and keep receiving as needed
logging.error('clientSendCmd: {}'.format(err))
if savedError is None:
savedError = err
cmd['callId'] = response.get('callId', -1)
# Check if need to continue to get more parts
incomplete = response.get('incomplete', False)
if savedError:
raise RequestError('clientSendCmd: {}'.format(savedError))
retVals = StructDict()
retVals.statusCode = response.get('status', -1)
if 'filename' in response:
retVals.filename = response['filename']
if 'fileList' in response:
retVals.fileList = response['fileList']
if 'fileTypes' in response:
retVals.fileTypes = response['fileTypes']
if data:
retVals.data = data
if retVals.filename is None:
raise StateError('clientSendCmd: filename field is None')
return retVals
def handleDataRequest(cmd):
savedError = None
incomplete = True
while incomplete:
response = Web.sendDataMsgFromThread(cmd, timeout=60)
if response.get('status') != 200:
raise RequestError('handleDataRequest: status not 200: {}'.format(response.get('status')))
try:
data = unpackDataMessage(response)
except Exception as err:
logging.error('handleDataRequest: unpackDataMessage: {}'.format(err))
if savedError is None:
savedError = err
cmd['callId'] = response.get('callId', -1)
incomplete = response.get('incomplete', False)
if savedError:
raise RequestError('handleDataRequest: unpackDataMessage: {}'.format(savedError))
return data
def handleRPCRequest(self, channelName, cmd, timeout=60):
"""Process RPC requests using websocket RequestHandler to send the request"""
"""Caller will catch exceptions"""
handler = self.handlers[channelName]
if handler is None:
raise StateError(f'RPC Handler {channelName} not registered')
savedError = None
incomplete = True
# print(f'handle request {cmd}')
if cmd.get('cmd') == 'rpc':
# if cmd is rpc, check and encode any byte args as base64
cmd = encodeByteTypeArgs(cmd)
# Convert numpy arguments to native python types
cmd['args'] = npToPy(cmd.get('args', ()))
cmd['kwargs'] = npToPy(cmd.get('kwargs', {}))
while incomplete:
response = handler.doRequest(cmd, timeout)
if response.get('status') != 200:
errStr = 'handleDataRequest: status {}, err {}'.format(
response.get('status'), response.get('error'))
self.setError(errStr)
raise RequestError(errStr)
try:
data = unpackDataMessage(response)
except Exception as err:
errStr = 'handleDataRequest: unpackDataMessage: {}'.format(err)
logging.error(errStr)
if savedError is None:
savedError = errStr
incomplete = response.get('incomplete', False)
cmd['callId'] = response.get('callId', -1)
cmd['incomplete'] = incomplete
if savedError:
self.setError(savedError)
raise RequestError(savedError)
serializationType = response.get('dataSerialization')
if serializationType == 'json':
if type(data) is bytes:
data = data.decode()
data = json.loads(data)
elif serializationType == 'pickle':
data = pickle.loads(data)
return data
def decodeMessageData(message):
"""
Given a message encoded with encodeMessageData (above), decode that message.
Validate and retrive orignal bytes.
Args:
message (dict): encoded message to decode
Returns:
The byte data of the original message from the sender
"""
data = None
if 'data' not in message:
raise RequestError('decodeMessageData: data field not in response')
decodedData = b64decode(message['data'])
if 'compressed' in message:
data = zlib.decompress(decodedData)
else:
data = decodedData
if 'hash' in message:
dataHash = hashlib.md5(data).hexdigest()
if dataHash != message['hash']:
raise RequestError(
'decodeMessageData: Hash checksum mismatch {} {}'.format(
dataHash, message['hash']))
return data
def listFiles(self, filepattern: str) -> List[str]:
"""Lists files matching the regex filePattern"""
fileDir, fileCheck = os.path.split(filepattern)
self._checkAllowedDirs(fileDir)
self._checkAllowedFileTypes(fileCheck)
if not os.path.isabs(filepattern):
errStr = "listFiles must have an absolute path: {}".format(
filepattern)
raise RequestError(errStr)
fileList = []
for filename in glob.iglob(filepattern, recursive=True):
if os.path.isdir(filename):
continue
fileList.append(filename)
fileList = self._filterFileList(fileList)
return fileList
def __init__(self, dsAccessionNumber, **entities):
"""
Args:
dsAccessionNumber: The OpenNeruo specific accession number for the dataset
to stream.
entities: BIDS entities (subject, session, task, run, suffix, datatype) that
define the particular subject/run of the data to stream
"""
subject = entities.get('subject')
run = entities.get('run')
if subject is None or run is None:
raise RequestError("OpenNeuroStream: Must specify subject and run number")
# TODO - Use OpenNeuroService when it is available, to download
# and access the dataset and get dataset entities
# OpenNeuroService to provide path to dataset
datasetPath = tmpDownloadOpenNeuro(dsAccessionNumber, subject, run)
super().__init__(datasetPath, **entities)
def listFiles(self, filePattern):
if self.local:
if not os.path.isabs(filePattern):
errStr = "listFiles must have an absolute path: {}".format(filePattern)
raise RequestError(errStr)
fileList = []
for filename in glob.iglob(filePattern, recursive=True):
if os.path.isdir(filename):
continue
fileList.append(filename)
else:
listCmd = projUtils.listFilesReqStruct(filePattern)
retVals = projUtils.clientSendCmd(self.commPipes, listCmd)
fileList = retVals.get('fileList')
if type(fileList) is not list:
errStr = "Invalid fileList reponse type {}: expecting list".format(type(fileList))
raise StateError(errStr)
return fileList
def _addResultValue(self, runId, trId, value):
"""Track classification result values, used to plot the results in the web browser."""
# This assume runIds starting at 1 (not zero based)
if not isinstance(runId, numbers.Number) or runId <= 0:
raise RequestError(
f'addResultValue: runId must be number > 0: {runId}')
x = trId
y = value
# Make sure dataPoints has at least as many arrays as runIds
for i in range(len(self.dataPoints), runId):
self.dataPoints.append([])
if not isinstance(x, numbers.Number):
# clear plot for this runId
self.dataPoints[runId - 1] = []
return
runVals = self.dataPoints[runId - 1]
for i, val in enumerate(runVals):
if val['x'] == x:
runVals[i] = {'x': x, 'y': y}
return
runVals.append({'x': x, 'y': y})
def runRemoteCall(self, callDict):
# print(f'remoteCall {callDict}')
className = callDict.get('class')
attributeName = callDict.get('attribute')
if None in (className, attributeName):
raise RequestError(f'Malformed remote call struct: missing one of '
f'class {className}, attribute {attributeName}')
classInstance = self.classInstanceDict.get(className)
if classInstance is None:
raise StateError(
f'RemoteHandler: class {className} not registered')
attributeInstance = getattr(classInstance, attributeName)
if not callable(attributeInstance):
return attributeInstance
args = callDict.get('args', ())
if args is None: # Can happen if key 'args' exists and is set to None
args = ()
kwargs = callDict.get('kwargs', {})
if kwargs is None:
kwargs = {}
res = attributeInstance(*args, **kwargs)
return res
def decodeByteTypeArgs(cmd) -> dict:
"""
Decodes rpc args that were previously encoded with encodeByteTypeArgs.
Args:
cmd: a dictionary with encoded args
Returns:
cmd: a dictionary with decoded args
"""
byteArgIndices = cmd.get('encodedByteArgs')
if byteArgIndices is not None:
args = cmd.get('args', ())
args = list(args)
for i in byteArgIndices:
tag = 'encodedBytes_' + str(i)
encdata = cmd.get(tag)
if encdata is None or args[i] != tag:
raise RequestError(
f'Byte encoded data error: index {i} tag {tag}')
decodedData = b64decode(encdata)
args[i] = decodedData
cmd.pop(tag, None)
cmd.pop('encodedByteArgs')
cmd['args'] = tuple(args)
return cmd
def unpackDataMessage(msg):
global multiPartDataCache
try:
if msg.get('status') != 200:
# On error delete any partial transfers
fileHash = msg.get('fileHash')
if fileHash is not None and fileHash in multiPartDataCache:
del multiPartDataCache[fileHash]
raise RequestError('unpackDataMessage: {} {}'.format(
msg.get('status'), msg.get('error')))
data = decodeMessageData(msg)
multipart = msg.get('multipart', False)
numParts = msg.get('numParts', 1)
partId = msg.get('partId', 1)
logging.debug('unpackDataMessage: callid {}, part {} of {}'.format(
msg.get('callId'), partId, numParts))
if multipart is False or numParts == 1:
# All data sent in a single message
return data
else:
assert numParts > 1
assert multipart is True
if partId > numParts:
raise RequestError(
'unpackDataMessage: Inconsistent parts: partId {} exceeds numParts {}'
.format(partId, numParts))
# get the data structure for this data
fileHash = msg.get('fileHash')
if partId > 1:
partialDataStruct = multiPartDataCache.get(fileHash)
if partialDataStruct is None:
raise RequestError(
'unpackDataMessage: partialDataStruct not found')
else:
partialDataStruct = StructDict({
'cachedDataParts': [None] * numParts,
'numCachedParts': 0
})
multiPartDataCache[fileHash] = partialDataStruct
partialDataStruct.cachedDataParts[partId - 1] = data
partialDataStruct.numCachedParts += 1
if partialDataStruct.numCachedParts == numParts:
# All parts of the multipart transfer have been received
# Concatenate the data into one bytearray
data = bytearray()
for i in range(numParts):
dataPart = partialDataStruct.cachedDataParts[i]
if dataPart is None:
raise StateError(
'unpackDataMessage: missing dataPart {}'.format(i))
data.extend(dataPart)
# Check fileHash and fileSize
dataHash = hashlib.md5(data).hexdigest()
dataSize = len(data)
if dataHash != fileHash:
raise RequestError(
"unpackDataMessage: File checksum mismatch {} {}".
format(dataHash, fileHash))
if dataSize != msg.get('fileSize', 0):
raise RequestError(
"unpackDataMessage: File size mismatch {} {}".format(
dataSize, msg.get('fileSize', 0)))
# delete the multipart data cache for this item
del multiPartDataCache[fileHash]
return data
# Multi-part transfer not complete, nothing to return
return None
except Exception as err:
# removed any cached data
fileHash = msg.get('fileHash')
if fileHash and fileHash in multiPartDataCache:
del multiPartDataCache[fileHash]
raise err
def initialize(cfg, args):
""" purpose: load information and add to config """
if cfg.sessionId in (None, '') or cfg.useSessionTimestamp is True:
cfg.useSessionTimestamp = True
cfg.sessionId = utils.dateStr30(time.localtime())
else:
cfg.useSessionTimestamp = False
# MERGE WITH PARAMS
if args.runs != '' and args.scans != '':
# use the run and scan numbers passed in as parameters
cfg.runNum = [int(x) for x in args.runs.split(',')]
cfg.scanNum = [int(x) for x in args.scans.split(',')]
else: # when you're not specifying on the command line it's already in a list
cfg.runNum = [int(x) for x in cfg.runNum]
cfg.scanNum = [int(x) for x in cfg.scanNum]
# GET DICOM DIRECTORY
if cfg.buildImgPath:
imgDirDate = datetime.now()
dateStr = cfg.date.lower()
if dateStr != 'now' and dateStr != 'today':
try:
imgDirDate = parser.parse(cfg.date)
except ValueError as err:
raise RequestError('Unable to parse date string {} {}'.format(
cfg.date, err))
datestr = imgDirDate.strftime("%Y%m%d")
imgDirName = "{}.{}.{}".format(datestr, cfg.subjectName,
cfg.subjectName)
cfg.dicomDir = os.path.join(cfg.local.dicomDir, imgDirName)
else:
cfg.dicomDir = cfg.local.dicomDir # then the whole path was supplied
########
cfg.bids_id = 'sub-{0:03d}'.format(cfg.subjectNum)
cfg.ses_id = 'ses-{0:02d}'.format(cfg.subjectDay)
# specify local directories
cfg.local.codeDir = os.path.join(cfg.local.rtcloudDir, 'projects',
cfg.projectName)
cfg.local.dataDir = os.path.join(cfg.local.codeDir, 'data')
cfg.local.subject_full_day_path = os.path.join(cfg.local.dataDir,
cfg.bids_id, cfg.ses_id)
cfg.local.subject_reg_dir = os.path.join(cfg.local.subject_full_day_path,
'registration_outputs')
cfg.local.wf_dir = os.path.join(cfg.local.dataDir, cfg.bids_id, 'ses-01',
'registration')
cfg.local.maskDir = os.path.join(cfg.local.codeDir, 'ROI')
cfg.subject_reg_dir = cfg.local.subject_reg_dir
cfg.wf_dir = cfg.local.wf_dir
cfg.n_masks = len(cfg.MASK)
# Copy mask files to registration directory
if not os.path.exists(cfg.local.subject_reg_dir):
os.makedirs(cfg.local.subject_reg_dir)
if not os.path.exists(cfg.local.wf_dir):
os.makedirs(cfg.local.wf_dir)
os.system(f'cp {cfg.local.maskDir}/* {cfg.local.subject_reg_dir}')
if args.dataRemote: # here we will need to specify separate paths for processing
cfg.server.codeDir = os.path.join(cfg.server.rtcloudDir, 'projects',
cfg.projectName)
cfg.server.dataDir = os.path.join(cfg.server.codeDir,
cfg.server.serverDataDir)
cfg.server.subject_full_day_path = os.path.join(
cfg.server.dataDir, cfg.bids_id, cfg.ses_id)
cfg.server.subject_reg_dir = os.path.join(
cfg.server.subject_full_day_path, 'registration_outputs')
cfg.server.wf_dir = os.path.join(cfg.server.dataDir, cfg.bids_id,
'ses-01', 'registration')
cfg.server.maskDir = os.path.join(cfg.server.codeDir, 'ROI')
cfg.subject_reg_dir = cfg.server.subject_reg_dir
cfg.wf_dir = cfg.server.wf_dir
cfg.ref_BOLD = os.path.join(cfg.wf_dir, 'ref_image.nii.gz')
cfg.MNI_ref_filename = os.path.join(cfg.wf_dir, cfg.MNI_ref_BOLD)
cfg.T1_to_BOLD = os.path.join(cfg.wf_dir, 'affine.txt')
cfg.MNI_to_T1 = os.path.join(cfg.wf_dir,
'ants_t1_to_mniInverseComposite.h5')
cfg.MASK_transformed = [''] * cfg.n_masks
cfg.local_MASK_transformed = [''] * cfg.n_masks
for m in np.arange(cfg.n_masks):
mask_name = cfg.MASK[m].split('.')[0] + '_space-native.nii.gz'
cfg.MASK_transformed[m] = os.path.join(cfg.subject_reg_dir, mask_name)
cfg.local_MASK_transformed[m] = os.path.join(cfg.local.subject_reg_dir,
mask_name)
# get conversion to flip dicom to nifti files
cfg.axesTransform = getTransform(('L', 'A', 'S'), ('P', 'L', 'S'))
return cfg