def testIsEmpty(tmpdir, bidsArchive4D):
datasetRoot = Path(tmpdir, "bids-archive")
archive = BidsArchive(datasetRoot)
assert archive is not None
assert archive.isEmpty()
assert not bidsArchive4D.isEmpty()
def testGetBidsRunInheritedEvents(tmpdir, validBidsI, sampleBidsEntities):
# Add an events file on the top level with data
rootPath = os.path.join(tmpdir, 'dataset')
incrementalDFDict = {col: [4] for col in DEFAULT_EVENTS_HEADERS}
incrementalDF = pd.DataFrame.from_dict(incrementalDFDict)
validBidsI.events = correctEventsFileDatatypes(incrementalDF)
validBidsI.writeToDisk(rootPath)
archive = BidsArchive(rootPath)
newDFDict = {col: [1, 2, 3] for col in DEFAULT_EVENTS_HEADERS}
newDF = pd.DataFrame.from_dict(newDFDict)
newDF = correctEventsFileDatatypes(newDF)
writeDataFrameToEvents(
newDF, '{dirPath}/task-{taskName}_events.tsv'.format(
dirPath=rootPath, taskName=sampleBidsEntities['task']))
archive._updateLayout()
# Get the BIDS run
run = archive.getBidsRun(**sampleBidsEntities)
# Ensure that the BIDS run has all of the data added at the top level
combinedDF = newDF.append(incrementalDF)
combinedDF.sort_values(by='onset', inplace=True, ignore_index=True)
assert combinedDF.equals(run._events)
def testDiskOutput(validBidsI, tmpdir):
# Write the archive
datasetRoot = os.path.join(tmpdir, "bids-pytest-dataset")
validBidsI.writeToDisk(datasetRoot)
# Validate the output can be opened by BidsArchive and verified against the
# source BIDS-Incremental
archive = BidsArchive(datasetRoot)
archiveImage = archive.getImages()[0]
# Remove pseudo entities to avoid conflict with the validBidsI
metadata = archive.getSidecarMetadata(archiveImage, includeEntities=True)
for entity in PYBIDS_PSEUDO_ENTITIES:
metadata.pop(entity)
incrementalFromArchive = BidsIncremental(archiveImage, metadata)
assert incrementalFromArchive == validBidsI
assert isValidBidsArchive(archive.rootPath)
# Try only writing data
datasetRoot = os.path.join(tmpdir, "bids-pytest-dataset-2")
validBidsI.writeToDisk(datasetRoot, onlyData=True)
assert not os.path.exists(os.path.join(datasetRoot, "README"))
assert not os.path.exists(
os.path.join(datasetRoot, "dataset_description.json"))
def testEmptyArchiveAppend(validBidsI, imageMetadata, tmpdir):
# Create in root with no BIDS-I, then append to make non-empty archive
datasetRoot = Path(tmpdir, testEmptyArchiveAppend.__name__)
archive = BidsArchive(datasetRoot)
archive.appendIncremental(validBidsI)
assert not archive.isEmpty()
assert archiveHasMetadata(archive, imageMetadata)
assert appendDataMatches(archive, validBidsI)
assert isValidBidsArchive(datasetRoot)
def __init__(self, archivePath, **entities):
"""
Args:
archivePath: Absolute path of the BIDS archive.
entities: BIDS entities (subject, session, task, run, suffix, datatype) that
define the particular subject/run of the data to stream
"""
self.bidsArchive = BidsArchive(archivePath)
self.bidsRun = self.bidsArchive.getBidsRun(**entities)
self.numVolumes = self.bidsRun.numIncrementals()
self.nextVol = 0
def testGetEvents(validBidsI, imageMetadata, tmpdir):
archive = BidsArchive(tmpdir)
archive.appendIncremental(validBidsI)
# Get the events from the archive as a pandas data frame
events = archive.getEvents()[0].get_df()
assert events is not None
# Check the required columns are present in the events file data frame
for column in ['onset', 'duration', 'response_time']:
assert column in events.columns
def testMetadataValidation(imageMetadata, caplog):
metadataCopy = imageMetadata.copy()
# Test failure on sample of fields that must be the same
matchFields = ["Modality", "MagneticFieldStrength", "ImagingFrequency",
"Manufacturer", "ManufacturersModelName", "InstitutionName",
"InstitutionAddress", "DeviceSerialNumber", "StationName",
"BodyPartExamined", "PatientPosition", "EchoTime",
"ProcedureStepDescription", "SoftwareVersions",
"MRAcquisitionType", "SeriesDescription", "ProtocolName",
"ScanningSequence", "SequenceVariant", "ScanOptions",
"SequenceName", "SpacingBetweenSlices", "SliceThickness",
"ImageType", "RepetitionTime", "PhaseEncodingDirection",
"FlipAngle", "InPlanePhaseEncodingDirectionDICOM",
"ImageOrientationPatientDICOM", "PartialFourier"]
for field in matchFields:
oldValue = metadataCopy.get(field, None)
# If field not present, append should work
if oldValue is None:
assert BidsArchive._metadataAppendCompatible(imageMetadata,
metadataCopy)
# If field is present, modify and ensure failure
else:
metadataCopy[field] = "not a valid value by any stretch of the word"
assert metadataCopy[field] != oldValue
compatible, errorMsg = BidsArchive._metadataAppendCompatible(
imageMetadata,
metadataCopy)
assert not compatible
assert f"Metadata doesn't match on field: {field}" in errorMsg
metadataCopy[field] = oldValue
# Test append-compatible when only one side has a particular metadata value
for field in matchFields:
for metadataDict in [imageMetadata, metadataCopy]:
oldValue = metadataDict.pop(field, None)
if oldValue is None:
continue
compatible, errorMsg = BidsArchive._metadataAppendCompatible(
imageMetadata,
metadataCopy)
assert compatible
metadataDict[field] = oldValue
def testGetEvents(validBidsI, imageMetadata, tmpdir):
archive = BidsArchive(tmpdir)
archive._appendIncremental(validBidsI)
# Get the events from the archive as a pandas data frame
events = archive.getEvents()[0].get_df()
events = correctEventsFileDatatypes(events)
assert events is not None
# Check the required columns are present in the events file data frame
for column in DEFAULT_EVENTS_HEADERS:
assert column in events.columns
# Check the columns are of the proper types
for column, dtype in BIDS_EVENT_COL_TO_DTYPE.items():
if column in events.columns:
assert events[column].dtype == dtype
def doRuns(cfg, bidsInterface, subjInterface, webInterface):
"""
Process a run of a bids dataset. The subject and run configuration
values will be supplied by the cfg parameter.
Args:
cfg: configurations parsed from the project toml config file
bidsInterface: client interface to bids data
webInterface: client interface to user web page
Returns: no return value
"""
subject = cfg.subjectName
run = cfg.runNum[0]
entities = {
'subject': subject,
'run': run,
'suffix': 'bold',
'datatype': 'func'
}
webInterface.clearRunPlot(run)
if cfg.writeBidsArchive is True:
# Create a new bids archive from the incrementals
bidsArchivePath = os.path.join(tmpDir,
'bids_archive_' + uuid.uuid4().hex)
print(f'BIDS Archive will be written to {bidsArchivePath}')
newArchive = BidsArchive(bidsArchivePath)
newRun = BidsRun(**entities)
# Initialize the bids stream
streamId = bidsInterface.initOpenNeuroStream(cfg.dsAccessionNumber,
**entities)
numVols = bidsInterface.getNumVolumes(streamId)
for idx in range(numVols):
bidsIncremental = bidsInterface.getIncremental(streamId, idx)
if cfg.writeBidsArchive is True:
newRun.appendIncremental(bidsIncremental)
imageData = bidsIncremental.getImageData()
avg_niftiData = numpy.mean(imageData)
print("| average activation value for TR %d is %f" %
(idx, avg_niftiData))
webInterface.plotDataPoint(run, idx, float(avg_niftiData))
if cfg.writeBidsArchive is True:
newArchive.appendBidsRun(newRun)
def testAppendNoMakePath(bidsArchive4D, validBidsI, tmpdir):
# Append to empty archive specifying not to make any files or directories
datasetRoot = Path(tmpdir, testEmptyArchiveAppend.__name__)
assert not BidsArchive(datasetRoot).appendIncremental(validBidsI,
makePath=False)
# Append to populated archive in a way that would require new directories
# and files without allowing it
validBidsI.setMetadataField('subject', 'invalidSubject')
validBidsI.setMetadataField('run', 42)
assert not bidsArchive4D.appendIncremental(validBidsI, makePath=False)
def testAppendBidsRun(tmpdir, bidsArchive4D, bidsArchiveMultipleRuns,
sampleBidsEntities):
archivePath = Path(tmpdir, "appendBidsRunArchive")
archive = BidsArchive(archivePath)
emptyRun = BidsRun()
archive.appendBidsRun(emptyRun)
run = bidsArchive4D.getBidsRun(**sampleBidsEntities)
archive.appendBidsRun(run)
assert archive.getBidsRun(**sampleBidsEntities) == run
def __init__(self, archivePath, **entities):
"""
Args:
archivePath: Absolute path of the BIDS archive.
entities: BIDS entities (subject, session, task, run, suffix, datatype) that
define the particular subject/run of the data to stream
"""
self.bidsArchive = BidsArchive(archivePath)
# TODO - when we have BidsRun
# self.bidsRun = self.bidsArchive.getBidsRun(**entities)
images = self.bidsArchive.getImages(**entities)
if len(images) == 0:
raise ValidationError('No matching images found')
if len(images) > 1:
raise ValidationError('More than one match, please give more specific subject/session/task/run')
self.bidsImage = images[0]
self.niftiImage = self.bidsImage.get_image()
self.filename = self.niftiImage.get_filename()
self.imgVolumes = nib.four_to_three(self.niftiImage)
self.metadata = self.bidsArchive.getSidecarMetadata(self.filename, includeEntities=True)
self.metadata.pop('extension')
self.numVolumes = len(self.imgVolumes)
self.nextVol = 0
class BidsStream:
"""
A class that opens a BIDS archive and prepares to stream the data as
BIDS incrementals.
"""
def __init__(self, archivePath, **entities):
"""
Args:
archivePath: Absolute path of the BIDS archive.
entities: BIDS entities (subject, session, task, run, suffix, datatype) that
define the particular subject/run of the data to stream
"""
self.bidsArchive = BidsArchive(archivePath)
self.bidsRun = self.bidsArchive.getBidsRun(**entities)
self.numVolumes = self.bidsRun.numIncrementals()
self.nextVol = 0
def getNumVolumes(self) -> int:
"""Return the number of brain volumes in the run"""
return self.numVolumes
def getIncremental(self, volIdx=-1) -> BidsIncremental:
"""
Get a BIDS incremental for the indicated index in the current subject/run
VolIdx acts similar to a file_seek pointer. If a volIdx >= 0 is supplied
the volume pointer is advanced to that position. If no volIdx or
a volIdx < 0 is supplied, then the next image volume after the previous
position is returned and the pointer is incremented.
Args:
volIdx: The volume index (or TR) within the run to retrieve.
Returns:
BidsIncremental of that volume index within this subject/run
"""
if volIdx >= 0:
# reset the next volume to the user specified volume
self.nextVol = volIdx
else:
# use the default next volume
pass
if self.nextVol < self.numVolumes:
incremental = self.bidsRun.getIncremental(self.nextVol)
self.nextVol += 1
return incremental
else:
return None
def appendDataMatches(archive: BidsArchive, reference: BidsIncremental,
startIndex: int = 0, endIndex: int = -1):
entities = filterEntities(reference.imageMetadata)
images = archive.getImages(**entities)
assert len(images) == 1
imageFromArchive = images[0].get_image()
fullImageData = getNiftiData(imageFromArchive)
if endIndex == -1:
endIndex = len(fullImageData)
appendedData = fullImageData[..., startIndex:endIndex]
appendedImage = nib.Nifti1Image(appendedData,
imageFromArchive.affine,
imageFromArchive.header)
return BidsIncremental(appendedImage, reference.imageMetadata) == reference
def archiveWithImage(image, metadata: dict, tmpdir):
"""
Create an archive on disk by hand with the provided image and metadata
"""
# Create ensured empty directory
while True:
id = str(randint(0, 1e6))
rootPath = Path(tmpdir, f"dataset-{id}/")
if not Path.exists(rootPath):
rootPath.mkdir()
break
# Create the archive by hand, with default readme and dataset description
Path(rootPath, 'README').write_text("README for pytest")
Path(rootPath, 'dataset_description.json') \
.write_text(json.dumps(DEFAULT_DATASET_DESC))
# Write the nifti image & metadata
dataPath = Path(rootPath, bids_build_path(metadata, BIDS_DIR_PATH_PATTERN))
dataPath.mkdir(parents=True)
filenamePrefix = bids_build_path(metadata, BIDS_FILE_PATTERN)
imagePath = Path(dataPath, filenamePrefix + '.nii')
metadataPath = Path(dataPath, filenamePrefix + '.json')
nib.save(image, str(imagePath))
# BIDS-I's takes care of this automatically, but must be done manually here
metadata['TaskName'] = metadata['task']
metadataPath.write_text(json.dumps(metadata))
del metadata['TaskName']
# BIDS-I's takes care of event file creation automatically, but must be done
# manually here
metadata['suffix'] = 'events'
metadata['extension'] = '.tsv'
eventsPath = Path(dataPath, bids_build_path(metadata, BIDS_FILE_PATTERN))
with open(eventsPath, mode='w') as eventsFile:
eventDefaultHeaders = ['onset', 'duration', 'response_time']
pd.DataFrame(columns=eventDefaultHeaders).to_csv(eventsFile, sep='\t')
# Create an archive from the directory and return it
return BidsArchive(rootPath)
def testAppendNoOverwriteDatasetMetadata(tmpdir, validBidsI):
rootPath = Path(tmpdir, "new-dataset")
archive = BidsArchive(rootPath)
EXPECTED_README = "The readme we expect"
validBidsI.readme = EXPECTED_README
archive._appendIncremental(validBidsI)
NEW_README = "The readme we don't expect"
validBidsI.readme = NEW_README
validBidsI.setMetadataField('subject', 'newSubject')
archive._appendIncremental(validBidsI)
with open(os.path.join(rootPath, 'README')) as readme:
readmeText = readme.readlines()
assert len(readmeText) == 1
assert readmeText[0] == EXPECTED_README
def openNeuroStreamTest(bidsInterface):
dsAccessionNumber = 'ds002338'
dsSubject = 'xp201'
datasetDir = tmpDownloadOpenNeuro(dsAccessionNumber, dsSubject, 1)
localEntities = {'subject': dsSubject, 'run': 1, 'suffix': 'bold', 'datatype': 'func'}
remoteEntities = {'subject': dsSubject, 'run': 1}
localBidsArchive = BidsArchive(datasetDir)
streamId = bidsInterface.initOpenNeuroStream(dsAccessionNumber, **remoteEntities)
for idx in range(3):
streamIncremental = bidsInterface.getIncremental(streamId)
localIncremental = localBidsArchive._getIncremental(idx, **localEntities)
print(f"OpenNeuro stream check: image {idx}")
assert streamIncremental == localIncremental
for idx in [5, 2, 7]:
streamIncremental = bidsInterface.getIncremental(streamId, volIdx=idx)
localIncremental = localBidsArchive._getIncremental(idx, **localEntities)
print(f"OpenNeuro stream check: image {idx}")
assert streamIncremental == localIncremental
# Resume without specifying volumes
for idx in [*range(8, 10)]:
streamIncremental = bidsInterface.getIncremental(streamId)
localIncremental = localBidsArchive._getIncremental(idx, **localEntities)
print(f"OpenNeuro stream check: image {idx}")
assert streamIncremental == localIncremental
numVols = bidsInterface.getNumVolumes(streamId)
assert numVols > 0 and numVols < 1000
# Check with local bidsRun
localBidsRun = localBidsArchive.getBidsRun(**localEntities)
assert numVols == localBidsRun.numIncrementals()
assert numVols > 10
for idx in [*range(6, 10)]:
streamIncremental = bidsInterface.getIncremental(streamId, volIdx=idx)
localIncremental = localBidsRun.getIncremental(idx)
print(f"OpenNeuro bidsRun check: image {idx}")
assert streamIncremental == localIncremental
def testNiftiHeaderValidation(sample4DNifti1, sample3DNifti1, sample2DNifti1,
caplog):
# Prepare test infrastructure
original3DHeader = sample3DNifti1.header.copy()
original4DHeader = sample4DNifti1.header.copy()
other3D = nib.Nifti1Image(sample3DNifti1.dataobj,
sample3DNifti1.affine,
sample3DNifti1.header)
assert other3D.header == original3DHeader
other4D = nib.Nifti1Image(sample4DNifti1.dataobj,
sample4DNifti1.affine,
sample4DNifti1.header)
assert other4D.header == original4DHeader
""" Test field values """
# Test equal headers
assert BidsArchive._imagesAppendCompatible(sample4DNifti1, other4D)
# Test unequal headers on variety of fields that must match
fieldsToModify = ["intent_code", "dim_info", "scl_slope", "sform_code"]
for field in fieldsToModify:
fieldArray = other4D.header[field]
oldValue = fieldArray.copy()
if np.sum(np.isnan(fieldArray)) > 0:
fieldArray = np.zeros(1)
else:
fieldArray = fieldArray + 1
other4D.header[field] = fieldArray
compatible, error = \
BidsArchive._imagesAppendCompatible(sample4DNifti1, other4D)
assert not compatible
assert "NIfTI headers don't match on field: " + field in error
other4D.header[field] = oldValue
""" Test special cases for dimensions and pixel dimensions being non-equal
but still append compatible """
# First three dimensions and pixel dimensions equal
assert BidsArchive._imagesAppendCompatible(sample3DNifti1, sample4DNifti1)
# Dimension 4 of the 3D image should not matter
for i in range(0, 100):
sample3DNifti1.header["dim"][4] = i
assert BidsArchive._imagesAppendCompatible(sample3DNifti1,
sample4DNifti1)
sample3DNifti1.header["dim"] = np.copy(original3DHeader["dim"])
assert sample3DNifti1.header == original3DHeader
""" Test special cases for dimensions and pixel dimensions being non-equal
and not append compatible """
# Ensure all headers are in their original states
assert sample4DNifti1.header == original4DHeader
assert other4D.header == original4DHeader
assert sample3DNifti1.header == original3DHeader
assert other3D.header == original3DHeader
# 4D with non-matching first 3 dimensions should fail
other4D.header["dim"][1:4] = other4D.header["dim"][1:4] * 2
compatible, errorMsg = BidsArchive._imagesAppendCompatible(sample4DNifti1,
other4D)
assert not compatible
assert "NIfTI headers not append compatible due to mismatch in dimensions "\
"and pixdim fields." in errorMsg
# Reset
other4D.header["dim"][1:4] = original4DHeader["dim"][1:4]
assert other4D.header == original4DHeader
# 3D and 4D in which first 3 dimensions don't match
other3D.header["dim"][1:3] = other3D.header["dim"][1:3] * 2
compatible, errorMsg = BidsArchive._imagesAppendCompatible(sample4DNifti1,
other3D)
assert not compatible
# Reset
other3D.header["dim"][1:3] = original3DHeader["dim"][1:3]
assert other3D.header == original3DHeader
# 2D and 4D are one too many dimensions apart
other4D.header['dim'][0] = 2
compatible, errorMsg = BidsArchive._imagesAppendCompatible(other4D,
sample4DNifti1)
assert not compatible
def testFailEmpty(tmpdir):
datasetRoot = Path(tmpdir, "bids-archive")
emptyArchive = BidsArchive(datasetRoot)
with pytest.raises(StateError):
emptyArchive.dirExistsInArchive("will fail anyway")
emptyArchive.getImages("will fail anyway")
emptyArchive.addImage(None, "will fall anyway")
emptyArchive.getSidecarMetadata("will fall anyway")
emptyArchive.addMetadata({"will": "fail"}, "will fall anyway")
emptyArchive.getIncremental(subject="will fall anyway",
session="will fall anyway",
task="will fall anyway",
suffix="will fall anyway",
datatype="will fall anyway")
def testEmptyArchiveCreation(tmpdir):
datasetRoot = Path(tmpdir, "bids-archive")
assert BidsArchive(datasetRoot) is not None
class BidsStream:
"""
A class that opens a BIDS archive and prepares to stream the data as
BIDS incrementals.
"""
def __init__(self, archivePath, **entities):
"""
Args:
archivePath: Absolute path of the BIDS archive.
entities: BIDS entities (subject, session, task, run, suffix, datatype) that
define the particular subject/run of the data to stream
"""
self.bidsArchive = BidsArchive(archivePath)
# TODO - when we have BidsRun
# self.bidsRun = self.bidsArchive.getBidsRun(**entities)
images = self.bidsArchive.getImages(**entities)
if len(images) == 0:
raise ValidationError('No matching images found')
if len(images) > 1:
raise ValidationError('More than one match, please give more specific subject/session/task/run')
self.bidsImage = images[0]
self.niftiImage = self.bidsImage.get_image()
self.filename = self.niftiImage.get_filename()
self.imgVolumes = nib.four_to_three(self.niftiImage)
self.metadata = self.bidsArchive.getSidecarMetadata(self.filename, includeEntities=True)
self.metadata.pop('extension')
self.numVolumes = len(self.imgVolumes)
self.nextVol = 0
def getNumVolumes(self) -> int:
"""Return the number of brain volumes in the run"""
# TODO - when we have BidsRun
# return self.bidsRun.getNumVolumes()
return self.numVolumes
def getIncremental(self, volIdx=-1) -> BidsIncremental:
"""
Get a BIDS incremental for the indicated index in the current subject/run
VolIdx acts similar to a file_seek pointer. If a volIdx >= 0 is supplied
the volume pointer is advanced to that position. If no volIdx or
a volIdx < 0 is supplied, then the next image volume after the previous
position is returned and the pointer is incremented.
Args:
volIdx: The volume index (or TR) within the run to retrieve.
Returns:
BidsIncremental of that volume index within this subject/run
"""
# TODO - when we have BidsRun
# return self.bidsRun.getIncremental(volIdx)
if volIdx >= 0:
# reset the next volume to the user specified volume
self.nextVol = volIdx
else:
# use the default next volume
pass
if self.nextVol < self.numVolumes:
incremental = BidsIncremental(self.imgVolumes[self.nextVol], self.metadata)
self.nextVol += 1
return incremental
else:
return None