def test_nodatatransform_reader():
filename = "mytest.bin"
outfile = Path(filename)
if outfile.exists():
outfile.unlink()
with BinFile(filename, "w") as f:
header = CFWBINARY()
header.setValue(1.0 / 240.0, 2019, 1, 28, 8, 30, 0.0, 0.0, 2, 0, 0,
constant.FORMAT_SHORT)
f.setHeader(header)
channel1 = CFWBCHANNEL()
channel1.setValue("I", "mmHg", 5.0, 0.0)
f.addChannel(channel1)
channel2 = CFWBCHANNEL("II", "mmHg", 1.0, 0.0)
f.addChannel(channel2)
chanData = []
d1 = [1, 2, 3, 4, 5, 6, 7, 8, 8, 7, 6, 5, 4, 3, 2, 1]
d2 = [8, 7, 6, 5, 4, 3, 2, 1, 1, 2, 3, 4, 5, 6, 7, 8]
data1 = []
data2 = []
for i in range(500):
data1 = data1 + d1
data2 = data2 + d2
chanData.append(data1)
chanData.append(data2)
f.writeHeader()
f.writeChannelData(chanData)
f.updateSamplesPerChannel(8000, True)
with BinFile(filename, "r") as g:
g.readHeader()
arr = g.readChannelData(0, 8000, False, False, 1.0, noDataScaling=True)
assert (len(g.channels) == 2)
assert (g.header.SamplesPerChannel == 8000)
# remove temporary file created
try:
outfile = Path(filename)
if outfile.exists():
outfile.unlink()
except:
# ignore error
pass
return
def runApp(fn, offset, length, width, height, useNumSamples, showHeaderOnly):
with BinFile(fn, "r") as f:
f.readHeader()
printHeaderInfo(f)
if not showHeaderOnly:
data = f.readChannelData(offset,
length,
useSecForOffset=not useNumSamples,
useSecForLength=not useNumSamples)
if (data is not None) and (len(data) > 0):
numSamples = len(data[0])
if numSamples > 0:
plotChannels(fn, f, data, width, height)
return
def renameChannels(self, print_rename_details: bool = False):
# progress
# need to support renaming of channel label
# also, support use of Regex for channel label, and renameTo expression
hasRenameTo = False
for cSettingInfo in self.channelInfoList:
renameTo = cSettingInfo.get("renameTo", "")
if len(renameTo) > 0:
hasRenameTo = True
break
# end-if
# end-for
if hasRenameTo:
print("Renaming channels in output files...")
numFilesChanged = 0
for fn in self.outputFileList:
print("processing {0}...".format(Path(fn).name))
updatedFile = False
with BinFile(fn, "r+") as f:
f.readHeader()
for c in f.channels:
cSettingInfo = self.getChannelInfo(c.Title)
if cSettingInfo is not None:
newLabel = cSettingInfo.get("renameTo", "")
if len(newLabel) > 0:
filename = Path(fn).name
oldLabel = c.Title
c.Title = newLabel
updatedFile = True
if print_rename_details:
print("{0} file: {1} -> {2}".format(
filename, oldLabel, newLabel))
if updatedFile:
f.writeHeader()
numFilesChanged += 1
# end-with
# end-for
if numFilesChanged == 0:
if print_rename_details:
print(
"No output files's channel labels need to be changed.")
def convert(self,
xmlFile: str,
tagsDict: Dict,
x: Xml2BinState,
print_processing_fn: bool = False):
"""
convert to BIN file from XML
"""
binFileOut = None
filename = ""
chanLabel = []
tempChanInfo = []
tempChanLabel = []
tempChanLabel2Index = {}
startWaveTm = datetime.datetime.min
numSamples = 0
totalNumSamplesWritten = 0
firstBinFile = True
firstMeasurement = True
# array of parName, startTm, vitalFileOut, filename
startVitalTm = datetime.datetime.min
vitalFileInfoArr = []
vitalParName2Info = {}
# progress
if (x.lastVitalFileInfoArr
is not None) and (len(x.lastVitalFileInfoArr) > 0):
for vinfo in x.lastVitalFileInfoArr:
vs_parameter = vinfo["par"]
vs_startTm = vinfo["startTm"]
vitalFilename = vinfo["filename"]
vitalFileOut = VitalFile(vitalFilename, "r+")
vitalFileOut.open()
vitalFileInfo = {
"par": vs_parameter,
"startTm": vs_startTm,
"vitalFileOut": vitalFileOut,
"filename": vitalFilename
}
vitalFileInfoArr.append(vitalFileInfo)
vitalParName2Info[vs_parameter] = vitalFileInfo
xml_unit = ""
xml_bed = ""
infilePath = Path(xmlFile)
if not infilePath.exists():
raise XmlConverterError("Cannot open file: {0}".format(xmlFile))
else:
if len(x.lastBinFilename) > 0:
binFileOut = BinFile(x.lastBinFilename, "r+")
binFileOut.open()
binFileOut.readHeader()
filename = x.lastBinFilename
firstBinFile = False
# copy BIN file's channel info
chanLabel = []
tempChanLabel = []
chanLabel2Index = {}
tempChanLabel2Index = {}
idx = 0
for c in binFileOut.channels:
chanLabel.append(c.Title)
chanLabel2Index[c.Title] = idx
tempChanLabel.append(c.Title)
tempChanLabel2Index[c.Title] = idx
idx += 1
self.header = binFileOut.header
second = int(math.floor(self.header.Second))
microsecond = int(
(self.header.Second - math.floor(self.header.Second)) *
100)
self.headerStartDt = datetime.datetime(
self.header.Year, self.header.Month, self.header.Day,
self.header.Hour, self.header.Minute, second, microsecond)
numSamples = self.header.SamplesPerChannel
# end-if len(x.lastBinFilename)
if print_processing_fn:
print("Processing XML file: {0}".format(infilePath.name))
tree = ET.parse(xmlFile)
root = tree.getroot()
# print("root tag = {0}".format(root.tag))
if root.tag == "BedMasterEx":
for child1 in root:
if child1.tag == "FileInfo":
for child3 in child1:
if child3.tag == "Unit":
xml_unit = child3.text
elif child3.tag == "Bed":
xml_bed = child3.text
if child1.tag == "Segment":
for child3 in child1:
if child3.tag == "Waveforms":
collectionTime, collectionTimeUTC = self.processWaveforms(
child3)
collectionTimeDt = parsetime(collectionTime)
tempChanInfo = []
tempChanLabel = []
tempChanLabel2Index = {}
if self.header is None:
self.headerStartDt = collectionTimeDt
self.header = CFWBINARY()
self.header.setValue(
1.0 / self.defaultSamplesPerSec,
collectionTimeDt.year,
collectionTimeDt.month,
collectionTimeDt.day,
collectionTimeDt.hour,
collectionTimeDt.minute,
collectionTimeDt.second, 0, 0)
# print(collectionTime, collectionTimeUTC)
idx = 0
for child4 in child3:
if (child4.tag == "WaveformData"):
ID, channel, hz, points, uom, wave = self.processWaveformData(
child4)
if self.inChannelPatternList(channel):
# print(channel, wave, points, pointsBytes, min_, max_, offset, gain, hz)
wavedata = self.decodeWave(wave)
# print(wavedata)
if self.defaultSamplesPerSec != hz:
wavedata = fixsamplingarr(
wavedata, hz,
self.defaultSamplesPerSec)
tempChanInfo.append({
"label": channel,
"data": wavedata,
"points": points,
"hz": hz,
"uom": uom
})
tempChanLabel.append(channel)
tempChanLabel2Index[channel] = idx
idx += 1
# end-for child4
if (firstBinFile is True) or (
len(tempChanLabel) > 0
and self.channelChanged(
chanLabel, tempChanLabel)):
if firstBinFile is False:
binFileOut.close()
binFileOut = None
# rename the file that we just closed (if filename pattern has {endtime})
self.renameOutputFnWithEndtime(
numSamples, tagsDict, x, filename)
if not (x.lastBinFilename
in self.outputFileSet):
self.outputFileSet.add(
x.lastBinFilename)
self.outputFileList.append(
x.lastBinFilename)
# reset new headerStartDt
self.headerStartDt = collectionTimeDt
self.header.setValue(
1.0 / self.defaultSamplesPerSec,
collectionTimeDt.year,
collectionTimeDt.month,
collectionTimeDt.day,
collectionTimeDt.hour,
collectionTimeDt.minute,
collectionTimeDt.second, 0, 0)
firstBinFile = False
self.header.NChannels = len(tempChanInfo)
fmt = self.outputFnTimeFormatDict.get(
"starttime",
None) if (self.outputFnTimeFormatDict
is not None) else None
tagsDict["starttime"] = dtTimestampFormat(
self.headerStartDt, fmt)
fmt = self.outputFnTimeFormatDict.get(
"exetime",
None) if (self.outputFnTimeFormatDict
is not None) else None
tagsDict["exetime"] = dtTimestampFormat(
x.timestampTm, fmt)
# we do not know the end at this point
tagsDict["endtime"] = "tempendtime" + str(
random.randint(10000, 100000))
filename = getOutputFilename(
self.outputDir, self.outputFnPattern,
tagsDict, self.outputFnExt)
x.lastBinFilename = filename
binFileOut = BinFile(filename, "w")
binFileOut.open()
binFileOut.setHeader(self.header)
chanData = []
chanLabel = []
for cinfo in tempChanInfo:
label = cinfo["label"]
cSettingInfo = self.getChannelInfo(
label)
if cSettingInfo is not None:
uom = cSettingInfo.get(
"uom", cinfo.get("uom", ""))
rangeLow = cSettingInfo.get(
"rangeLow", 0)
rangeHigh = cSettingInfo.get(
"rangeHigh", 100)
offset = cSettingInfo.get(
"offset", 0)
scale = cSettingInfo.get(
"scale", 1)
else:
uom = cinfo.get("uom", "")
rangeLow = 0
rangeHigh = 100
offset = 0
scale = 1
channel = CFWBCHANNEL()
channel.setValue(
label, uom, scale, offset,
rangeLow, rangeHigh)
binFileOut.addChannel(channel)
chanData.append(cinfo["data"])
chanLabel.append(label)
binFileOut.writeHeader()
firstMeasurement = False
numSamples = binFileOut.writeChannelData(
chanData)
totalNumSamplesWritten += numSamples
binFileOut.updateSamplesPerChannel(
numSamples, True)
elif len(tempChanInfo) > 0:
chanData = []
chanLabel = []
for cinfo in tempChanInfo:
label = cinfo["label"]
chanData.append(cinfo["data"])
chanLabel.append(label)
# gap handling
endDt = self.headerStartDt + datetime.timedelta(
seconds=int(numSamples /
self.defaultSamplesPerSec))
gap = collectionTimeDt - endDt
actualGapInSec = int(gap.total_seconds())
if (not self.ignoreGapBetweenSegs
) and firstMeasurement:
gapInSec = actualGapInSec
elif not self.ignoreGap:
gapInSec = actualGapInSec
else:
gapInSec = 0
if self.warningOnGaps and (gapInSec != 0):
print(
"Waveforms CollectionTime: {0} shows gap (or overlap) = {1} secs"
.format(collectionTime, gapInSec))
firstMeasurement = False
numSamplesWritten = binFileOut.writeChannelData(
chanData, self.defaultSamplesPerSec,
gapInSec)
totalNumSamplesWritten += numSamplesWritten
numSamples = numSamplesWritten + binFileOut.header.SamplesPerChannel
binFileOut.header.SamplesPerChannel = numSamples
self.header.SamplesPerChannel = numSamples
binFileOut.updateSamplesPerChannel(
numSamples, True)
# end-if firstBinFile
# end-if "measurement"
if child3.tag == "VitalSigns":
collectionTime, collectionTimeUTC = self.processVitalSigns(
child3)
collectionTimeDt = parsetime(collectionTime)
vs_parameter = ""
vs_time = ""
vs_value = ""
vs_uom = ""
vs_alarmLimitLow = ""
vs_alarmLimitHigh = ""
for child4 in child3:
if (child4.tag == "VitalSign"):
vs_parameter, vs_time, vs_value, vs_uom, vs_alarmLimitLow, vs_alarmLimitHigh = self.processVitalSign(
child4)
# print(vs_parameter)
if (vs_parameter is not None
) and len(vs_parameter) > 0:
vitalFileInfo = None
if vs_parameter in vitalParName2Info:
vitalFileInfo = vitalParName2Info.get(
vs_parameter)
else:
vs_time_dt = parsetime(vs_time)
fmt = self.outputFnTimeFormatDict.get(
"starttime", None) if (
self.outputFnTimeFormatDict
is not None) else None
tagsDict[
"starttime"] = dtTimestampFormat(
vs_time_dt, fmt)
fmt = self.outputFnTimeFormatDict.get(
"exetime", None) if (
self.outputFnTimeFormatDict
is not None) else None
tagsDict[
"exetime"] = dtTimestampFormat(
x.timestampTm, fmt)
# we do not know the end at this point
tagsDict[
"endtime"] = "0000" # "tempendtime" + str(random.randint(10000, 100000))
# array of parName, startTm, vitalFileOut, filename
vitalFilename = getOutputFilename(
self.outputDir,
self.outputFnPattern + "_" +
vs_parameter, tagsDict,
"vital")
vitalFileOut = VitalFile(
vitalFilename, "w")
vitalFileOut.open()
startVitalTm = vs_time_dt
vitalFileInfo = {
"par": vs_parameter,
"startTm": startVitalTm,
"vitalFileOut": vitalFileOut,
"filename": vitalFilename
}
vitalFileInfoArr.append(
vitalFileInfo)
vitalParName2Info[
vs_parameter] = vitalFileInfo
vs_header = VITALBINARY(
vs_parameter, vs_uom, xml_unit,
xml_bed, startVitalTm.year,
startVitalTm.month,
startVitalTm.day,
startVitalTm.hour,
startVitalTm.minute,
startVitalTm.second)
vitalFileOut.setHeader(vs_header)
vitalFileOut.writeHeader()
if vitalFileInfo is not None:
vs_value_num = DEFAULT_VS_LIMIT_LOW
try:
vs_value_num = float(vs_value)
except:
pass
vs_time_dt = parsetime(vs_time)
vs_offset_num = (
vs_time_dt -
vitalFileInfo["startTm"]
).total_seconds()
vs_low_num = DEFAULT_VS_LIMIT_LOW
try:
vs_low_num = float(
vs_alarmLimitLow)
except:
pass
vs_high_num = DEFAULT_VS_LIMIT_HIGH
try:
vs_high_num = float(
vs_alarmLimitHigh)
except:
pass
vitalFileOut = vitalFileInfo[
"vitalFileOut"]
vitalFileOut.writeVitalData(
vs_value_num, vs_offset_num,
vs_low_num, vs_high_num)
# end-for child1
# end-if root
# end-if
if binFileOut is not None:
binFileOut.close()
binFileOut = None
# rename the file that we just closed (if filename pattern has {endtime})
self.renameOutputFnWithEndtime(numSamples, tagsDict, x, filename)
if not (x.lastBinFilename in self.outputFileSet):
self.outputFileSet.add(x.lastBinFilename)
self.outputFileList.append(x.lastBinFilename)
for vf in vitalFileInfoArr:
vitalFileOut = vf["vitalFileOut"]
if vitalFileOut is not None:
vitalFileOut.close()
vf["vitalFileOut"] = None
x.addOrUpdateLastVitalFileInfo(vf["par"], vf["startTm"],
vf["filename"])
return totalNumSamplesWritten
def convert(self, xmlFile: str, tagsDict: Dict, x: Xml2BinState, print_processing_fn: bool = False):
"""
convert to BIN file from XML
"""
binFileOut = None
filename = ""
chanLabel = []
tempChanInfo = []
tempChanLabel = []
tempChanLabel2Index = {}
startWaveTm = datetime.datetime.min
numSamples = 0
totalNumSamplesWritten = 0
firstBinFile = True
firstMeasurement = True
infilePath = Path(xmlFile)
if not infilePath.exists():
raise XmlConverterError("Cannot open file: {0}".format(xmlFile))
else:
if len(x.lastBinFilename) > 0:
binFileOut = BinFile(x.lastBinFilename, "r+")
binFileOut.open()
binFileOut.readHeader()
filename = x.lastBinFilename
firstBinFile = False
# copy BIN file's channel info
chanLabel = []
tempChanLabel = []
chanLabel2Index = {}
tempChanLabel2Index = {}
idx = 0
for c in binFileOut.channels:
chanLabel.append(c.Title)
chanLabel2Index[c.Title] = idx
tempChanLabel.append(c.Title)
tempChanLabel2Index[c.Title] = idx
idx += 1
self.header = binFileOut.header
second = int(math.floor(self.header.Second))
microsecond = int((self.header.Second - math.floor(self.header.Second)) * 100)
self.headerStartDt = datetime.datetime(self.header.Year, self.header.Month, self.header.Day,
self.header.Hour, self.header.Minute, second, microsecond)
numSamples = self.header.SamplesPerChannel
# end-if len(x.lastBinFilename)
if print_processing_fn:
print("Processing XML file: {0}".format(infilePath.name))
tree = ET.parse(xmlFile)
root = tree.getroot()
# print("root tag = {0}".format(root.tag))
if root.tag == "cpcArchive":
for child1 in root:
if child1.tag == "cpc":
cpc_datetime, cpc_tzoffset = self.processCpc(child1)
for child2 in child1:
if child2.tag == "device":
for child3 in child2:
if child3.tag == "measurements":
pollTime, tz_offset = self.processMeassurement(child3)
pollTimeDt = parsetime(pollTime.replace('T', ' ').replace('Z', ''))
if pollTimeDt is None:
# use cpc_datetime if measurement does not have PollTime
cpc_dt1 = cpc_datetime
cpc_dt_parts = cpc_dt1.split('.', 2)
if len(cpc_dt_parts) > 1:
cpc_dt1 = cpc_dt_parts[0]
pollTimeDt = parsetime(cpc_dt1.replace('T', ' ').replace('Z', ''))
# if still cannot get a valid PollTime
# just skip for now... maybe need to print warning
if pollTimeDt is None:
continue
else:
tempChanInfo = []
tempChanLabel = []
tempChanLabel2Index = {}
if self.header is None:
self.headerStartDt = pollTimeDt
self.header = CFWBINARY()
self.header.setValue(1.0 / self.defaultSamplesPerSec, pollTimeDt.year, pollTimeDt.month,
pollTimeDt.day, pollTimeDt.hour, pollTimeDt.minute, pollTimeDt.second, 0, 0)
# print(pollTime, tz_offset)
idx = 0
for child4 in child3:
if (child4.tag == "mg"):
channel, wave, points, pointsBytes, min_, max_, offset, gain, hz = self.processMg(child4)
if self.inChannelPatternList(channel):
# print(channel, wave, points, pointsBytes, min_, max_, offset, gain, hz)
wavedata = self.decodeWave(wave)
# print(wavedata)
if self.defaultSamplesPerSec != hz:
wavedata = fixsamplingarr(wavedata, hz, self.defaultSamplesPerSec)
tempChanInfo.append({"label": channel, "data": wavedata, "points": points, "pointsBytes": pointsBytes,
"min": min_, "max": max_, "offset": offset, "gain": gain, "hz": hz})
tempChanLabel.append(channel)
tempChanLabel2Index[channel] = idx
idx += 1
# end-for child4
# progress
if (firstBinFile is True) or (len(tempChanLabel) > 0 and self.channelChanged(chanLabel, tempChanLabel)):
if firstBinFile is False:
binFileOut.close()
binFileOut = None
# rename the file that we just closed (if filename pattern has {endtime})
self.renameOutputFnWithEndtime(numSamples, tagsDict, x, filename)
if not (x.lastBinFilename in self.outputFileSet):
self.outputFileSet.add(x.lastBinFilename)
self.outputFileList.append(x.lastBinFilename)
# reset new headerStartDt
self.headerStartDt = pollTimeDt
self.header.setValue(1.0 / self.defaultSamplesPerSec, pollTimeDt.year, pollTimeDt.month,
pollTimeDt.day, pollTimeDt.hour, pollTimeDt.minute, pollTimeDt.second, 0, 0)
firstBinFile = False
self.header.NChannels = len(tempChanInfo)
fmt = self.outputFnTimeFormatDict.get("starttime", None) if (self.outputFnTimeFormatDict is not None) else None
tagsDict["starttime"] = dtTimestampFormat(self.headerStartDt, fmt)
fmt = self.outputFnTimeFormatDict.get("exetime", None) if (self.outputFnTimeFormatDict is not None) else None
tagsDict["exetime"] = dtTimestampFormat(x.timestampTm, fmt)
# we do not know the end at this point
tagsDict["endtime"] = "tempendtime" + str(random.randint(10000, 100000))
filename = getOutputFilename(self.outputDir, self.outputFnPattern, tagsDict, self.outputFnExt)
x.lastBinFilename = filename
binFileOut = BinFile(filename, "w")
binFileOut.open()
binFileOut.setHeader(self.header)
chanData = []
chanLabel = []
for cinfo in tempChanInfo:
label = cinfo["label"]
cSettingInfo = self.getChannelInfo(label)
if cSettingInfo is not None:
uom = cSettingInfo.get("uom", "")
rangeLow = cSettingInfo.get("rangeLow", cinfo["min"])
rangeHigh = cSettingInfo.get("rangeHigh", cinfo["max"])
offset = cSettingInfo.get("offset", cinfo["offset"])
scale = cSettingInfo.get("scale", cinfo["gain"])
else:
uom = ""
rangeLow = cinfo["min"]
rangeHigh = cinfo["max"]
offset = cinfo["offset"]
scale = cinfo["gain"]
channel = CFWBCHANNEL()
channel.setValue(label, uom, scale, offset, rangeLow, rangeHigh)
binFileOut.addChannel(channel)
chanData.append(cinfo["data"])
chanLabel.append(label)
binFileOut.writeHeader()
firstMeasurement = False
numSamples = binFileOut.writeChannelData(chanData)
totalNumSamplesWritten += numSamples
binFileOut.updateSamplesPerChannel(numSamples, True)
elif len(tempChanInfo) > 0:
chanData = []
chanLabel = []
for cinfo in tempChanInfo:
label = cinfo["label"]
chanData.append(cinfo["data"])
chanLabel.append(label)
# gap handling
endDt = self.headerStartDt + datetime.timedelta(seconds=int(numSamples / self.defaultSamplesPerSec))
gap = pollTimeDt - endDt
actualGapInSec = int(gap.total_seconds())
if (not self.ignoreGapBetweenSegs) and firstMeasurement:
gapInSec = actualGapInSec
elif not self.ignoreGap:
gapInSec = actualGapInSec
else:
gapInSec = 0
if self.warningOnGaps and (gapInSec != 0):
if pollTime is None or len(pollTime) == 0:
print("cpc datetime: {0} shows gap (or overlap) = {1} secs".format(cpc_datetime, gapInSec))
else:
print("Measurement POLLTIME: {0} shows gap (or overlap) = {1} secs".format(pollTime, gapInSec))
firstMeasurement = False
numSamplesWritten = binFileOut.writeChannelData(chanData, self.defaultSamplesPerSec, gapInSec)
totalNumSamplesWritten += numSamplesWritten
numSamples = numSamplesWritten + binFileOut.header.SamplesPerChannel
binFileOut.header.SamplesPerChannel = numSamples
self.header.SamplesPerChannel = numSamples
binFileOut.updateSamplesPerChannel(numSamples, True)
# end-if firstBinFile
# end-if "measurement"
# end-for child3
# end-for child2
# end-for child1
# end-if root
# end-if
if binFileOut is not None:
binFileOut.close()
binFileOut = None
# rename the file that we just closed (if filename pattern has {endtime})
self.renameOutputFnWithEndtime(numSamples, tagsDict, x, filename)
if not (x.lastBinFilename in self.outputFileSet):
self.outputFileSet.add(x.lastBinFilename)
self.outputFileList.append(x.lastBinFilename)
return totalNumSamplesWritten