def rebuildModel(self):
numSeconds = self.modelSecondsSpinBox.value()
spikeFreq = self.modelFrequencySpinBox.value()
amp = self.modelAmpSpinBox.value()
noiseAmp = self.modelNoiseAmpSpinBox.value()
fs = 10000 # 10 kHz
t, spikeTrain, data = getSpikeTrain(numSeconds=numSeconds,
spikeFreq=spikeFreq,
fs=fs,
amp=amp,
noiseAmp=noiseAmp)
# (t, data) need to be one column (for bAnalysis)
t = t.reshape(t.shape[0],-1)
data = data.reshape(data.shape[0],-1)
self._startSec = 0
self._stopSec = numSeconds
#t = t[:,0]
#data = data[:,0]
print(' model t:', t.shape)
print(' model data:', data.shape)
modelDict = {
'sweepX': t,
'sweepY': data,
'mode': 'I-Clamp'
}
self._ba = sanpy.bAnalysis(fromDict=modelDict)
class xxx_Test_Analysis(unittest.TestCase):
# this patter is wierd to me?
path = 'data/19114001.abf'
ba = sanpy.bAnalysis(path)
expectedNumSpikes = 102
expectedNumErrors = 1
def test_0_load(self):
logger.info('RUNNING')
self.assertFalse(self.ba.loadError)
self.assertIsNotNone(self.ba.sweepX)
self.assertIsNotNone(self.ba.sweepY)
self.assertEqual(len(self.ba.sweepX), len(self.ba.sweepY))
def test_1_detect(self):
logger.info('RUNNING')
# grab detection parameters
dDict = sanpy.bAnalysis.getDefaultDetection()
# detect
self.ba.spikeDetect(dDict)
self.assertEqual(self.ba.numSpikes,
self.expectedNumSpikes) # expecting 102 spikes
self.assertEqual(len(self.ba.dfError),
self.expectedNumErrors) # expecting 102 spikes
def test_2_stats(self):
logger.info('RUNNING')
thresholdSec = self.ba.getStat('thresholdSec')
self.assertEqual(len(thresholdSec),
self.expectedNumSpikes) # expecting 102 spikes
def loadSanPyDir(path):
"""
Load folder of .abf
Ret:
list: list of dict
"""
print('app.py loadSanPyDir() path:', path)
retList = []
for file in os.listdir(path):
if file.startswith('.'):
continue
if file.endswith('.abf'):
#print(' loadSanPyDir() file: ', file)
filePath = os.path.join(path, file)
ba = sanpy.bAnalysis(filePath)
headerDict = ba.api_getHeader()
item = {
'header': headerDict,
'ba': ba
}
retList.append(item)
#
return retList
def getFileList(path):
retFileList = []
useExtension = '.abf'
videoFileIdx = 0
fileDict = {}
fileDict['File Name'] = ''
#fileDict['path'] = ''
fileDict['kHz'] = ''
fileDict['Duration (Sec)'] = ''
fileDict['Number of Sweeps'] = ''
for file in os.listdir(path):
if file.startswith('.'):
continue
if file.endswith(useExtension):
fullPath = os.path.join(path, file)
fileDict = {} # WOW, I need this here !!!!!!!!
fileDict['File Name'] = file
#fileDict['path'] = fullPath
tmp_ba = sanpy.bAnalysis(file=fullPath)
pntsPerMS = tmp_ba.dataPointsPerMs
numSweeps = len(tmp_ba.sweepList)
durationSec = max(tmp_ba.abf.sweepX)
fileDict['kHz'] = pntsPerMS
fileDict['Duration (Sec)'] = int(round(durationSec))
fileDict['Number of Sweeps'] = numSweeps
retFileList.append(fileDict)
videoFileIdx += 1
if len(retFileList) == 0:
retFileList.append(fileDict)
return retFileList
def getFileRow(self, path, loadData=False):
"""
Get dict representing one file (row in table). Loads bAnalysis to get headers.
On load error of proper file type (abf, csv), ba.loadError==True
Args:
path (Str): Full path to file.
#rowIdx (int): Optional row index to assign in column 'Idx'
Return:
(tuple): tuple containing:
- ba (bAnalysis): [sanpy.bAnalysis](/api/bAnalysis).
- rowDict (dict): On success, otherwise None.
fails when path does not lead to valid bAnalysis file.
"""
if not os.path.isfile(path):
logger.warning(f'Did not find file "{path}"')
return None, None
fileType = os.path.splitext(path)[1]
if fileType not in self.theseFileTypes:
logger.warning(f'Did not load file type "{fileType}"')
return None, None
# load bAnalysis
#logger.info(f'Loading bAnalysis "{path}"')
ba = sanpy.bAnalysis(path, loadData=loadData)
if ba.loadError:
logger.error(f'Error loading bAnalysis file "{path}"')
#return None, None
# not sufficient to default everything to empty str ''
# sanpyColumns can only have type in ('float', 'str')
rowDict = dict.fromkeys(self.sanpyColumns.keys() , '')
for k in rowDict.keys():
if self.sanpyColumns[k]['type'] == str:
rowDict[k] = ''
elif self.sanpyColumns[k]['type'] == float:
rowDict[k] = np.nan
#if rowIdx is not None:
# rowDict['Idx'] = rowIdx
if ba.loadError:
rowDict['I'] = 0
else:
rowDict['I'] = 2 # need 2 because checkbox value is in (0,2)
rowDict['File'] = ba.getFileName() #os.path.split(ba.path)[1]
rowDict['Dur(s)'] = ba.recordingDur
rowDict['Sweeps'] = ba.numSweeps
rowDict['kHz'] = ba.recordingFrequency
rowDict['Mode'] = ba.recordingMode
rowDict['dvdtThreshold'] = 20
rowDict['mvThreshold'] = -20
return ba, rowDict
def test():
path = '/home/cudmore/Sites/SanPy/data/19114001.abf'
ba = sanpy.bAnalysis(path)
ba.spikeDetect()
be = bExport(ba)
df = be.getSummary()
logger.info('')
print(df)
def loadFile(name):
filePath = os.path.join(myPath, name)
global ba
ba = sanpy.bAnalysis(filePath)
dDict = ba.getDefaultDetection()
dDict['dvdtThreshold'] = myThreshold
ba.spikeDetect(dDict)
start = 0
stop = len(ba.abf.sweepX) - 1
global subSetOfPnts
subSetOfPnts = range(start, stop, plotEveryPoint)
def main():
path = '/home/cudmore/Sites/SanPy/data/19114001.abf'
ba = sanpy.bAnalysis(path)
ba.spikeDetect()
print(ba.numSpikes)
import sys
app = QtWidgets.QApplication([])
pr = plotRecording(ba=ba)
pr.show()
sys.exit(app.exec_())
def loadOneAnalysis(self, path, uuid=None, allowAutoLoad=True):
"""
Load one bAnalysis either from original file path or uuid of h5 file.
If from h5, we still need to reload sweeps !!!
They are binary and fast, saving to h5 (in this case) is slow.
"""
logger.info(f'path:"{path}" uuid:"{uuid}" allowAutoLoad:"{allowAutoLoad}"')
ba = None
if uuid is not None and uuid:
# load from h5
hdfFile = os.path.splitext(self.dbFile)[0] + '.h5'
hdfPath = os.path.join(self.path, hdfFile)
with pd.HDFStore(hdfPath) as hdfStore:
try:
dfAnalysis = hdfStore[uuid]
ba = sanpy.bAnalysis(fromDf=dfAnalysis)
# WHY IS THIS BROKEN ?????????????????????????????????????????????????????????????
# This is SLOPPY ON MY PART, WE ALWAYS NEED TO RELOAD FILTER
# ADD SOME COD THAT CHECKS THIS AND REBUILDS AS NECC !!!!!!!!!!
if path.endswith('.tif'):
# kymograph
ba._loadTif()
else:
ba._loadAbf()
ba.rebuildFiltered()
logger.info(f'Loaded ba from h5 uuid {uuid} and now ba:{ba}')
except (KeyError):
logger.error(f'Did not find uuid in h5 file, uuid:{uuid}')
if allowAutoLoad and ba is None:
# load from path
ba = sanpy.bAnalysis(path)
logger.info(f'Loaded ba from path {path} and now ba:{ba}')
#
return ba
def testPlot():
import os
file_path = os.path.realpath(__file__) # full path to this file
file_path = os.path.split(file_path)[0] # folder for this file
path = os.path.join(file_path, '../../../data/19114001.abf')
ba = bAnalysis(path)
if ba.loadError:
print('error loading file')
return
ba.spikeDetect()
# create plugin
ap = plotRecording(ba=ba)
def loadFolder(self):
"""
Load using cloudDict
"""
"""
# use ['download_url'] to download abf file (no byte conversion)
response[0] = {
name : 171116sh_0018.abf
path : data/171116sh_0018.abf
sha : 5f3322b08d86458bf7ac8b5c12564933142ffd17
size : 2047488
url : https://api.github.com/repos/cudmore/SanPy/contents/data/171116sh_0018.abf?ref=master
html_url : https://github.com/cudmore/SanPy/blob/master/data/171116sh_0018.abf
git_url : https://api.github.com/repos/cudmore/SanPy/git/blobs/5f3322b08d86458bf7ac8b5c12564933142ffd17
download_url : https://raw.githubusercontent.com/cudmore/SanPy/master/data/171116sh_0018.abf
type : file
_links : {'self': 'https://api.github.com/repos/cudmore/SanPy/contents/data/171116sh_0018.abf?ref=master', 'git': 'https://api.github.com/repos/cudmore/SanPy/git/blobs/5f3322b08d86458bf7ac8b5c12564933142ffd17', 'html': 'https://github.com/cudmore/SanPy/blob/master/data/171116sh_0018.abf'}
}
"""
owner = self._cloudDict['owner']
repo_name = self._cloudDict['repo_name']
path = self._cloudDict['path']
url = f'https://api.github.com/repos/{owner}/{repo_name}/contents/{path}'
# response is a list of dict
response = requests.get(url).json()
#print('response:', type(response))
for idx, item in enumerate(response):
if not item['name'].endswith('.abf'):
continue
print(idx)
# use item['git_url']
for k,v in item.items():
print(' ', k, ':', v)
#
# test load
download_url = response[1]['download_url']
content = requests.get(download_url).content
fileLikeObject = io.BytesIO(content)
ba = sanpy.bAnalysis(byteStream=fileLikeObject)
#print(ba._abf)
#print(ba.api_getHeader())
ba.spikeDetect()
print(ba.numSpikes)
def parse_contents_abf(contents, filename, date):
"""
parses binary abf file
"""
print('app2.parse_contents_abf() filename:', filename)
content_type, content_string = contents.split(',')
print(' content_type:', content_type)
decoded = base64.b64decode(content_string)
#print(' type(decoded)', type(decoded))
fileLikeObject = io.BytesIO(decoded)
#print(' type(fileLikeObject)', type(fileLikeObject))
#print(len(fileLikeObject.getvalue()))
#major = pyabf.abfHeader.abfFileFormat(fileLikeObject)
#print(' pyabf major:', major)
print(' *** instantiating sanpy.bAnalysis with byte stream')
try:
global ba
ba = sanpy.bAnalysis(byteStream=fileLikeObject)
global baList
baList.append(ba)
#print(ba.abf.headerText)
# TODO: Add to file list
# {'File Name':fileName, 'kHz':, kHz, 'Duration (Sec)':durSeconds, 'Number of Sweeps':numberOfSweeps}
print(' todo: keep track of ba for this file and add to file list')
# todo: get rid of this weirdness
start = 0
#stop = len(ba.abf.sweepX) - 1
stop = len(ba.sweepX) - 1
print(' stop:', stop)
print(' plotEveryPoint:', plotEveryPoint)
global subSetOfPnts
subSetOfPnts = slice(start, stop, plotEveryPoint)
#print(' subSetOfPnts:', subSetOfPnts)
except:
print('*** exception in app2.parse_contents_abf():')
print(sys.exc_info())
return html.Div([
html.H5(f'Loaded file: {filename}'),
])
def test_plot(path):
print('=== test_plot() path:', path)
ba = sanpy.bAnalysis(path)
# detect
dDict = sanpy.bAnalysis.getDefaultDetection()
dDict['dvdThreshold'] = 50
ba.spikeDetect(dDict)
# plot
bp = sanpy.bAnalysisPlot(ba)
fig = bp.plotDerivAndRaw()
fig = bp.plotSpikes()
plt.show()
def test_plugin(): import sys from PyQt5 import QtCore, QtWidgets, QtGui import sanpy import sanpy.interface # create a PyQt application app = QtWidgets.QApplication([]) # load and analyze sample data path = '/home/cudmore/Sites/SanPy/data/19114001.abf' ba = sanpy.bAnalysis(path) ba.spikeDetect() # open the interface for the 'saveAnalysis' plugin. sa = sanpy.interface.plugins.plotScatter(ba=ba, startStop=None) sys.exit(app.exec_())
def loadOneFile(bucket, key, s3=None):
ba = None
if s3 is None:
s3 = boto3.resource('s3')
file_stream = io.BytesIO()
try:
s3.Object(bucket, key).download_fileobj(file_stream)
except (boto3.exceptions.botocore.exceptions.ClientError) as e:
print(e)
#raise
except (boto3.exceptions.botocore.exceptions.EndpointConnectionError) as e:
print(e)
#raise
else:
ba = sanpy.bAnalysis(byteStream=file_stream)
print(' loadOneFile() loaded key:', key, 'ba:', ba)
#
return ba
def run():
# create client assuming ~/.aws/xxx has been defined
s3 = boto3.resource('s3')
myBucketName = 'sanpy-data'
keyList = fetchFileList(myBucketName, s3=s3)
print(f'loading {len(keyList)} keys/files')
for key in keyList:
file_stream = io.BytesIO()
# works with boto3.client('s3')
#s3Client.download_fileobj(myBucketName, key, file_stream)
s3.Object(myBucketName, key).download_fileobj(file_stream)
ba = sanpy.bAnalysis(byteStream=file_stream)
print(' loaded key:', key, 'ba:', ba)
# download one file
'''
def test2():
# load abf
path = '/home/cudmore/Sites/SanPy/data/19114000.abf'
ba = sanpy.bAnalysis(path)
# detect
sweepNumber = 0
detectionClass = ba.detectionClass
detectionClass['verbose'] = True
ba.spikeDetect2__(sweepNumber, detectionClass)
'''
printSpikeNum = 4
print(f'== printing spike {printSpikeNum}')
ba.printSpike(printSpikeNum)
'''
#ba.printErrors()
sd = ba.getSpikeDictionaries()
'''
def test_hd5_2():
folderPath = '/home/cudmore/Sites/SanPy/data'
if 1:
# save analysisDir hdf
bad = analysisDir(folderPath)
print('bad._df:')
print(bad._df)
#bad.saveDatabase() # save .csv
bad.saveHdf() # save ALL bAnalysis in .h5
if 0:
# load h5 and reconstruct a bAnalysis object
start = time.time()
hdfPath = '/home/cudmore/Sites/SanPy/data/sanpy_recording_db.h5'
with pd.HDFStore(hdfPath, 'r') as hdfStore:
for key in hdfStore.keys():
dfTmp = hdfStore[key]
#path = dfTmp.iloc[0]['path']
print('===', key)
#if not key.startswith('/r6'):
# continue
#for col in dfTmp.columns:
# print(' ', col, dfTmp[col])
#print(key, type(dfTmp), dfTmp.shape)
#print('dfTmp:')
#print(dfTmp)
#print(dfTmp.iloc[0]['_sweepX'])
# load bAnalysis from a pandas DataFrame
ba = sanpy.bAnalysis(fromDf=dfTmp)
print(ba)
ba.spikeDetect() # this should reproduce exactly what was save ... It WORKS !!!
stop = time.time()
logger.info(f'h5 load took {round(stop-start,3)} seconds')
def parse_contents_abf(contents, filename, date):
"""
parses binary abf file
"""
print('parse_contents_abf() filename:', filename)
content_type, content_string = contents.split(',')
#print(' content_type:', content_type)
decoded = base64.b64decode(content_string)
#print(' type(decoded)', type(decoded))
fileLikeObject = io.BytesIO(decoded)
#print(' type(fileLikeObject)', type(fileLikeObject))
#print(len(fileLikeObject.getvalue()))
#major = pyabf.abfHeader.abfFileFormat(fileLikeObject)
#print(' pyabf major:', major)
print(' instantiating sanpy.bAnalysis with byte stream')
try:
global ba
ba = sanpy.bAnalysis(byteStream=fileLikeObject)
except:
print('*** exception in parse_contents_abf():')
print(sys.exc_info())
'''
return html.Div([
html.H6(f'Loaded file: {filename}'),
])
'''
'''
return html.Div([
html.Hr(),
html.H6(f'Loaded file: {filename}'),
])
'''
return html.Label(f'Loaded file: {filename}')
def test_save_load():
# load an abf
path = '/home/cudmore/Sites/SanPy/data/19114001.abf'
ba = sanpy.bAnalysis(path)
# analyze
ba.spikeDetect()
print(ba)
# save
parentPath, fileName = os.path.split(path)
saveFolder = os.path.join(parentPath, 'sanpy_analysis')
if not os.path.isdir(saveFolder):
logger.info(f'making folder: {saveFolder}')
os.mkdir(saveFolder)
baseName = os.path.splitext(fileName)[0]
saveFile = baseName + '_detection.json'
savePath = os.path.join(saveFolder, saveFile)
logger.info(f'savePath:{savePath}')
detectionClass = ba.detectionClass
# convert no jsonable classes to str representation
defaultValue = detectionClass._dDict['detectionType']['defaultValue']
detectionClass._dDict['detectionType']['defaultValue'] = defaultValue.name
currentValue = detectionClass._dDict['detectionType']['currentValue']
detectionClass._dDict['detectionType']['currentValue'] = currentValue.name
#detectionClass.print()
detectionClass.save(savePath)
detectionClass._dDict = None
detectionClass.load(savePath)
detectionClass.print()
def old_setFile(self, filePath, plotRaw=False):
"""
when main application changes file
"""
if not os.path.isfile(filePath):
print('bExportWidget.setFile did not find path:', filePath)
return False
self.filePath = filePath
self.ba = bAnalysis(filePath)
if self.ba.loadError:
print('there was an error loading file', filePath)
return False
self.mySweepX = self.ba.abf.sweepX
self.mySweepY = self.ba.abf.sweepY
self.mySweepX_Downsample = self.ba.abf.sweepX
self.mySweepY_Downsample = self.ba.abf.sweepY
if plotRaw:
self.plotRaw()
return True
self.figure.savefig(fullSavePath)
def center(self):
"""
Center the window on the screen
"""
qr = self.frameGeometry()
cp = QtWidgets.QDesktopWidget().availableGeometry().center()
qr.moveCenter(cp)
self.move(qr.topLeft())
if __name__ == '__main__':
path = '../data/19114001.abf'
ba = bAnalysis(path)
if ba.loadError:
logger.error(f'Error loading file: {path}')
else:
app = QtWidgets.QApplication(sys.argv)
app.aboutToQuit.connect(app.deleteLater)
sweepX = ba.sweepX
sweepY = ba.sweepY
xyUnits = ('Time (sec)', 'Vm (mV)')
type = 'vm'
GUI = bExportWidget(sweepX,
sweepY,
path=path,
xyUnits=xyUnits,
type=type)
app.selectSpikeList(spikeList, doZoom=doZoom)
def test_print_classes():
"""testing"""
print('__name__:', __name__)
for name, obj in inspect.getmembers(sys.modules[__name__]):
if inspect.isclass(obj):
print(name, ':', obj)
print('===')
for name, obj in inspect.getmembers(sanpy.interface.plugins):
if inspect.isclass(obj):
print(name, ':', obj)
if __name__ == '__main__':
#print_classes()
#sys.exit(1)
bp = bPlugins()
pluginList = bp.pluginList()
print('pluginList:', pluginList)
abfPath = '/Users/cudmore/Sites/SanPy/data/19114001.abf'
ba = sanpy.bAnalysis(abfPath)
bp.runPlugin('plotRecording', ba)
#bp.runPlugin('plotRecording3', ba)
def hoverEvent(self, event):
if event.isExit():
return
xPos = event.pos().x()
yPos = event.pos().y()
xPos = int(xPos)
yPos = int(yPos)
myTif = self.ba.tifData
intensity = myTif[yPos, xPos] # flipped
#logger.info(f'x:{xPos} y:{yPos} intensity:{intensity}')
self.tifCursorLabel.setText(f'Cursor:{intensity}')
self.tifCursorLabel.update()
if __name__ == '__main__':
path = '/media/cudmore/data/rabbit-ca-transient/jan-12-2022/Control/220110n_0003.tif.frames/220110n_0003.tif'
ba = sanpy.bAnalysis(path)
print(ba)
app = QtWidgets.QApplication(sys.argv)
kw = kymWidget(ba)
kw.show()
sys.exit(app.exec_())
def __init__(self, index, path, fromDict=None):
"""
path: (str) full path to .mp4 video file
fromDict: (dict) construct from database dict
"""
if not os.path.isfile(path):
print('error: bVideoFile() could not open file path:', path)
return
self.path = path
videoFileName = os.path.basename(path)
# abb 20201009
#self.dict = OrderedDict()
self.dict = self._getDefaultDict()
self.dict['file'] = videoFileName
self.loadError = False # abb 202012
# load abf file and grab parameters
if fromDict is None:
#
loadWasGood = True
'''
try:
ba = sanpy.bAnalysis(file=path) # load file as abf file (may fail)
except (NotImplementedError) as e:
print('bFileList.bVideoFile.__init__() exception, did not load file:', path)
loadWasGood = False
'''
ba = sanpy.bAnalysis(file=path) # load file as abf file (may fail)
if ba.loadError:
print('bFileList.bVideoFile.__init__() exception, did not load file:', path)
loadWasGood = False
self.loadError = True
else:
#
pntsPerMS = ba.dataPointsPerMs
numSweeps = len(ba.sweepList)
durationSec = max(ba.abf.sweepX)
acqDate = ba.acqDate
acqTime = ba.acqTime
##
##
## THIS IS PROBABLY AN ERROR
##
##
dvdtThreshold = None
minSpikeVm = None # abb 202012
numSpikes = None
analysisDate = None
analysisTime = None
else:
# loading from a saved _db.json file ???
pntsPerMS = fromDict['kHz']
numSweeps = fromDict['numSweeps']
durationSec = fromDict['durationSec']
acqDate = fromDict['acqDate']
acqTime = fromDict['acqTime']
dvdtThreshold = fromDict['dvdtThreshold']
minSpikeVm = fromDict['minSpikeVm'] # abb 202012
numSpikes = fromDict['numSpikes']
analysisDate = fromDict['analysisDate']
analysisTime = fromDict['analysisTime']
if loadWasGood:
self.dict['kHz'] = pntsPerMS
self.dict['numSweeps'] = numSweeps
self.dict['durationSec'] = int(round(durationSec))
self.dict['acqDate'] = acqDate
self.dict['acqTime'] = acqTime
self.dict['dvdtThreshold'] = dvdtThreshold
self.dict['minSpikeVm'] = minSpikeVm # abb 202012
self.dict['numSpikes'] = numSpikes
self.dict['analysisDate'] = analysisDate
self.dict['analysisTime'] = analysisTime
def old_test_requests():
"""
this gets all files, including
https://api.github.com/repos/cudmore/SanPy/git/trees/master?recursive=1
{
"path": "data",
"mode": "040000",
"type": "tree",
"sha": "8b97ef351ea95308b524b6febb2890f000b86388",
"url": "https://api.github.com/repos/cudmore/SanPy/git/trees/8b97ef351ea95308b524b6febb2890f000b86388"
},
{
"path": "data/171116sh_0018.abf",
"mode": "100644",
"type": "blob",
"sha": "5f3322b08d86458bf7ac8b5c12564933142ffd17",
"size": 2047488,
"url": "https://api.github.com/repos/cudmore/SanPy/git/blobs/5f3322b08d86458bf7ac8b5c12564933142ffd17"
},
Then this url:
https://api.github.com/repos/cudmore/SanPy/git/blobs/5f3322b08d86458bf7ac8b5c12564933142ffd17
returns a dict d{} with
{
"sha": "5f3322b08d86458bf7ac8b5c12564933142ffd17",
"node_id": "MDQ6QmxvYjE3MTA2NDA5Nzo1ZjMzMjJiMDhkODY0NThiZjdhYzhiNWMxMjU2NDkzMzE0MmZmZDE3",
"size": 2047488,
"url": "https://api.github.com/repos/cudmore/SanPy/git/blobs/5f3322b08d86458bf7ac8b5c12564933142ffd17",
"coontent": "<CONTENTS>"
"encoding": "base64"
}
https://api.github.com/repos/:owner/:repo_name/contents/:path
"""
import requests
import io
# this works
'''
url = "https://github.com/cudmore/SanPy/blob/master/data/19114001.abf?raw=true"
# Make sure the url is the raw version of the file on GitHub
download = requests.get(url).content
'''
owner = 'cudmore'
repo_name = 'SanPy'
path = 'data'
url = f'https://api.github.com/repos/{owner}/{repo_name}/contents/{path}'
response = requests.get(url).json()
print('response:', type(response))
#print(response.json())
for idx, item in enumerate(response):
if not item['name'].endswith('.abf'):
continue
print(idx)
# use item['git_url']
for k, v in item.items():
print(' ', k, ':', v)
#
# grab the first file
#gitURl = response[0]['git_url']
'''
print(' === gitURL:', gitURL)
#download = requests.get(gitURl).content
downloadRespoonse = requests.get(gitURL).json()
print(' downloadRespoonse:', type(downloadRespoonse))
content = downloadRespoonse['content']
#print(' ', downloadRespoonse)
#decoded = download.decode('utf-8')
#print(' decoded:', type(decoded))
'''
# use response[0]['download_url'] to directly download file
#gitURL = 'https://raw.githubusercontent.com/cudmore/SanPy/master/data/SAN-AP-example-Rs-change.abf'
download_url = response[1]['download_url']
content = requests.get(download_url).content
#import base64
#myBase64 = base64.b64encode(bytes(content, 'utf-8'))
#myBase64 = base64.b64encode(bytes(content, 'base64'))
'''
myBase64 = base64.b64encode(bytes(content, 'utf-8'))
print('myBase64:', type(myBase64))
'''
#decoded = content.decode('utf-8')
#print(download)
#import pyabf
fileLikeObject = io.BytesIO(content)
ba = sanpy.bAnalysis(byteStream=fileLikeObject)
print(ba._abf)
print(ba.api_getHeader())
def loadFromDb(path):
"""
load from SanPy folder DB
TODO: This is managing 2x lists, one from db and other from abf/tif files in folder.
path (str): folder
"""
baList = []
dbPath = os.path.join(path, 'sanpy_recording_db.csv')
dfFolder = pd.read_csv(dbPath)
fileList = getFileList(path)
baList = []
for idx, filePath in enumerate(fileList):
# debug
if 0:
if idx == 4:
break
parent = os.path.split(filePath)[
0] # corresponds to Olympus export folder
grandparent = os.path.split(parent)[0]
condition = os.path.split(grandparent)[1]
fileName = os.path.split(filePath)[1]
ba = sanpy.bAnalysis(filePath)
oneFile = dfFolder[dfFolder['File'] == fileName]
mvThreshold = oneFile['mvThreshold'].values[0]
kLeft = oneFile['kLeft'].values[0]
kTop = oneFile['kTop'].values[0]
kRight = oneFile['kRight'].values[0]
kBottom = oneFile['kBottom'].values[0]
theRect = [kLeft, kTop, kRight, kBottom]
#print(oneFile)
print(idx, fileName, mvThreshold, theRect)
ba._updateTifRoi(theRect)
detectionClass = sanpy.bDetection() # gets default detection class
detectionType = sanpy.bDetection.detectionTypes.mv
detectionClass['condition'] = condition
detectionClass[
'detectionType'] = detectionType # set detection type to ('dvdt', 'vm')
detectionClass['dvdtThreshold'] = math.nan
detectionClass['mvThreshold'] = mvThreshold
detectionClass['peakWindow_ms'] = 500
detectionClass['preSpikeClipWidth_ms'] = 100
detectionClass['postSpikeClipWidth_ms'] = 1000
detectionClass['refractory_ms'] = 700
if fileName == '220110n_0032.tif':
# default is 170 ms
print("xxx", fileName, "detectionClass['refractory_ms']:",
detectionClass['refractory_ms'])
detectionClass['refractory_ms'] = 1500
# todo: set detection refractory_ms
# detect
ba.spikeDetect(detectionClass=detectionClass)
#
# convert each ba sweepY to nM
#ba._sweepY[:,0] = convertTomM(ba._sweepY[:,0]) # assuming one sweep
# does not work because oneDf is constructed on fly
# append some columns for nano-molar
#oneDf = ba.asDataFrame()
#peakVal = oneDf['peakVal']
#peak_nM = convertTomM(peakVal)
#oneDf['peak_nM'] = peak_nM
baList.append(ba)
#
return baList
def test_fft():
if 0:
# abf data
path = '/Users/cudmore/Sites/SanPy/data/fft/2020_07_07_0000.abf'
ba = sanpy.bAnalysis(path)
x = ba.sweepX()
y = ba.sweepY()
# 20-30 sec
startSec = 16.968 # good signal
stopSec = 31.313
# reduce to get fft with N=1024 in excel
dataPointsPerMs = ba.dataPointsPerMs # 10 for 10 kHz
startPnt = round(startSec * 1000 * dataPointsPerMs)
stopPnt = round(stopSec * 1000 * dataPointsPerMs)
numPnts = stopPnt-startPnt
print(f'N={numPnts}')
t = x[startPnt:stopPnt]
y = y[startPnt:stopPnt]
#y -= np.nanmean(y)
dt = 0.0001
fs = 1/dt
NFFT = 512 # 2**16 #512 * 100 # The number of data points used in each block for the FFT
medianPnts = 50 # 5 ms
fileName = ba.getFileName()
if 1:
# sin wave data
# Fixing random state for reproducibility
np.random.seed(19680801)
durSec = 10.24 # to get 1024 points at dt=0.01 (power of two)
dt = 0.01
fs = 1/dt
nfft = 512 # The number of data points used in each block for the FFT
medianPnts = 5 #
t = np.arange(0, durSec, dt)
nse = np.random.randn(len(t))
r = np.exp(-t / 0.05)
cnse = np.convolve(nse, r) * dt
cnse = cnse[:len(t)]
secondFre = 7
y = 0.1 * np.sin(2 * np.pi * t) + cnse
y += 0.1 * np.sin(secondFre * 2 * np.pi * t) + cnse
fileName = 'fakeSin'
#def replot_fft():
#
# filter
yFiltered = scipy.ndimage.median_filter(y, medianPnts)
# subsample down to 1024
'''
from scipy.interpolate import interp1d
t2 = interp1d(np.arange(1024), t, 'linear')
yFiltered2 = interp1d(t, yFiltered, 'linear')
print(f'N2={len(t2)}')
plt.plot(t2, yFiltered2)
'''
# save [t,y] to csv
'''
import pandas as pd
tmpDf = pd.DataFrame(columns=['t', 'y'])
tmpDf['t'] = t
tmpDf['y'] = y
csvPath = fileName + '.csv'
print('saving csv:', csvPath)
tmpDf.to_csv(csvPath, index=False)
'''
'''
cutOff = 10 #20 # 20, cutOff of filter (Hz)
order = 50 # 40 # order of filter
sos = butter_lowpass_sos(cutOff, fs, order)
'''
#yFiltered = scipy.signal.sosfilt(sos, yFiltered)
#
#Fs = 1/dt # The sampling frequency (samples per time unit)
#NFFT = 512 # The number of data points used in each block for the FFT
# plot
fig, (ax0, ax1, ax2) = plt.subplots(3, 1)
ax0.plot(t, y, 'k')
ax0.plot(t, yFiltered, 'r')
def myDetrend(x):
y = plt.mlab.detrend_linear(x)
y = plt.mlab.detrend_mean(y)
return y
# The power spectral density ЁЭСГЁЭСеЁЭСе by Welch's average periodogram method
print(' fs:', fs, 'nfft:', nfft)
ax1.clear()
Pxx, freqs = ax1.psd(yFiltered, NFFT=nfft, Fs=fs, detrend=myDetrend)
ax1.set_xlim([0, 20])
ax1.set_ylabel('PSD (dB/Hz)')
#ax1.callbacks.connect('xlim_changed', self.on_xlims_change)
'''
ax1.clear()
ax1.plot(freqs, Pxx)
ax1.set_xlim([0, 20])
'''
'''
# recompute the ax.dataLim
ax1.relim()
# update ax.viewLim using the new dataLim
ax1.autoscale_view()
plt.draw()
'''
'''
ax2.plot(freqs, Pxx)
ax2.set_xlim([0, 20])
'''
maxPsd = np.nanmax(Pxx)
maxPnt = np.argmax(Pxx)
print(f'Max PSD freq is {freqs[maxPnt]} with power {maxPsd}')
scipy_f, scipy_Pxx = scipy.signal.periodogram(yFiltered, fs)
ax2.plot(scipy_f[1:-1], scipy_Pxx[1:-1]) # drop freq 0
ax2.set_xlim([0, 20])
ax2.set_xlabel('Frequency (Hz)')
ax2.set_ylabel('scipy_Pxx')
#
plt.show()
def getFileList(path, bucketName=None):
"""
Get list of bAnalysis from path
Returns:
list of bAnalysis
"""
logger.info(f'path: {path} bucketName: {bucketName}')
baList = []
retFileList = []
useExtension = '.abf'
videoFileIdx = 0
fileDict = {}
fileDict['Type'] = 'file'
fileDict['File Name'] = ''
#fileDict['path'] = ''
fileDict['kHz'] = ''
fileDict['Dur(s)'] = ''
fileDict['Sweeps'] = ''
error = False
doAws = bucketName is not None
fileList = []
if doAws:
#bucket = 'sanpy-data'
s3 = sanpy.awsUtil.getConnection()
fileList = sanpy.awsUtil.fetchFileList(bucketName, folder='.', s3=s3)
else:
if not os.path.isdir(path):
# ERROR
error = True
else:
fileList = os.listdir(path)
if not error:
for file in fileList:
if file.startswith('.'):
continue
if file.endswith(useExtension):
if doAws:
pass
else:
fullPath = os.path.join(path, file)
fileDict = {} # WOW, I need this here !!!!!!!!
fileDict['Type'] = 'file'
fileDict['File Name'] = file
#fileDict['path'] = fullPath
if doAws:
ba = sanpy.awsUtil.loadOneFile(bucketName, file, s3=s3)
else:
ba = sanpy.bAnalysis(file=fullPath)
baList.append(ba)
'''
if videoFileIdx == 0:
print(ba.abf.headerText)
sweepUnitsC # what we are clamping (mV, pA)
sweepUnitsX
sweepUnitsY
'''
# TODO: get this from bAnalysis header
baHeader = ba.api_getHeader()
recording_kHz = baHeader['recording_kHz'] #ba.dataPointsPerMs
numSweeps = len(ba.sweepList)
recordingDur_sec = baHeader[
'recordingDur_sec'] #max(ba.abf.sweepX)
fileDict['kHz'] = recording_kHz
fileDict['Dur(s)'] = round(recordingDur_sec, 3)
fileDict['Sweeps'] = numSweeps
retFileList.append(fileDict)
videoFileIdx += 1
#
if len(retFileList) == 0:
retFileList.append(fileDict)
df = pd.DataFrame(retFileList)
return df, baList
def old_load(path):
fileList = getFileList(path)
# load and analyze, some files are tweaked for thir analysis
baList = []
for idx, file in enumerate(fileList):
# debug
if 1:
if idx == 4:
break
ba = sanpy.bAnalysis(file)
#detectionClass = sanpy.bDetection() # gets default detection class
# get condition from grandparent folder in ['Thapsigargin', 'Control']
# file path looks like this:
# /media/cudmore/data/rabbit-ca-transient/Control/220110n_0049.tif.frames/220110n_0049.tif
parent = os.path.split(file)[0] # corresponds to Olympus export folder
grandparent = os.path.split(parent)[0]
condition = os.path.split(grandparent)[1]
#print(condition, file)
if condition == 'Control':
mvThreshold = 1.4
elif condition == 'Thapsigargin':
mvThreshold = 1.1
else:
print(f'XXX ERROR: Case not taken for condition: "{condition}"')
detectionClass = sanpy.bDetection() # gets default detection class
# special cases
fileName = os.path.split(file)[1]
if fileName == '220110n_0017.tif':
mvThreshold = 1.2
elif fileName == '220110n_0021.tif':
mvThreshold = 1.2
elif fileName == '220110n_0023.tif':
mvThreshold = 1.2
elif fileName == '220110n_0024.tif':
mvThreshold = 1.2
elif fileName == '220110n_0032.tif':
# default is 170 ms
print("xxx", fileName, "detectionClass['refractory_ms']:",
detectionClass['refractory_ms'])
detectionClass['refractory_ms'] = 1500
# todo: set detection refractory_ms
elif fileName == '220110n_0055.tif':
mvThreshold = 1.05
detectionType = sanpy.bDetection.detectionTypes.mv
detectionClass['condition'] = condition
detectionClass[
'detectionType'] = detectionType # set detection type to ('dvdt', 'vm')
detectionClass['dvdtThreshold'] = math.nan
detectionClass['mvThreshold'] = mvThreshold
detectionClass['peakWindow_ms'] = 500
ba.spikeDetect(detectionClass=detectionClass)
baList.append(ba)
return baList
