def rmsmap(fbin, spectra=True):
"""
Computes RMS map in time domain and spectra for each channel of Neuropixel probe
:param fbin: binary file in spike glx format (will look for attached metatdata)
:type fbin: str or pathlib.Path
:param spectra: whether to compute the power spectrum (only need for lfp data)
:type: bool
:return: a dictionary with amplitudes in channeltime space, channelfrequency space, time
and frequency scales
"""
if not isinstance(fbin, spikeglx.Reader):
sglx = spikeglx.Reader(fbin)
rms_win_length_samples = 2**np.ceil(np.log2(sglx.fs * RMS_WIN_LENGTH_SECS))
# the window generator will generates window indices
wingen = dsp.WindowGenerator(ns=sglx.ns,
nswin=rms_win_length_samples,
overlap=0)
# pre-allocate output dictionary of numpy arrays
win = {
'TRMS': np.zeros((wingen.nwin, sglx.nc)),
'nsamples': np.zeros((wingen.nwin, )),
'fscale': dsp.fscale(WELCH_WIN_LENGTH_SAMPLES,
1 / sglx.fs,
one_sided=True),
'tscale': wingen.tscale(fs=sglx.fs)
}
win['spectral_density'] = np.zeros((len(win['fscale']), sglx.nc))
# loop through the whole session
for first, last in wingen.firstlast:
D = sglx.read_samples(first_sample=first,
last_sample=last)[0].transpose()
# remove low frequency noise below 1 Hz
D = dsp.hp(D, 1 / sglx.fs, [0, 1])
iw = wingen.iw
win['TRMS'][iw, :] = dsp.rms(D)
win['nsamples'][iw] = D.shape[1]
if spectra:
# the last window may be smaller than what is needed for welch
if last - first < WELCH_WIN_LENGTH_SAMPLES:
continue
# compute a smoothed spectrum using welch method
_, w = signal.welch(D,
fs=sglx.fs,
window='hanning',
nperseg=WELCH_WIN_LENGTH_SAMPLES,
detrend='constant',
return_onesided=True,
scaling='density',
axis=-1)
win['spectral_density'] += w.T
# print at least every 20 windows
if (iw % min(20, max(int(np.floor(wingen.nwin / 75)), 1))) == 0:
print_progress(iw, wingen.nwin)
return win
def http_download_file(full_link_to_file,
*,
clobber=False,
offline=False,
username='',
password='',
cache_dir=''):
"""
:param full_link_to_file: http link to the file.
:type full_link_to_file: str
:param clobber: [False] If True, force overwrite the existing file.
:type clobber: bool
:param username: [''] authentication for password protected file server.
:type username: str
:param password: [''] authentication for password protected file server.
:type password: str
:param cache_dir: [''] directory in which files are cached; defaults to user's
Download directory.
:type cache_dir: str
:return: (str) a list of the local full path of the downloaded files.
"""
if not full_link_to_file:
return ''
# default cache directory is the home dir
if not cache_dir:
cache_dir = str(Path.home().joinpath("Downloads"))
# This is the local file name
file_name = cache_dir + os.sep + os.path.basename(full_link_to_file)
# do not overwrite an existing file unless specified
if not clobber and os.path.exists(file_name):
return file_name
elif offline:
return file_name
# This should be the base url you wanted to access.
baseurl = os.path.split(str(full_link_to_file))[0]
# Create a password manager
manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
if (len(password) != 0) & (len(username) != 0):
manager.add_password(None, baseurl, username, password)
# Create an authentication handler using the password manager
auth = urllib.request.HTTPBasicAuthHandler(manager)
# Create an opener that will replace the default urlopen method on further calls
opener = urllib.request.build_opener(auth)
urllib.request.install_opener(opener)
# Open the url and get the length
u = urllib.request.urlopen(full_link_to_file)
file_size = int(u.getheader('Content-length'))
print(f"Downloading: {file_name} Bytes: {file_size}")
file_size_dl = 0
block_sz = 8192 * 64 * 8
f = open(file_name, 'wb')
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
print_progress(file_size_dl, file_size, prefix='', suffix='')
f.close()
return file_name
def http_download_file(full_link_to_file,
chunks=None,
*,
clobber=False,
username='',
password='',
cache_dir='',
return_md5=False,
headers=None):
"""
:param full_link_to_file: http link to the file.
:type full_link_to_file: str
:param clobber: [False] If True, force overwrite the existing file.
:type clobber: bool
:param username: [''] authentication for password protected file server.
:type username: str
:param password: [''] authentication for password protected file server.
:type password: str
:param cache_dir: [''] directory in which files are cached; defaults to user's
Download directory.
:param: headers: [{}] additional headers to add to the request (auth tokens etc..)
:type cache_dir: str
:return: (str) a list of the local full path of the downloaded files.
"""
from ibllib.io import hashfile
if not full_link_to_file:
return ''
# default cache directory is the home dir
if not cache_dir:
cache_dir = str(Path.home().joinpath("Downloads"))
# This is the local file name
file_name = str(cache_dir) + os.sep + os.path.basename(full_link_to_file)
# do not overwrite an existing file unless specified
if not clobber and os.path.exists(file_name):
return (file_name,
hashfile.md5(file_name)) if return_md5 else file_name
# This should be the base url you wanted to access.
baseurl = os.path.split(str(full_link_to_file))[0]
# Create a password manager
manager = urllib.request.HTTPPasswordMgrWithDefaultRealm()
if (len(password) != 0) & (len(username) != 0):
manager.add_password(None, baseurl, username, password)
# Create an authentication handler using the password manager
auth = urllib.request.HTTPBasicAuthHandler(manager)
# Create an opener that will replace the default urlopen method on further calls
opener = urllib.request.build_opener(auth)
urllib.request.install_opener(opener)
# Support for partial download.
req = urllib.request.Request(full_link_to_file)
if chunks is not None:
first_byte, n_bytes = chunks
req.add_header("Range",
"bytes=%d-%d" % (first_byte, first_byte + n_bytes - 1))
# add additional headers
if headers is not None:
for k in headers:
req.add_header(k, headers[k])
# Open the url and get the length
try:
u = urllib.request.urlopen(req)
except urllib.error.HTTPError as e:
_logger.error(f"{str(e)} {full_link_to_file}")
raise e
file_size = int(u.getheader('Content-length'))
print(f"Downloading: {file_name} Bytes: {file_size}")
file_size_dl = 0
block_sz = 8192 * 64 * 8
md5 = hashlib.md5()
f = open(file_name, 'wb')
while True:
buffer = u.read(block_sz)
if not buffer:
break
file_size_dl += len(buffer)
f.write(buffer)
if return_md5:
md5.update(buffer)
print_progress(file_size_dl, file_size, prefix='', suffix='')
f.close()
return (file_name, md5.hexdigest()) if return_md5 else file_name
def test_simple_print(self):
print('waitbar')
for p in range(10):
time.sleep(0.05)
print_progress(p, 9)
def populate_dj_with_phy(probe_label,
eid=None,
subj=None,
date=None,
sess_no=None,
one=None):
if one is None:
one = ONE()
if eid is None:
eid = one.search(subject=subj, date=date, number=sess_no)[0]
sess_path = one.path_from_eid(eid)
alf_path = sess_path.joinpath('alf', probe_label)
cluster_path = Path(alf_path, 'spikes.clusters.npy')
template_path = Path(alf_path, 'spikes.templates.npy')
# Compare spikes.clusters with spikes.templates to find which clusters have been merged
phy_clusters = np.load(cluster_path)
id_phy = np.unique(phy_clusters)
orig_clusters = np.load(template_path)
id_orig = np.unique(orig_clusters)
uuid_list = alf.io.load_file_content(
alf_path.joinpath('clusters.uuids.csv'))
# First deal with merged clusters and make sure they have cluster uuids assigned
# Find the original cluster ids that have been merged into a new cluster
merged_idx = np.setdiff1d(id_orig, id_phy)
# See if any clusters have been merged, if not skip to the next bit
if np.any(merged_idx):
# Make association between original cluster and new cluster id and save in dict
merge_list = {}
for m in merged_idx:
idx = phy_clusters[np.where(orig_clusters == m)[0][0]]
if idx in merge_list:
merge_list[idx].append(m)
else:
merge_list[idx] = [m]
# Create a dataframe from the dict
merge_clust = pd.DataFrame(
columns={'cluster_idx', 'merged_uuid', 'merged_id'})
for key, value in merge_list.items():
value_uuid = uuid_list['uuids'][value]
merge_clust = merge_clust.append(
{
'cluster_idx': key,
'merged_uuid': tuple(value_uuid),
'merged_idx': tuple(value)
},
ignore_index=True)
# Get the dj table that has previously stored merged clusters and store in frame
merge = cluster_table.MergedClusters()
merge_dj = pd.DataFrame(columns={'cluster_uuid', 'merged_uuid'})
merge_dj['cluster_uuid'] = merge.fetch('cluster_uuid').astype(str)
merge_dj['merged_uuid'] = tuple(map(tuple, merge.fetch('merged_uuid')))
# Merge the two dataframe to see if any merge combinations already have a cluster_uuid
merge_comb = pd.merge(merge_dj,
merge_clust,
on=['merged_uuid'],
how='outer')
# Find the merged clusters that do not have a uuid assigned
no_uuid = np.where(pd.isnull(merge_comb['cluster_uuid']))[0]
# Assign new uuid to new merge pairs and add to the merge table
for nid in no_uuid:
new_uuid = str(uuid.uuid4())
merge_comb['cluster_uuid'].iloc[nid] = new_uuid
merge.insert1(dict(
cluster_uuid=new_uuid,
merged_uuid=merge_comb['merged_uuid'].iloc[nid]),
allow_direct_insert=True)
# Add all the uuids to the cluster_uuid frame with index according to cluster id from phy
for idx, c_uuid in zip(merge_comb['cluster_idx'].values,
merge_comb['cluster_uuid'].values):
uuid_list.loc[idx] = c_uuid
csv_path = Path(alf_path, 'merge_info.csv')
merge_comb = merge_comb.reindex(columns=[
'cluster_idx', 'cluster_uuid', 'merged_idx', 'merged_uuid'
])
try:
merge_comb.to_csv(csv_path, index=False)
except Exception as err:
print(err)
print('Close merge_info.csv file and then relaunch script')
sys.exit(1)
else:
print('No merges detected, continuing...')
# Now populate datajoint with cluster labels
user = one._par.ALYX_LOGIN
current_date = datetime.now().replace(microsecond=0)
try:
cluster_group = alf.io.load_file_content(
alf_path.joinpath('cluster_group.tsv'))
except Exception as err:
print(err)
print('Could not find cluster group file output from phy')
sys.exit(1)
try:
cluster_notes = alf.io.load_file_content(
alf_path.joinpath('cluster_notes.tsv'))
cluster_info = pd.merge(cluster_group,
cluster_notes,
on=['cluster_id'],
how='outer')
except Exception as err:
cluster_info = cluster_group
cluster_info['notes'] = None
cluster_info = cluster_info.where(cluster_info.notnull(), None)
cluster_info['cluster_uuid'] = uuid_list['uuids'][
cluster_info['cluster_id']].values
# dj table that holds data
cluster = cluster_table.ClusterLabel()
# Find clusters that have already been labelled by user
old_clust = cluster & cluster_info & {'user_name': user}
dj_clust = pd.DataFrame()
dj_clust['cluster_uuid'] = (old_clust.fetch('cluster_uuid')).astype(str)
dj_clust['cluster_label'] = old_clust.fetch('cluster_label')
# First find the new clusters to insert into datajoint
idx_new = np.where(
np.isin(cluster_info['cluster_uuid'],
dj_clust['cluster_uuid'],
invert=True))[0]
cluster_uuid = cluster_info['cluster_uuid'][idx_new].values
cluster_label = cluster_info['group'][idx_new].values
cluster_note = cluster_info['notes'][idx_new].values
if idx_new.size != 0:
print('Populating dj with ' + str(idx_new.size) + ' new labels')
else:
print('No new labels to add')
for iIter, (iClust, iLabel, iNote) in enumerate(
zip(cluster_uuid, cluster_label, cluster_note)):
cluster.insert1(dict(cluster_uuid=iClust,
user_name=user,
label_time=current_date,
cluster_label=iLabel,
cluster_note=iNote),
allow_direct_insert=True)
print_progress(iIter, cluster_uuid.size, '', '')
# Next look through clusters already on datajoint and check if any labels have
# been changed
comp_clust = pd.merge(cluster_info, dj_clust, on='cluster_uuid')
idx_change = np.where(
comp_clust['group'] != comp_clust['cluster_label'])[0]
cluster_uuid = comp_clust['cluster_uuid'][idx_change].values
cluster_label = comp_clust['group'][idx_change].values
cluster_note = comp_clust['notes'][idx_change].values
# Populate table
if idx_change.size != 0:
print('Replacing label of ' + str(idx_change.size) + ' clusters')
else:
print('No labels to change')
for iIter, (iClust, iLabel, iNote) in enumerate(
zip(cluster_uuid, cluster_label, cluster_note)):
prev_clust = cluster & {'user_name': user} & {'cluster_uuid': iClust}
cluster.insert1(dict(*prev_clust.proj(),
label_time=current_date,
cluster_label=iLabel,
cluster_note=iNote),
allow_direct_insert=True,
replace=True)
print_progress(iIter, cluster_uuid.size, '', '')
print('Upload to datajoint complete')
def print_progress(self):
"""
Prints progress using a terminal progress bar
"""
print_progress(self.iw, self.nwin)
