def test_overlapping_name():
test_schema = dj.schema(PREFIX + '_overlapping_schema', connection=dj.conn(**CONN_INFO))
@test_schema
class Unit(dj.Manual):
definition = """
id: int # simple id
"""
# hack to update the locals dictionary
locals()
@test_schema
class Cell(dj.Manual):
definition = """
type: varchar(32) # type of cell
"""
class Unit(dj.Part):
definition = """
-> master
-> Unit
"""
test_schema.drop()
def test_failure_to_create_table(self):
unprivileged = dj.schema(schema.schema.database, namespace, connection=self.connection)
@unprivileged
class Try(dj.Manual):
definition = """ # should not matter really
id : int
---
value : float
"""
Try().insert1((1, 1.5))
def test_part():
# Lookup and part with the same name. See issue #365
local_schema = dj.schema(schema.database)
@local_schema
class Type(dj.Lookup):
definition = """
type : varchar(255)
"""
contents = zip(('Type1', 'Type2', 'Type3'))
@local_schema
class TypeMaster(dj.Manual):
definition = """
master_id : int
"""
class Type(dj.Part):
definition = """
import datajoint as dj
from . import PREFIX, CONN_INFO
schema = dj.schema(PREFIX + '_advanced', locals(), connection=dj.conn(**CONN_INFO))
@schema
class Person(dj.Manual):
definition = """
person_id : int
----
full_name : varchar(60)
sex : enum('M','F')
"""
def fill(self):
"""
fill fake names from www.fakenamegenerator.com
"""
self.insert((
(0, "May K. Hall", "F"),
(1, "Jeffrey E. Gillen", "M"),
(2, "Hanna R. Walters", "F"),
(3, "Russel S. James", "M"),
(4, "Robbin J. Fletcher", "F"),
(5, "Wade J. Sullivan", "M"),
(6, "Dorothy J. Chen", "F"),
(7, "Michael L. Kowalewski", "M"),
(8, "Kimberly J. Stringer", "F"),
(9, "Mark G. Hair", "M"),
(10, "Mary R. Thompson", "F"),
from itertools import count
import datajoint as dj
from scipy.interpolate import InterpolatedUnivariateSpline
from .preprocess import Sync, fill_nans, SpikeMethod, Prepare, BehaviorSync, ExtractRaw, ManualSegment, Method
from .preprocess import Spikes as PreSpikes
from . import preprocess, vis
import numpy as np
import pandas as pd
from scipy import integrate
schema = dj.schema('pipeline_quality', locals())
def integration_factory(frame_times, trace):
def responses(bins):
if not hasattr(responses, 'spline'):
responses.spline = InterpolatedUnivariateSpline(frame_times, trace, k=1, ext=1)
spline = responses.spline
ret = np.zeros(len(bins) - 1)
for j, (a, b) in enumerate(zip(bins[:-1], bins[1:])):
ret[j] = integrate.quad(spline, a, b)[0]
return ret
return responses
@schema
class Spikes(dj.Computed):
definition = """
"""
This schema copies recent data from common_psy for uploaded to the cloud
"""
import datajoint as dj
from . import mice # needed for referencing
import numpy as np
schema = dj.schema('pipeline_vis', locals())
schema.spawn_missing_classes()
@schema
class Session(dj.Manual):
definition = """ # Visual stimulus session, populated by the stimulus program.
-> mice.Mice
psy_id : smallint unsigned # unique psy session number
---
stimulus="grating" : varchar(255) # experiment type
monitor_distance : float # (cm) eye-to-monitor distance
monitor_size=19 : float # (inches) size diagonal dimension
monitor_aspect=1.25 : float # physical aspect ratio of monitor
resolution_x=1280 : smallint # (pixels)
resolution_y=1024 : smallint # display resolution along y
psy_ts=CURRENT_TIMESTAMP : timestamp # automatic
"""
@schema
class Condition(dj.Manual):
definition = """ # trial condition -- one condition per trial. All stimulus conditions refer to Condition.
import pymysql
import seaborn as sns
import sympy
from scipy import optimize, stats, signal
import datajoint as dj
from scipy.interpolate import interp1d, InterpolatedUnivariateSpline
import pycircstat as circ
from datajoint import schema
from locking.data import Runs, GlobalEFieldPeaksTroughs, peakdet, Cells, LocalEODPeaksTroughs, Baseline, \
GlobalEODPeaksTroughs, BaseEOD
from pycircstat import event_series as es
schema = schema('efish_analyses', locals())
def vector_strength_at(f, trial, alpha=None):
if alpha is None:
return 1 - circ.var((trial % (1. / f)) * f * 2 * np.pi)
else:
return 1 - circ.var((trial % (1. / f)) * f * 2 * np.pi), np.sqrt(- np.log(alpha) / len(trial))
def _neg_vs_at(f, spikes):
return -np.mean([1 - circ.var((trial % (1. / f)) * f * 2 * np.pi) for trial in spikes])
def find_best_locking(spikes, f0, tol=3):
"""
import numpy as np
from scipy.interpolate import interp1d
from scipy.signal import convolve
from scipy import linalg, stats
import io
import imageio
import datajoint as dj
from . import preprocess
from pipeline import experiment
from . import vis
from distutils.version import StrictVersion
assert StrictVersion(dj.__version__) >= StrictVersion('0.3.8')
schema = dj.schema('pipeline_tuning', locals())
def hamming(half, dim):
""" normalized hamming kernel """
k = np.hamming(np.floor(half) * 2 + 1)
return k.reshape([1] * dim + [k.size]) / k.sum()
@schema
class CaKernel(dj.Lookup):
definition = """ # options for calcium response kinetics.
kernel : tinyint # calcium option number
-----
transient_shape : enum('exp','onAlpha') # calcium transient shape
latency = 0 : float # (s) assumed neural response latency
from pycircstat import event_series as es
import pycircstat as circ
from .utils.data import peakdet
from .utils.locking import find_significant_peaks, find_best_locking, vector_strength_at
from . import colordict
import numpy as np
import pandas as pd
import seaborn as sns
import sympy
from scipy import optimize, stats, signal
from matplotlib.ticker import PercentFormatter
import datajoint as dj
from .data import Runs, Cells, BaseEOD, Baseline
schema = dj.schema('efish_analysis', locals())
class PlotableSpectrum:
def plot(self, ax, restrictions, f_max=2000, ncol=None):
sns.set_context('paper')
# colors = ['#1b9e77', '#d95f02', '#7570b3', '#e7298a']
# colors = ['deeppink', 'dodgerblue', sns.xkcd_rgb['mustard'], sns.xkcd_rgb['steel grey']]
markers = [(4, 0, 90), '^', 'D', 's', 'o']
stim, eod, baseline, beat = sympy.symbols('f_s, EODf, f_b, \Delta')
for fos in ((self * Runs()).proj() & restrictions).fetch(as_dict=True):
if isinstance(self, FirstOrderSpikeSpectra):
peaks = (FirstOrderSignificantPeaks() * restrictions & fos)
elif isinstance(self, SecondOrderSpikeSpectra):
import datajoint as dj
from .experiment import BrainArea, Layer
from .preprocess import ExtractRaw, Prepare, Slice, MaskCoordinates
import numpy as np
schema = dj.schema('pipeline_map', locals())
schema.spawn_missing_classes()
@schema
class LayerMembership(dj.Computed):
definition = """
-> ExtractRaw.GalvoROI
-> Slice
-> MaskCoordinates
---
-> Layer
"""
key_source = ExtractRaw.GalvoROI() * Slice() & dict(extract_method=2)
def _make_tuples(self, key):
z = (ExtractRaw.GalvoROI() * MaskCoordinates() & key).fetch['zloc']
self.insert1(dict(key, layer=str(Layer().get_layers(z))))
@schema
class AreaMembership(dj.Computed):
definition = """
# brain area membership of cells
import datajoint as dj
from datajoint.jobs import key_hash
import numpy as np
import matplotlib.pyplot as plt
from commons import lab
from pipeline import shared
import os
from . import experiment, notify
from .utils import h5
from .exceptions import PipelineException
schema = dj.schema('pipeline_treadmill', locals())
@schema
class Treadmill(dj.Computed):
definition = """ # treadmill velocity synchronized to behavior clock
-> experiment.Scan
---
treadmill_raw :longblob # raw treadmill counts
treadmill_time :longblob # (secs) velocity timestamps in behavior clock
treadmill_vel :longblob # (cm/sec) wheel velocity
treadmill_ts=CURRENT_TIMESTAMP :timestamp
"""
@property
def key_source(self):
return experiment.Scan() & experiment.Scan.BehaviorFile().proj()
'''
MAP Motion Tracking Schema
'''
import datajoint as dj
from . import experiment
from . import get_schema_name
schema = dj.schema(get_schema_name('tracking'))
[experiment] # NOQA flake8
@schema
class TrackingDevice(dj.Lookup):
definition = """
tracking_device: varchar(20) # device type/function
---
tracking_position: varchar(20) # device position
sampling_rate: decimal(8, 4) # sampling rate (Hz)
tracking_device_description: varchar(100) # device description
"""
contents = [('Camera 0', 'side', 300, 'Chameleon3 CM3-U3-13Y3M-CS (FLIR)'),
('Camera 1', 'bottom', 300,
'Chameleon3 CM3-U3-13Y3M-CS (FLIR)')]
@schema
class Tracking(dj.Imported):
'''
Video feature tracking.
from mpl_toolkits.mplot3d import Axes3D
import io
from PIL import Image
import itertools
from pipeline import experiment, ephys, psth, tracking, lab, histology, ccf
from pipeline.plot import behavior_plot, unit_characteristic_plot, unit_psth, histology_plot
from pipeline import get_schema_name
from pipeline.plot.util import _plot_with_sem, _jointplot_w_hue
from pipeline.util import _get_trial_event_times
import warnings
warnings.filterwarnings('ignore')
schema = dj.schema(get_schema_name('report'))
os.environ['DJ_SUPPORT_FILEPATH_MANAGEMENT'] = "TRUE"
store_stage = pathlib.Path(dj.config['stores']['report_store']['stage'])
if dj.config['stores']['report_store']['protocol'] == 's3':
store_location = (
pathlib.Path(dj.config['stores']['report_store']['bucket']) /
pathlib.Path(dj.config['stores']['report_store']['location']))
store_location = 'S3: ' + str(store_location)
else:
store_location = pathlib.Path(
dj.config['stores']['report_store']['location'])
mpl.rcParams['font.size'] = 16
import pathlib
from glob import glob
from datetime import datetime
import uuid
import numpy as np
import datajoint as dj
from pipeline import lab
from pipeline import tracking
from pipeline import experiment
from pipeline.ingest import behavior as behavior_ingest
from collections import defaultdict
from .. import get_schema_name
schema = dj.schema(get_schema_name('ingest_tracking'))
log = logging.getLogger(__name__)
[behavior_ingest] # NOQA schema only use
def get_tracking_paths():
"""
retrieve behavior rig paths from dj.config
config should be in dj.config of the format:
dj.config = {
...,
'custom': {
"tracking_data_paths":
"""
Sample schema with realistic tables for testing
"""
import random
import numpy as np
import datajoint as dj
import os, signal
from . import PREFIX, CONN_INFO
schema = dj.schema(PREFIX + '_test1', connection=dj.conn(**CONN_INFO))
@schema
class Test(dj.Lookup):
definition = """
key : int # key
---
value : int # value
"""
contents = [(k, 2*k) for k in range(10)]
@schema
class TestExtra(dj.Manual):
"""
clone of Test but with an extra field
"""
definition = Test.definition + "\nextra : int # extra int\n"
from . import PREFIX, CONN_INFO
import datajoint as dj
from nose.tools import assert_true
schema = dj.schema(PREFIX + '_keywords', locals(), connection=dj.conn(**CONN_INFO))
class A(dj.Manual):
definition = """
a_id: int # a id
"""
class B(dj.Manual):
source = None
definition = """
-> self.source
b_id: int # b id
"""
class H(dj.Part):
definition = """
-> master
name: varchar(128) # name
"""
class C(dj.Part):
definition = """
-> master
-> master.H
# this code is to go into a python file
import datajoint as dj
schema = dj.schema('behavior_experiment')
@schema
class Mouse(dj.Manual):
definition = """
# a table of all mice used for behavior
subject_id: int # unique ID for this mouse
---
dob: date
sex = 'U': enum('F','M', 'U') # sex of the mouse
strain = 'B6': enum('IT-Cre', 'PT-Cre', 'B6')
"""
@schema
class Task(dj.Manual):
definition = """
# task type
task_name: varchar(31)
---
target_size=null: int # size of target in mm
target_position=null: enum('left','right')
"""
@schema
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from PIL import Image
from current_clamp import *
from current_clamp_features import extract_istep_features
from read_metadata import *
from file_io import load_current_step
# from pymysql import IntegrityError
import datajoint as dj
schema = dj.schema('yueqi_ephys', locals())
'''
class DjImportedFromDirectory(dj.Imported):
# Subclass of Imported. Initialize with data directory.
def __init__(self, directory=''):
self.directory = directory
super().__init__()
'''
@schema
class EphysExperimentsForAnalysis(dj.Manual):
definition = """
# Ephys experiments (excel files) for analysis
experiment: varchar(128) # excel files to use for analysis
---
project: varchar(128) # which project the data belongs to
"""
a schema for testing external attributes
"""
import tempfile
import datajoint as dj
from . import PREFIX, CONN_INFO
import numpy as np
schema = dj.schema(PREFIX + '_extern', connection=dj.conn(**CONN_INFO))
dj.config['external'] = {
'protocol': 'file',
'location': 'dj-store/external'}
dj.config['external-raw'] = {
'protocol': 'file',
'location': 'dj-store/raw'}
dj.config['external-compute'] = {
'protocol': 's3',
'location': '/datajoint-projects/test',
'user': '******',
'token': '2e05709792545ce'}
dj.config['cache'] = tempfile.mkdtemp('dj-cache')
@schema
sys.path.append('../lib')
from utils import run
if args.Environment == 'AWS':
credFiles = '/home/ubuntu/data/Github/VaultBrain/credFiles_aws.yaml'
yaml_file = 'shape_params-aws.yaml'
else:
credFiles = '/Users/kuiqian/Github/VaultBrain/credFiles.yaml'
yaml_file = 'shape_params.yaml'
dj.config['database.host'] = get_dj_creds(credFiles)['database.host']
dj.config['database.user'] = get_dj_creds(credFiles)['database.user']
dj.config['database.port'] = get_dj_creds(credFiles)['database.port']
dj.config['database.password'] = get_dj_creds(credFiles)['database.password']
dj.conn()
schema = dj.schema('kui_diffusionmap')
schema.spawn_missing_classes()
stack = args.stack
img_file = 'CSHL_scoremaps_new/' + stack + '/'
img_fp = os.environ['ROOT_DIR'] + img_file
scripts_dir = os.environ['REPO_DIR']
def setup_download_from_s3(rel_fp, recursive=True):
s3_fp = 's3://mousebrainatlas-data/' + rel_fp
local_fp = os.environ['ROOT_DIR'] + rel_fp
if os.path.exists(local_fp):
print('ALREADY DOWNLOADED FILE')
import datajoint as dj
schema = dj.schema("common_mice", locals())
@schema
class Death(dj.Manual):
definition = None
@schema
class Founders(dj.Manual):
definition = None
@schema
class Gels(dj.Manual):
definition = None
@schema
class Genotypes(dj.Manual):
definition = None
@schema
class Lanes(dj.Manual):
definition = None
@schema
from scipy.signal import butter, filtfilt
import datajoint as dj
import pycircstat as circ
import datajoint as dj
from .utils.modeling import estimate_fundamental, get_best_time_window, get_harm_coeff, \
second_order_critical_vector_strength, val_at
from .analysis import TrialAlign
from . import mkdir
from .utils.data import peakdet
from .data import Runs, Cells, EFishes, PaperCells
from scipy import interp
from . import colordict, markerdict
schema = dj.schema('efish_modeling', locals())
# =======================================================================================
@schema
class NoHarmonics(dj.Lookup):
definition = """
no_harmonics : int # number of harmonics that are fitted
"""
contents = [(8, )]
@schema
class EODFit(dj.Computed):
def test_fail_create_schema(self):
"""creating a schema with no CREATE privilege"""
return dj.schema('forbidden_schema', namespace, connection=self.connection)
import os
import sys
sys.path.append('/notebooks')
import pickle
import numpy as np
import datajoint as dj
import nnsetup
from mlutils.data.datasets import StaticImageSet
dj.config['database.host'] = os.environ['DJ_HOST']
dj.config['database.user'] = os.environ['DJ_USERNAME']
dj.config['database.password'] = os.environ['DJ_PASSWORD']
dj.config['enable_python_native_blobs'] = True
schema = dj.schema('mdep_random_subsets_synth', locals())
dj.config['schema_name'] = "mdep_random_subsets_synth"
from nnfabrik.main import *
from nnsetup.tables import TrainedModel
def load_obj(file):
with open('./data/' + file + '.pkl', 'rb') as f:
return pickle.load(f)
Fabrikant().insert1(dict(fabrikant_name='Matthias Depoortere',
email="*****@*****.**",
affiliation='sinzlab',
dj_username="******"),
skip_duplicates=True)
# The minimal example that creates the error
import datajoint as dj
schema = dj.schema('sandbox', locals())
@schema
class Basic(dj.Manual):
definition = """
exp_date :date # primary key with type date
---
path :varchar(20) # variable used in Dependent
"""
@schema
class Dependent(dj.Computed):
definition = """
-> Basic
---
new_path :varchar(21) # variable computed
"""
def _make_tuples(self, key):
p = (Basic() & key).fetch1['path']
new_path = p + 'new'
self.insert1(key, new_path=new_path)
import datajoint as dj
import os, sys
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import microscopy, lab, organism, setup
schema = dj.schema('hillman_experiment')
@schema
class Specimen(dj.Manual):
definition = """
specimen : varchar(64)
---
-> lab.LabMember.proj(source='user')
-> organism.Species
-> [nullable] organism.Species.Genotype
-> [nullable] organism.TissueType
-> [nullable] organism.PreparationType
prep_time=NULL : datetime
specimen_description ='' : varchar(1024)
"""
@schema
class Session(dj.Manual):
definition = """
session_name : varchar(32)
---
-> lab.LabMember
-> setup.Scapeconfig
-> [nullable] lab.Project
import datajoint as dj
schema = dj.schema('common_optical', locals())
@schema
class LoomMap(dj.Imported):
definition = None
def _make_tuples(self, key):
raise NotImplementedError("This table is implemented from matlab.")
@schema
class SpotMap(dj.Imported):
definition = None
def _make_tuples(self, key):
raise NotImplementedError("This table is implemented from matlab.")
@schema
class Structure(dj.Imported):
definition = None
def _make_tuples(self, key):
raise NotImplementedError("This table is implemented from matlab.")
@schema
class StructureMask(dj.Imported):
from sklearn.metrics import roc_curve
import datajoint as dj
from . import rf, trippy
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from pprint import pprint
import pandas as pd
import os
try:
import c2s
except:
warn("c2s was not found. You won't be able to populate ExtracSpikes")
schema = dj.schema('pipeline_preprocessing', locals())
def normalize(img):
return (img - img.min())/(img.max()-img.min())
def bugfix_reshape(a):
return a.ravel(order='C').reshape(a.shape, order='F')
@schema
class SpikeInference(dj.Lookup):
definition = ...
def infer_spikes(self, X, dt, trace_name='ca_trace'):
assert self.fetch1['language'] == 'python', "This tuple cannot be computed in python."
fps = 1 / dt
spike_rates = []
import itertools
import h5py
import datajoint as dj
from aod_cells.bernoulli import RankDegenerateBernoulliProcess
from aod_cells.utils import *
from djaddon import gitlog, hdf5
schema = dj.schema('fabee_aod_cell_detection', locals())
# schema = dj.schema('fabee_cell_detection', locals())
preprocessors = {
'histogram_equalization':
lambda x: histeq(unsharp_masking(medianfilter(center(average_channels(x)))), 500),
# 'center_medianfilter':
# lambda x: medianfilter(center(average_channels(x))),
'basic':
lambda x: unsharp_masking(medianfilter(center(average_channels(x)))),
'whitening':
lambda x: whiten(unsharp_masking(medianfilter(center(average_channels(x))))),
'normalize':
lambda x: local_standardize(medianfilter(average_channels(x))),
}
groups = dict(
manolis082015=[
('data/2015-08-25_12-49-41_2015-08-25_13-02-18.h5',),
('data/2015-08-25_13-49-54_2015-08-25_13-57-23.h5',),
('data/2015-08-25_14-36-29_2015-08-25_14-44-41.h5',),
],
# alyx.manual1.py:
# manually converted Alyx schema -
# preserves Alyx' object inheritance model & uuid aspects;
# some bugs exist w/r/t primary keys - intended mainly as a reference.
import datajoint as dj
schema = dj.schema(dj.config['names.%s' % __name__], locals())
# #####################################################################
# reference.py ########################################################
# #####################################################################
@schema
class User(dj.Manual):
# <class 'misc.models.OrderedUser'>
# <class 'django.contrib.auth.models.User'>
definition = """
username: varchar(255) # username
---
password: varchar(255) # password
email: varchar(255) # email address
last_login: datetime # last login
first_name: varchar(255) # first name
last_name: varchar(255) # last name
date_joined: datetime # date joined
is_active: boolean # active
is_staff: boolean # staff status
is_superuser: boolean # superuser status
"""
import datajoint as dj
import os
if os.environ.get('MODE') == 'test':
dj.config['database.prefix'] = 'test_'
elif os.environ.get('MODE') == 'update':
dj.config['database.prefix'] = 'update_'
if os.environ.get('MODE') == 'test':
schema = dj.schema('test_ibl_storage')
else:
schema = dj.schema('ibl_storage')
@schema
class S3Access(dj.Manual):
definition = """
s3_id: tinyint # unique id for each S3 pair
---
access_key: varchar(128) # S3 access key
secret_key: varchar(128) # S3 secret key
"""
# attempt to get S3 access/secret key from different sources
access_key = os.environ.get('S3_ACCESS')
secret_key = os.environ.get('S3_SECRET')
if (access_key is None or secret_key is None) and len(S3Access.fetch()) > 0:
# if there are multiple entries in S3, it won't work
access_key, secret_key = S3Access.fetch1('access_key', 'secret_key')
from neuro_data.utils.data import h5cached, SplineCurve, FilterMixin, fill_nans, NaNSpline
from neuro_data.static_images import datasets
dj.config['external-data'] = {'protocol': 'file', 'location': '/external/'}
experiment = dj.create_virtual_module('experiment', 'pipeline_experiment')
reso = dj.create_virtual_module('reso', 'pipeline_reso')
meso = dj.create_virtual_module('meso', 'pipeline_meso')
fuse = dj.create_virtual_module('fuse', 'pipeline_fuse')
pupil = dj.create_virtual_module('pupil', 'pipeline_eye')
stimulus = dj.create_virtual_module('stimulus', 'pipeline_stimulus')
shared = dj.create_virtual_module('shared', 'pipeline_shared')
anatomy = dj.create_virtual_module('anatomy', 'pipeline_anatomy')
treadmill = dj.create_virtual_module('treadmill', 'pipeline_treadmill')
schema = dj.schema('neurodata_static')
# set of attributes that uniquely identifies the frame content
UNIQUE_FRAME = {
'stimulus.Frame': ('image_id', 'image_class'),
'stimulus.MonetFrame': ('rng_seed', 'orientation'),
'stimulus.TrippyFrame': ('rng_seed', ),
'stimulus.ColorFrameProjector': ('image_id', 'image_class'),
}
IMAGE_CLASSES = 'image_class in ("imagenet", "imagenet_v2_gray", "imagenet_v2_rgb", "imagenet_v2_rgb_g_b_channels_separately_joined")' # all valid natural image classes
@schema
class StaticScanCandidate(dj.Manual):
definition = """ # list of scans to process
import datajoint as dj
import numpy as np
import pandas as pd
from os import path
import logging
from . import reference, subject, acquisition, data
from .ingest import get_raw_field as grf
try:
from oneibl.one import ONE
except:
# TODO: consider issuing a warning about not being able to perform ingestion without ONE access
pass
logger = logging.getLogger(__name__)
schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_behavior')
@schema
class Eye(dj.Imported):
definition = """
# eye recording
-> acquisition.Session
---
eye_sample_ids: longblob # Sample ids corresponding to the timestamps
eye_timestamps: longblob # Timestamps for pupil tracking timeseries (seconds)
eye_raw_dir=null: varchar(256) # directory of the raw datafile
eye_area: longblob # Area of pupil (pixels^2)
eye_x_pos: longblob # x position of pupil (pixels)
eye_y_pos: longblob # y position of pupil (pixels)
eye_blink: longblob # Boolean array saying whether eye was blinking in each frame
eye_fps: float # Frames per second
from multiprocessing.pool import Pool
import datajoint as dj
from tqdm import tqdm
from ..utils.measures import corr
import numpy as np
from scipy import stats
from .parameters import DataConfig
from .data import MovieMultiDataset, MovieScan
from ..utils.logging import Messager
schema = dj.schema('nips2018_oracle', locals())
def spearm(pair):
return stats.spearmanr(*pair)[0]
@schema
class MovieOracle(dj.Computed, Messager):
definition = """
# oracle computation for hollymonet data
-> MovieMultiDataset
-> DataConfig
---
"""
'''
Schema of virus
'''
import re
import os
from datetime import datetime
import numpy as np
import scipy.io as sio
import datajoint as dj
import h5py as h5
from . import reference, subject, utilities, stimulation
schema = dj.schema(dj.config['custom'].get('database.prefix', '') + 'virus')
@schema
class VirusSource(dj.Lookup):
definition = """
virus_source: varchar(64)
"""
contents = zip(['UNC', 'UPenn', 'MIT', 'Stanford', 'Homemade'])
@schema
class Virus(dj.Manual):
definition = """
virus: varchar(64) # name of the virus
---
-> VirusSource
import csv
import re
import json
import math
import numpy as np
import scipy.io as scio
import datajoint as dj
from datetime import datetime
from pipeline import lab, ephys, experiment, ccf, histology, report
from pipeline import get_schema_name, dict_to_hash
from pipeline.ingest import behavior as behavior_ingest
schema = dj.schema(get_schema_name('ingest_histology'))
log = logging.getLogger(__name__)
def get_histology_paths():
"""
retrieve histology paths from dj.config
config should be in dj.config of the format:
dj.config = {
...,
'custom': {
'histology_data_paths': ['/path/string', '/path2/string']
}
...
<- Indicator
<- Pharmacology
~ Potential Expansions
Computed traces and receptive fields
[... <- Roi <- Trace <- RF ]
Inclusion of stacks and images for a given field
[... <- Field <- Zstack ]
[ <- Image ]
"""
import datajoint as dj
schema = dj.schema(p['database'],locals())
@schema
class Animal(dj.Manual):
definition = """
# Basic animal info
animid :varchar(255) # unique ID given to the animal
---
genbkgline :varchar(255) # Genetic background line
genbkglinerem :varchar(255) # Comments about background line
genline_reporter :varchar(255) # Genetic reporter line
genline_reporterrem :varchar(255) # Comments about reporter line
animspecies="mouse" :enum("mouse","rat","zebrafish") # animal species
animgender :enum("M","F") # gender.
animage :varchar(255) # Whether to have this or DOB?
import datajoint as dj
from . import reference
import os
mode = os.environ.get('MODE')
if mode == 'update':
schema = dj.schema('ibl_subject')
else:
schema = dj.schema(dj.config.get('database.prefix', '') + 'ibl_subject')
@schema
class Species(dj.Lookup):
# <class 'subjects.models.Species'>
definition = """
binomial: varchar(255) # binomial
---
species_uuid: uuid
species_nickname: varchar(255) # nick name
species_ts=CURRENT_TIMESTAMP: timestamp
"""
@schema
class Source(dj.Lookup):
# <class 'subjects.models.Source'>
definition = """
source_name: varchar(255) # name of source
---
source_uuid: uuid
from tqdm import tqdm
import cv2
import numpy as np
import json
import os
from commons import lab
from datajoint.autopopulate import AutoPopulate
from .utils.eye_tracking import ROIGrabber, PupilTracker, CVROIGrabber, ManualTracker
from . import config
from .utils import h5
from . import experiment, notify
from .exceptions import PipelineException
schema = dj.schema('pipeline_eye', locals())
DEFAULT_PARAMETERS = {'relative_area_threshold': 0.002,
'ratio_threshold': 1.5,
'error_threshold': 0.1,
'min_contour_len': 5,
'margin': 0.02,
'contrast_threshold': 5,
'speed_threshold': 0.1,
'dr_threshold': 0.1,
'gaussian_blur': 5,
'extreme_meso': 0,
'running_avg': 0.4,
'exponent': 9
}
from oneibl.one import ONE
import brainbox.behavior.wheel as wh
from ibllib.io.extractors.training_wheel import extract_wheel_moves, extract_first_movement_times, infer_wheel_units
from ibllib.io.params import getfile
logging.basicConfig(
format='%(asctime)s - %(message)s',
handlers=[
logging.FileHandler("/src/IBL-pipeline/ibl_pipeline/process/logs/process_wheel.log"),
logging.StreamHandler()],
level=25)
logger = logging.getLogger(__name__)
schema = dj.schema('group_shared_wheel') # group_shared_wheel
@schema
class WheelMoveSet(dj.Imported):
definition = """
# Wheel movements occurring within a session
-> acquisition.Session
---
n_movements: int # total number of movements within the session
total_displacement: float # total displacement of the wheel during session in radians
total_distance: float # total movement of the wheel in radians
n_direction_changes: int # total number of direction changes within a session
"""
class Move(dj.Part):
import datajoint as dj
schema = dj.schema('morphology_reconstructions', locals())
import datajoint as dj
import pathlib
import importlib
import inspect
from element_interface.utils import find_root_directory, find_full_path
schema = dj.schema()
_linking_module = None
def activate(scan_schema_name,
*,
create_schema=True,
create_tables=True,
linking_module=None):
"""
activate(scan_schema_name, *, create_schema=True, create_tables=True, linking_module=None)
:param scan_schema_name: schema name on the database server to activate the `scan` module
:param create_schema: when True (default), create schema in the database if it does not yet exist.
:param create_tables: when True (default), create tables in the database if they do not yet exist.
:param linking_module: a module name or a module containing the
required dependencies to activate the `scan` module:
Upstream tables:
+ Session: parent table to Scan, typically identifying a recording session
+ Equipment: Reference table for Scan, specifying the equipment used for the acquisition of this scan
+ Location: Reference table for ScanLocation, specifying the brain location where this scan is acquired
Functions:
+ get_imaging_root_data_dir() -> list
Retrieve the full path for the root data directory - e.g. containing the imaging recording files or analysis results for all subject/sessions.
:return: a string (or list of string) for full path to the root data directory
import pandas as pd
from . import aodpre
import warnings
from IPython import embed
import glob
import numpy as np
import dateutil.parser
from . import utils
import cv2
import os,shutil
try:
from pupil_tracking.pupil_tracker_aod import PupilTracker
except ImportError:
warnings.warn("Failed to import pupil_tacking library. You won't be able to populate trk.EyeFrame")
schema = dj.schema('pipeline_aod_pupiltracking', locals())
@schema
class TrackInfo(dj.Imported):
definition = """
# machine independent path of eye videos
->aodpre.Scan
---
base_video_path: varchar(100) # base path of the video
"""
def _make_tuples(self, key):
print("key = ", key)
# embed()
import datajoint as dj
from datajoint.errors import DataJointError
import json
import uuid
import re
import pdb
from . import alyxraw, reference, acquisition, ephys
from . import get_raw_field as grf
from ibl_pipeline import acquisition as acquisition_real
from ibl_pipeline import ephys as ephys_real
schema = dj.schema(dj.config.get('database.prefix', '') +
'ibl_ingest_histology')
# Temporary probe trajectories and channel brain location based on methods
@schema
class Provenance(dj.Lookup):
definition = """
provenance : tinyint unsigned # provenance code
---
provenance_description : varchar(128) # type of trajectory
"""
contents = [
(70, 'Ephys aligned histology track'),
(50, 'Histology track'),
(30, 'Micro-manipulator'),
(10, 'Planned'),
]
# -*- coding: utf-8 -*-
import datajoint as dj
print("Welcome to the database 'demo1'")
schema = dj.schema('dj_test', locals())
@schema
class Subject(dj.Manual):
definition = """
# Basic subject info
subject_id : int # internal subject id
---
real_id : varchar(40) # real-world name
species = "mouse" : enum('mouse', 'monkey', 'human') # species
date_of_birth=null : date # animal's date of birth
sex="unknown" : enum('M','F','unknown') #
caretaker="Unknown" : varchar(20) # person responsible for working with this subject
animal_notes="" : varchar(4096) # strain, genetic manipulations, etc
"""
s = Subject()
p = s.primary_key
@schema
class Experiment(dj.Manual):
definition = """
# Basic subject info
import os
import datajoint as dj
from . import stack, anatomy, meso
dj.config['database.prefix'] = os.environ.get('DJ_PREFIX', '')
schema = dj.schema(dj.config['database.prefix'] + 'pipeline_meso')
@schema
class StackCoordinates(dj.Computed):
definition = """ # centroids of each unit in motor/stack coordinate system
-> meso.ScanSet # animal_id, session, scan_idx, channel, field, segmentation_method, pipe_version
-> stack.Registration.proj(session='scan_session') # animal_id, stack_session, stack_idx, volume_id, session, scan_idx, field, stack_channel, scan_channel, registration_method
"""
class UnitInfo(dj.Part):
definition = """ # ScanSet.UnitInfo centroids mapped to stack coordinates
-> master # this will add field and channels back
-> meso.ScanSet.Unit
---
stack_x : float
stack_y : float
stack_z : float
"""
def make(self, key):
from scipy import ndimage
# Get registration grid (px -> stack_coordinate)
def test_insert_failure(self):
unprivileged = dj.schema(schema.schema.database, namespace, connection=self.connection)
unprivileged.spawn_missing_classes()
assert_true(issubclass(Language, dj.Lookup) and len(Language()) == len(schema.Language()),
'failed to spawn missing classes')
Language().insert1(('Socrates', 'Greek'))
import os
import datajoint as dj
from datajoint.hash import key_hash
from . import experiment
dj.config['database.prefix'] = os.environ.get('DJ_PREFIX', '')
schema = dj.schema(dj.config['database.prefix'] + 'pipeline_notification')
# Decorator for notification functions. Ignores exceptions.
def ignore_exceptions(f):
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
print('Ignored exception:', e)
return wrapper
@schema
class SlackConnection(dj.Manual):
definition = """
# slack domain and api key for notification
domain : varchar(128) # slack domain
---
api_key : varchar(128) # api key for bot connection
"""
import datajoint as dj
schema = dj.schema('common_two_photon', locals())
@schema
class Align(dj.Imported):
definition = None
def _make_tuples(self, key):
raise NotImplementedError("This table is implemented from matlab.")
@schema
class CaOpt(dj.Lookup):
definition = None
@schema
class CellClass(dj.Lookup):
definition = None
@schema
class CellClassification(dj.Computed):
definition = None
def _make_tuples(self, key):
raise NotImplementedError("This table is implemented from matlab.")
# -*- coding: utf-8 -*-
import datajoint as dj
print("Welcome to the database 'demo1'")
schema = dj.schema('dj_test', locals())
@schema
class Subject(dj.Manual):
definition = """
# Basic subject info
subject_id : int # internal subject id
---
real_id : varchar(40) # real-world name
species = "mouse" : enum('mouse', 'monkey', 'human') # species
date_of_birth=null : date # animal's date of birth
sex="unknown" : enum('M','F','unknown') #
caretaker="Unknown" : varchar(20) # person responsible for working with this subject
animal_notes="" : varchar(4096) # strain, genetic manipulations, etc
"""
s = Subject()
p = s.primary_key
@schema
class Experiment(dj.Manual):
definition = """
import datajoint as dj
from datajoint.jobs import key_hash
from . import experiment
schema = dj.schema('pipeline_notification', locals())
# Decorator for notification functions. Ignores exceptions.
def ignore_exceptions(f):
def wrapper(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as e:
print('Ignored exception:', e)
return wrapper
@schema
class SlackConnection(dj.Manual):
definition = """
# slack domain and api key for notification
domain : varchar(128) # slack domain
---
api_key : varchar(128) # api key for bot connection
"""
@schema
class SlackUser(dj.Manual):
definition = """
# information for user notification
import datajoint as dj
from tqdm import tqdm
import json
import numpy as np
import pandas as pd
from scipy import stats
from neuro_data.utils.measures import corr
from .configs import DataConfig
from .data_schemas import StaticMultiDataset, StaticScan
from .. import logger as log
schema = dj.schema('neurodata_static_stats', locals())
data_schemas = dj.create_virtual_module('data_schemas', 'neurodata_static')
@schema
class Oracle(dj.Computed):
definition = """
# oracle computation for static images
-> StaticMultiDataset
-> DataConfig
---
"""
@property
def key_source(self):
return StaticMultiDataset() * DataConfig()
import datajoint as dj
schema = dj.schema("dimitri_brain_state", locals())
schema.spawn_missing_classes()
import datajoint as dj
import json
import uuid
from . import alyxraw, reference, subject, action
from .. import acquisition
from . import get_raw_field as grf
schema = dj.schema(
dj.config.get('database.prefix', '') + 'ibl_ingest_acquisition')
@schema
class Session(dj.Computed):
definition = """
-> alyxraw.AlyxRaw.proj(session_uuid='uuid')
---
session_number=null: int
subject_uuid: uuid
session_start_time: datetime
session_end_time=null: datetime
session_lab=null: varchar(255)
session_location=null: varchar(255)
session_type=null: varchar(255)
session_narrative=null: varchar(2048)
task_protocol=null: varchar(255)
session_ts=CURRENT_TIMESTAMP: timestamp
"""
key_source = (alyxraw.AlyxRaw
& 'model="actions.session"').proj(session_uuid='uuid')
import datajoint as dj
from . import rf
schema = dj.schema('pipeline_structural_stacks', locals())
import datajoint as dj
import numpy as np
from workflow_calcium_imaging.pipeline import db_prefix, session, scan, imaging, trial, \
event
schema = dj.schema(db_prefix + 'analysis')
@schema
class ActivityAlignmentCondition(dj.Manual):
definition = """
-> imaging.Activity
-> event.AlignmentEvent
trial_condition: varchar(128) # user-friendly name of condition
---
condition_description='': varchar(1000)
bin_size=0.04: float # bin-size (in second) used to compute the PSTH
"""
class Trial(dj.Part):
definition = """ # Trials (or subset) to compute event-aligned activity
-> master
-> trial.Trial
"""
@schema
class ActivityAlignment(dj.Computed):
definition = """
-> ActivityAlignmentCondition
import numpy as np
from nose.tools import assert_true
import datajoint as dj
from . import PREFIX, CONN_INFO
schema = dj.schema(PREFIX + '_nantest', locals(), connection=dj.conn(**CONN_INFO))
@schema
class NanTest(dj.Manual):
definition = """
id :int
---
value=null :double
"""
class TestNaNInsert:
@classmethod
def setup_class(cls):
cls.rel = NanTest()
with dj.config(safemode=False):
cls.rel.delete()
a = np.array([0, 1/3, np.nan, np.pi, np.nan])
cls.rel.insert(((i, value) for i, value in enumerate(a)))
cls.a = a
def test_insert_nan(self):
"""Test fetching of null values"""
b = self.rel.fetch('value', order_by='id')
assert_true((np.isnan(self.a) == np.isnan(b)).all(),
import datajoint as dj
from pipeline import experiment
anatomy = dj.create_virtual_module("anatomy", "pipeline_anatomy")
schema = dj.schema("pipeline_collection", create_tables=True)
@schema
class Study(dj.Manual):
definition = """
# class of experiments towards specific aim
study_name :varchar(50) # name of study
---
description :varchar(450) # brief description of experimental details and study goals
"""
@schema
class CurationPurpose(dj.Lookup):
definition = """
# Targeted use for scans added to CuratedScans
scan_purpose :varchar(100) # scan use
---
scan_criteria :varchar(450) # brief description of properties scan must have
purpose_ts = CURRENT_TIMESTAMP :timestamp # automatic at time of first entry
"""
@schema
class CuratedScan(dj.Manual):
""" Lookup tables shared among multi-photon pipelines. """
import datajoint as dj
schema = dj.schema('pipeline_shared', locals(), create_tables=False)
@schema
class Field(dj.Lookup):
definition = """ # fields in mesoscope scans
field : tinyint
"""
contents = [[i] for i in range(1, 25)]
@schema
class Channel(dj.Lookup):
definition = """ # recording channel, directly related to experiment.PMTFilterSet.Channel
channel : tinyint
"""
contents = [[i] for i in range(1, 5)]
@schema
class PipelineVersion(dj.Lookup):
definition = """ # versions for the reso pipeline
pipe_version : smallint
"""
contents = [[i] for i in range(3)]
@schema
class SegmentationMethod(dj.Lookup):
definition = """ # methods for mask extraction for multi-photon scans
def test_drop_database():
schema = dj.schema(PREFIX + '_drop_test', connection=dj.conn(reset=True, **CONN_INFO))
assert_true(schema.exists)
schema.drop()
assert_false(schema.exists)
schema.drop() # should do nothing
