def import_csv(context,
container_uri=None,
protocol_uri=None,
files=[],
timezone=None,
csv_header_row=CSV_HEADER_ROW,
first_measurement_column=FIRST_MEASUREMENT_COLUMN_NUMBER,
date_column=DATE_COLUMN_NUMBER):
assert(not protocol_uri is None)
assert(not container_uri is None)
container = asclass("us.physion.ovation.domain.mixin.EpochGroupContainer", context.getObjectWithURI(container_uri))
protocol = asclass("Protocol", context.getObjectWithURI(protocol_uri))
if timezone is None:
timezone = DateTimeZone.getDefault().getID()
for f in files:
_import_file(context,
container,
protocol,
f,
csv_header_row,
timezone,
first_measurement_column,
date_column)
def import_wrapper(data_context,
container=None,
protocol=None,
files=None,
sources=None,
equipment_setup_root=None,
**args):
container = data_context.getObjectWithURI(container)
protocol_entity = data_context.getObjectWithURI(protocol)
if protocol_entity:
protocol = asclass("Protocol", protocol_entity)
else:
protocol = None
sources = [data_context.getObjectWithURI(source) for source in sources]
for file in files:
import_file(file,
container,
equipment_setup_root,
sources,
protocol=protocol)
return 0
def should_cast_experiment_from_procedure_element_via_explicit_cast(self):
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol,
None, None)
assert_equals(
self.expt.getUuid().toString(),
asclass("us.physion.ovation.domain.Experiment",
epoch.getParent()).getUuid().toString())
def import_csv(context,
container_uri=None,
protocol_uri=None,
files=[],
timezone=None,
csv_header_row=CSV_HEADER_ROW,
first_measurement_column=FIRST_MEASUREMENT_COLUMN_NUMBER,
date_column=DATE_COLUMN_NUMBER):
assert (not protocol_uri is None)
assert (not container_uri is None)
container = asclass("us.physion.ovation.domain.mixin.EpochGroupContainer",
context.getObjectWithURI(container_uri))
protocol = asclass("Protocol", context.getObjectWithURI(protocol_uri))
if timezone is None:
timezone = DateTimeZone.getDefault().getID()
for f in files:
_import_file(context, container, protocol, f, csv_header_row, timezone,
first_measurement_column, date_column)
def import_segment(epoch_group,
segment,
sources,
protocol=None,
equipment_setup_root=None):
ctx = epoch_group.getDataContext()
if protocol is None:
protocol = ctx.getProtocol(NEO_PROTOCOL)
if protocol is None:
protocol = ctx.insertProtocol(NEO_PROTOCOL, NEO_PROTOCOL_TEXT)
segment_duration = max(arr.t_stop for arr in segment.analogsignals)
segment_duration.units = 'ms' #milliseconds
start_time = DateTime(epoch_group.getStart())
inputSources = Maps.newHashMap()
outputSources = Maps.newHashMap()
for s in sources:
if s:
s = asclass("Source", s)
inputSources.put(s.getLabel(), s)
device_parameters = dict(("{}.{}".format(equipment_setup_root, k), v) for (k,v) in segment.annotations.items())
epoch = epoch_group.insertEpoch(inputSources,
outputSources,
start_time,
start_time.plusMillis(int(segment_duration)),
protocol,
to_map(segment.annotations),
to_map(device_parameters)
)
if segment.index is not None:
epoch.addProperty('index', box_number(segment.index))
if len(segment.analogsignalarrays) > 0:
log_warning("Segment contains AnalogSignalArrays. Import of AnalogSignalArrays is currently not supported")
for analog_signal in segment.analogsignals:
import_analog_signal(epoch, analog_signal, equipment_setup_root)
import_timeline_annotations(epoch, segment, start_time)
if len(segment.spikes) > 0:
logging.warning("Segment contains Spikes. Import of individual Spike data is not yet implemented (but SpikeTrains are).")
import_spiketrains(epoch, protocol, segment)
def should_give_data_path(self):
df = pd.DataFrame({'ColA' : np.random.randn(10),
'ColB' : np.random.randn(10)})
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol, None, None)
m = insert_tabular_measurement(epoch, set(), set(), 'tabular', df)
while(m.getDataContext().getFileService().hasPendingUploads()):
pass
m = asclass('Measurement', m.refresh())
expected = m.getLocalDataPath().get()
actual = datapath(m)
assert_equals(expected, actual)
def should_round_trip_pandas_data_frame_artifact(self):
expected = pd.DataFrame({
'ColA': np.random.randn(10),
'ColB': np.random.randn(10)
})
project = list(self.expt.getProjects())[0]
record = project.addAnalysisRecord('name', to_map({}), None,
to_map({}))
m = add_tabular_analysis_artifact(record, 'tabular', expected)
while (m.getDataContext().getFileService().hasPendingUploads()):
pass
m = asclass('Measurement', m.refresh())
actual = as_data_frame(m, index_col=0)
assert_frame_equal(expected, actual)
def should_round_trip_data_frame(self):
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol, None, None)
df = pd.DataFrame({'col1': np.random.rand(10), 'col2': np.random.rand(10)})
source_name = 'source'
s = epoch.getDataContext().insertSource('source-name', 'source-id')
epoch.addInputSource(source_name, s)
name = 'data_name'
m = insert_tabular_measurement(epoch, {source_name}, {'amp'}, name, df)
actual = None
while actual is None:
m = asclass('Measurement', m.refresh())
try:
actual = pd.read_csv(m.getLocalDataPath().get(), index_col=0)
except Exception as e:
pass
assert_true(((actual-df) < .001).all().all(), msg="Approximately equal")
def should_round_trip_pandas_data_frame(self):
expected = pd.DataFrame({
'ColA': np.random.randn(10),
'ColB': np.random.randn(10)
})
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol,
None, None)
m = insert_tabular_measurement(epoch, set(), set(), 'tabular',
expected)
while (m.getDataContext().getFileService().hasPendingUploads()):
pass
m = asclass('Measurement', m.refresh())
actual = as_data_frame(m, index_col=0)
assert_frame_equal(expected, actual)
def should_cast_experiment_from_procedure_element(self):
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol,
None, None)
assert_equals(
self.expt.getUuid().toString(),
asclass("Experiment", epoch.getParent()).getUuid().toString())
# Example of how to get project named "Surprise laser probe" by searching for its name
projects = con.getProjects()
project_uri = None
print "fetching projects..."
for project in projects:
if project.getName() == "Surprise laser probe":
print "Found project!"
project_uri = project.getURI().toString()
break
print "done fetching!"
# Now grab the project by URI. Note that URI needs to be known for this to work.
# Ideally, you would only iterate once and save the URIs of the projects locally
if project_uri is not None:
project = asclass('Project', con.getObjectWithURI(project_uri))
print project.getName()
else:
raise ValueError, "could not find project"
# Now lets insert an experiment names "session_001"
# First make sure experiment does not exist so we don't insert double sessions
experiments = project.getExperiments()
for experiment in experiments:
if experiment.getPurpose() == "session_008":
date = experiment.getStart().toString()
raise ValueError, "Experiment already exists and has date of " + date
# Grab the local current time with the argument being the EST time zone
#current_time = datetime.datetime.now(pytz.timezone("US/Eastern"))
experiment_time = datetime.datetime(2014, 8, 18, tzinfo = pytz.timezone("US/Eastern"))
def import_ncs(self,
container,
sources,
label,
ncs_files=list(),
event_file=None,
start_id=None,
end_id=None,
include_interepoch=True):
self.sources = sources
self.source_map = Maps.newHashMap()
[self.source_map.put(source.getLabel(), source) for source in sources]
# TODO figure out what should go into device params
# device parameters
self.device_parameters = {}
epochs = {}
epoch_boundaries = None
group = None
for f in ncs_files:
logging.info("Reading %s", f)
with open(f, 'rb') as ncs_file:
reader = BinaryReader(ncs_file, NEURALYNX_ENDIAN)
header = parse_header(reader)
device_name = header["AcqEntName"]
csc_data = CscData(header, ncs_blocks(reader, header))
open_time = header["Time Opened"]
# We assume all times are datetime in given local zone
start = self.timeFormatter.parseDateTime(str(open_time)).toDateTime(self.timezone)
logging.info("Start done")
if group is None:
logging.info("Inserting top-level EpochGroup: %s", label)
group = asclass("us.physion.ovation.domain.mixin.EpochGroupContainer", container).insertEpochGroup(label,
start,
self.protocol,
to_map(self.protocol_parameters),
to_map(self.device_parameters)
)
if event_file is None or start_id is None:
logging.info("No event file present")
if not None in epochs:
epochs[None] = self.insert_epoch(group, open_time, None, False)
self.append_response(epochs[None], device_name, csc_data, open_time, None)
else:
logging.info("Event file present")
if epoch_boundaries is None:
logging.info("Determining Epoch boundaries")
with open(event_file, 'rb') as ef:
reader = BinaryReader(ef, NEURALYNX_ENDIAN)
header = parse_header(reader)
epoch_boundaries = list(EpochBoundaries(header,
nev_events(reader, header),
start_id,
end_id,
include_interepoch).boundaries)
current_epoch = None
for epoch_boundary in epoch_boundaries:
if not epoch_boundary in epochs:
epochs[epoch_boundary] = self.insert_epoch(group,
epoch_boundary.start,
epoch_boundary.end,
epoch_boundary.interepoch)
epoch = epochs[epoch_boundary]
self.append_response(epoch, device_name, csc_data, epoch_boundary.start, epoch_boundary.end)
current_epoch = epoch
def should_cast_experiment_from_procedure_element_via_explicit_cast(self):
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol, None, None)
assert_equals(self.expt.getUuid().toString(),
asclass("us.physion.ovation.domain.Experiment", epoch.getParent()).getUuid().toString())
def should_cast_none(self):
assert_equals(None, asclass("Experiment", None))
def should_cast_experiment_from_procedure_element(self):
epoch = self.expt.insertEpoch(DateTime(), DateTime(), self.protocol, None, None)
assert_equals(self.expt.getUuid().toString(),
asclass("Experiment", epoch.getParent()).getUuid().toString())
def should_cast_none(self):
assert_equals(None, asclass("Experiment", None))
def import_block(epoch_group_container,
block,
equipment_setup_root,
sources,
protocol=None,
protocol_parameters={},
device_parameters={},
group_label=None,
file_mtime=None):
"""Import a `Neo <http://neuralensemble.org/neo/>`_ `Block` as a single Ovation `EpochGroup`
Parameters
----------
block : neo.Block
`neo.Block` to import
epoch_group_container : ovation.EpochGroup or ovation.Experiment
Container for the inserted `ovation.EpochGroup`
equipment_setup : ovation.EquipmentSetup
Experiment `EquipmentSetup` for the data contained in the file to be imported
equipment_setup_root : str
Root path for equipment setup describing equipment that recorded the data to be imported
source : iterable of us.physion.ovation.domain.Source
Experimental `Subjects` for data contained in `block`
protocol : ovation.Protocol, optional
Ovation `Protocol` for the EpochGroup (if present)
protocol_parameters : Mapping, optional
device_parameters : Mapping, optional
group_label : string, optional
EpochGroup label. If `None`, and `block.name` is not `None`, `block.name` will be used
for the EpochGroup label.
Returns
-------
The inserted `ovation.EpochGroup`
"""
if group_label is None:
if not (block.name is None):
group_label = block.name
else:
group_label = "Neo importer"
merged_protocol_parameters = protocol_parameters.copy()
merged_protocol_parameters.update(block.annotations)
#Convert a datetime.datetime to a DateTime
if block.rec_datetime is not None:
start_time = DateTime(*(block.rec_datetime.timetuple()[:7]))
else:
log_warning("Block does not contain a recording date/time. Using file modification time instead.")
start_time = DateTime(*(datetime.fromtimestamp(file_mtime).timetuple()[:7]))
epochGroup = asclass("us.physion.ovation.domain.mixin.EpochGroupContainer", epoch_group_container).insertEpochGroup(group_label,
start_time,
protocol,
to_map(merged_protocol_parameters),
to_map(device_parameters)
)
if len(block.recordingchannelgroups) > 0:
log_warning("Block contains RecordingChannelGroups. Import of RecordingChannelGroups is currently not supported.")
log_info("Importing segments from {}".format(block.file_origin))
for seg in block.segments:
log_info("Importing segment {} from {}".format(str(seg.index), block.file_origin))
import_segment(epochGroup,
seg,
sources,
protocol=protocol,
equipment_setup_root=equipment_setup_root)
log_info("Waiting for uploads to complete...")
fs = epoch_group_container.getDataContext().getFileService()
while(fs.hasPendingUploads()):
fs.waitForPendingUploads(10, TimeUnit.SECONDS)
return epochGroup
def should_set_device_parameters(self):
assert_equals(self.device_info.keys(),
to_dict(asclass("Experiment", self.epoch_group.getParent()).getEquipmentSetup().getDeviceDetails()).keys())
def import_wrapper(data_context,
container=None,
protocol=None,
files=None,
sources=None,
epoch_group_label=None,
event_file=None,
protocol_parameters=None,
epoch_start_event_id=None,
epoch_end_event_id=None,
include_interepoch=False,
timezone=None,
**args):
container = data_context.getObjectWithURI(container)
protocol_entity = data_context.getObjectWithURI(protocol)
if protocol_entity:
protocol = asclass("Protocol", protocol_entity)
else:
protocol = None
if protocol_parameters is None:
protocol_parameters = {}
else:
protocol_parameters = dict(protocol_parameters)
if not sources is None:
sources = [asclass('Source', data_context.getObjectWithURI(source)) for source in sources]
importer = NeuralynxImporter(
protocol=protocol,
protocol_parameters=protocol_parameters,
timezone=timezone)
data_context.beginTransaction()
logging.info("Starting import...")
try:
importer.import_ncs(container,
sources,
epoch_group_label,
ncs_files=files,
event_file=event_file,
start_id=epoch_start_event_id,
end_id=epoch_end_event_id,
include_interepoch=include_interepoch)
except Exception as e:
logging.error("Import failed, aborting.")
logging.error("Error: %s" % e)
traceback.print_exc(file=sys.stdout)
data_context.abortTransaction()
else:
logging.info("Import complete.")
data_context.commitTransaction()
time.sleep(2) # wait for file service to recognize pending uploads
logging.info("Waiting for file uploads...")
file_service = data_context.getFileService()
while file_service.hasPendingUploads():
logging.info(' .')
time.sleep(2)
logging.info("Done.")
return 0
