def configure(self, view_model):
# type: (PearsonCorrelationCoefficientAdapterModel) -> None
"""
Store the input shape to be later used to estimate memory usage.
:param time_series: the input time-series index for which correlation coefficient should be computed
:param t_start: the physical time interval start for the analysis
:param t_end: physical time, interval end
"""
if view_model.t_start >= view_model.t_end or view_model.t_start < 0:
raise LaunchException(
"Can not launch operation without monitors selected !!!")
self.input_time_series_index = self.load_entity_by_gid(
view_model.time_series.hex)
self.input_shape = (int((view_model.t_end - view_model.t_start) /
self.input_time_series_index.sample_period),
self.input_time_series_index.data_length_2d,
self.input_time_series_index.data_length_3d,
self.input_time_series_index.data_length_4d)
def launch(self, view_model):
# type: (ProjectionMatrixImporterModel) -> [ProjectionMatrixIndex]
"""
Creates ProjectionMatrix entity from uploaded data.
:raises LaunchException: when
* no projection_file or sensors are specified
* the dataset is invalid
* number of sensors is different from the one in dataset
"""
if view_model.projection_file is None:
raise LaunchException("Please select MATLAB file which contains data to import")
if view_model.sensors is None:
raise LaunchException("No sensors selected. Please initiate upload again and select one.")
if view_model.surface is None:
raise LaunchException("No source selected. Please initiate upload again and select a source.")
surface_index = self.load_entity_by_gid(view_model.surface.hex)
expected_surface_shape = surface_index.number_of_vertices
sensors_index = self.load_entity_by_gid(view_model.sensors.hex)
expected_sensors_shape = sensors_index.number_of_sensors
self.logger.debug("Reading projection matrix from uploaded file...")
if view_model.projection_file.endswith(".mat"):
projection_data = self.read_matlab_data(view_model.projection_file, view_model.dataset_name)
else:
projection_data = self.read_list_data(view_model.projection_file)
if projection_data is None or len(projection_data) == 0:
raise LaunchException("Invalid (empty) dataset...")
if projection_data.shape[0] != expected_sensors_shape:
raise LaunchException("Invalid Projection Matrix shape[0]: %d Expected: %d" % (projection_data.shape[0],
expected_sensors_shape))
if projection_data.shape[1] != expected_surface_shape:
raise LaunchException("Invalid Projection Matrix shape[1]: %d Expected: %d" % (projection_data.shape[1],
expected_surface_shape))
projection_matrix_type = determine_projection_type(sensors_index)
surface_ht = h5.load_from_index(surface_index)
sensors_ht = h5.load_from_index(sensors_index)
projection_matrix = ProjectionMatrix(sources=surface_ht, sensors=sensors_ht,
projection_type=projection_matrix_type,
projection_data=projection_data)
return h5.store_complete(projection_matrix, self.storage_path)
def _parse_header(self):
"""
Reads the ordinals from the header and updates self.permutation
"""
self.line += 1
try:
ordinals = [int(v) for v in self.rows[0]]
except ValueError:
raise LaunchException("Invalid ordinal in header %s" %
self.rows[0])
header_i = list(enumerate(ordinals))
header_i.sort(
key=lambda i__ordinal: i__ordinal[1]) # sort by the column ordinal
inverse_permutation = [i for i, ordinal_ in header_i]
for i in range(len(self.permutation)):
self.permutation[inverse_permutation[i]] = i
self.rows = self.rows[1:] # consume header
def initiate_operation(self, current_user, project_id, adapter_instance,
temporary_storage, method_name=ABCAdapter.LAUNCH_METHOD, visible=True, **kwargs):
"""
Gets the parameters of the computation from the previous inputs form,
and launches a computation (on the cluster or locally).
Invoke custom method on an Adapter Instance. Make sure when the
operation has finished that the correct results are stored into DB.
"""
if not isinstance(adapter_instance, ABCAdapter):
self.logger.warning("Inconsistent Adapter Class:" + str(adapter_instance.__class__))
raise LaunchException("Developer Exception!!")
# Prepare Files parameters
files = {}
kw2 = copy(kwargs)
for i, j in kwargs.iteritems():
if isinstance(j, FieldStorage) or isinstance(j, Part):
files[i] = j
del kw2[i]
temp_files = {}
try:
for i, j in files.iteritems():
if j.file is None:
kw2[i] = None
continue
uq_name = utils.date2string(datetime.now(), True) + '_' + str(i)
# We have to add original file name to end, in case file processing
# involves file extension reading
file_name = TEMPORARY_PREFIX + uq_name + '_' + j.filename
file_name = os.path.join(temporary_storage, file_name)
kw2[i] = file_name
temp_files[i] = file_name
file_obj = open(file_name, 'wb')
file_obj.write(j.file.read())
file_obj.close()
self.logger.debug("Will store file:" + file_name)
kwargs = kw2
except Exception, excep:
self._handle_exception(excep, temp_files, "Could not launch operation: invalid input files!")
def launch(self, surface_type, data_file, should_center=False):
"""
Execute import operations:
"""
try:
surface = make_surface(surface_type)
if surface is None:
raise ParseException("Could not determine surface type! %s" %
surface_type)
surface.storage_path = self.storage_path
surface.zero_based_triangles = True
with open(data_file) as f:
obj = ObjSurface(f)
if should_center:
vertices = center_vertices(obj.vertices)
else:
vertices = obj.vertices
surface.vertices = vertices
surface.triangles = obj.triangles
if obj.have_normals:
self.log.debug("OBJ came with normals included")
surface.vertex_normals = obj.normals
else:
self.log.warning(
"OBJ came without normals. We will try to compute them...")
validation_result = surface.validate()
if validation_result.warnings:
self.add_operation_additional_info(validation_result.summary())
return [surface]
except ParseException as excep:
self.log.exception(excep)
raise LaunchException(excep)
def __get_range_values(self, kwargs, ranger_name):
"""
For the ranger given by ranger_name look in kwargs and return
the array with all the possible values.
"""
if ranger_name not in kwargs:
return None
if str(kwargs[ranger_name]) not in kwargs:
return None
range_values = []
try:
range_data = json.loads(str(kwargs[str(kwargs[ranger_name])]))
except Exception:
try:
range_data = [x.strip() for x in str(kwargs[str(kwargs[ranger_name])]).split(',') if len(x.strip()) > 0]
return range_data
except Exception, excep:
self.logger.warning("Could not launch operation !")
self.logger.exception(excep)
raise LaunchException("Could not launch with no data from:" + str(ranger_name))
def create_region_ts(self, data_shape, connectivity):
if connectivity.number_of_regions != data_shape[1]:
raise LaunchException(
"Data has %d channels but the connectivity has %d nodes" %
(data_shape[1], connectivity.number_of_regions))
ts_idx = TimeSeriesRegionIndex()
ts_idx.fk_connectivity_gid = connectivity.gid
region_map_indexes = dao.get_generic_entity(RegionMappingIndex,
connectivity.gid,
'fk_connectivity_gid')
ts_idx.has_surface_mapping = False
if len(region_map_indexes) > 0:
ts_idx.fk_region_mapping_gid = region_map_indexes[0].gid
ts_idx.has_surface_mapping = True
ts_h5_path = self.path_for(TimeSeriesRegionH5, ts_idx.gid)
ts_h5 = TimeSeriesRegionH5(ts_h5_path)
ts_h5.connectivity.store(uuid.UUID(connectivity.gid))
return TimeSeriesRegion(), ts_idx, ts_h5
def configure(self, simulator_gid):
"""
Make preparations for the adapter launch.
"""
self.log.debug("%s: Configuring simulator adapter..." % str(self))
self.algorithm, history_gid = SimulatorService().deserialize_simulator(
simulator_gid, self.storage_path)
self.branch_simulation_state_gid = history_gid
# for monitor in self.algorithm.monitors:
# if issubclass(monitor, Projection):
# # TODO: add a service that loads a RM with Surface and Connectivity
# pass
try:
self.algorithm.preconfigure()
except ValueError as err:
raise LaunchException(
"Failed to configure simulator due to invalid Input Values. It could be because "
"of an incompatibility between different version of TVB code.",
err)
def configure(self, view_model):
# type: (SimulatorAdapterModel) -> None
"""
Make preparations for the adapter launch.
"""
self.log.debug("%s: Configuring simulator adapter..." % str(self))
self.algorithm = self._prepare_simulator_from_view_model(view_model)
self.branch_simulation_state_gid = view_model.history_gid
# for monitor in self.algorithm.monitors:
# if issubclass(monitor, Projection):
# # TODO: add a service that loads a RM with Surface and Connectivity
# pass
try:
self.algorithm.preconfigure()
except ValueError as err:
raise LaunchException(
"Failed to configure simulator due to invalid Input Values. It could be because "
"of an incompatibility between different version of TVB code.",
err)
def launch(self, view_model):
# type: (GIFTISurfaceImporterModel) -> [SurfaceIndex]
"""
Execute import operations:
"""
parser = GIFTIParser(self.storage_path, self.operation_id)
try:
surface = parser.parse(view_model.data_file, view_model.data_file_part2, view_model.file_type,
should_center=view_model.should_center)
surface.compute_triangle_normals()
surface.compute_vertex_normals()
validation_result = surface.validate()
if validation_result.warnings:
self.add_operation_additional_info(validation_result.summary())
surface_idx = h5.store_complete(surface, self.storage_path)
return [surface_idx]
except ParseException as excep:
logger = get_logger(__name__)
logger.exception(excep)
raise LaunchException(excep)
def _ensure_region_mapping_index(self,
region_mapping_volume,
measure=None):
# type: (RegionVolumeMappingIndex, DataTypeMatrix) -> RegionVolumeMappingIndex
if region_mapping_volume is None:
if measure is not None and hasattr(measure, "fk_connectivity_gid"):
region_mapping_volume = dao.get_generic_entity(
RegionVolumeMappingIndex, measure.fk_connectivity_gid,
'fk_connectivity_gid')
if region_mapping_volume is not None and len(
region_mapping_volume):
region_mapping_volume = region_mapping_volume[0]
else:
region_mapping_volume = None
else:
region_mapping_volume = dao.try_load_last_entity_of_type(
self.current_project_id, RegionVolumeMappingIndex)
if region_mapping_volume is None:
raise LaunchException(
'You should have a compatible volume mapping to launch this viewer'
)
return region_mapping_volume
def launch(self,
projection_file,
surface,
sensors,
dataset_name=DEFAULT_DATASET_NAME):
"""
Creates ProjectionMatrix entity from uploaded data.
"""
if surface is None:
raise LaunchException(
"No source selected. Please initiate upload again and select a source."
)
if isinstance(sensors, SensorsEEG):
projection_matrix = ProjectionSurfaceEEG(
storage_path=self.storage_path)
else:
projection_matrix = ProjectionSurfaceMEG(
storage_path=self.storage_path)
expected_shape = surface.number_of_vertices
return self.generic_launch(projection_file, dataset_name, surface,
sensors, expected_shape, projection_matrix)
def _decrypt_content(view_model, trait_upload_field_name):
if TvbProfile.current.UPLOAD_KEY_PATH is None or not os.path.exists(
TvbProfile.current.UPLOAD_KEY_PATH):
raise LaunchException(
"We can not process Encrypted files at this moment, "
"due to missing PK for decryption! Please contact the administrator!"
)
upload_path = getattr(view_model, trait_upload_field_name)
# Get the encrypted password
with open(view_model.encrypted_aes_key, 'rb') as f:
encrypted_password = f.read()
# Read the private key
with open(TvbProfile.current.UPLOAD_KEY_PATH, "rb") as key_file:
private_key = serialization.load_pem_private_key(
key_file.read(), password=None, backend=default_backend())
# Decrypt the password using the private key
decrypted_password = private_key.decrypt(
encrypted_password,
padding.OAEP(mgf=padding.MGF1(algorithm=hashes.SHA256()),
algorithm=hashes.SHA256(),
label=None))
decrypted_password = decrypted_password.decode()
# Get path to decrypted file
decrypted_download_path = upload_path.replace(ENCRYPTED_DATA_SUFFIX,
DECRYPTED_DATA_SUFFIX)
# Use the decrypted password to decrypt the message
pyAesCrypt.decryptFile(upload_path, decrypted_download_path,
decrypted_password,
TvbProfile.current.hpc.CRYPT_BUFFER_SIZE)
view_model.__setattr__(trait_upload_field_name,
decrypted_download_path)
def _fit_topology(self, figure, topography, topography_data, color_bar_min,
color_bar_max):
"""
Trim data, to make sure everything is inside the head contour.
"""
x_arr = topography_data["x_arr"]
y_arr = topography_data["y_arr"]
circle_x = topography_data["circle_x"]
circle_y = topography_data["circle_y"]
rad = topography_data["rad"]
try:
topo = griddata(topography_data["sproj"][:, 0],
topography_data["sproj"][:, 1],
numpy.ravel(numpy.array(topography)),
x_arr,
y_arr,
interp='linear')
except KeyError, err:
self.log.exception("Could not execute matplotlib.mlab.griddata...")
raise LaunchException(
"The measure points location is not compatible with this viewer "
"(maybe all on one line)!", err)
def compute_parameters(self, input_data, colors=None, rays=None, step=None):
"""
Build the required HTML response to be displayed.
:raises LaunchException: when number of regions in input_data is less than 3
"""
if input_data.number_of_regions <= 3:
raise LaunchException('The connectivity matrix you selected has fewer nodes than acceptable for display!')
normalized_weights = self._normalize_weights(input_data.ordered_weights)
weights = Connectivity2DViewer._get_weights(normalized_weights, input_data.hemispheres)
# Compute shapes and colors ad adjacent data
norm_rays, min_ray, max_ray = self._normalize_rays(rays, input_data.number_of_regions)
colors, step = self._prepare_colors(colors, input_data.number_of_regions, step)
if numpy.all((input_data.hemispheres == False)):
right_json = ""
else:
right_json = self._get_json(input_data.ordered_labels[input_data.hemispheres],
input_data.ordered_centres[input_data.hemispheres], weights[1],
math.pi, 1, 2, numpy.asarray(norm_rays)[input_data.hemispheres],
numpy.asarray(colors)[input_data.hemispheres], X_CANVAS_SMALL, Y_CANVAS_SMALL)
if numpy.all((input_data.hemispheres == True)):
left_json = ""
else:
left_json = self._get_json(input_data.ordered_labels[~input_data.hemispheres],
input_data.ordered_centres[~input_data.hemispheres], weights[0],
math.pi, 1, 2, numpy.asarray(norm_rays)[~input_data.hemispheres],
numpy.asarray(colors)[~input_data.hemispheres], X_CANVAS_SMALL, Y_CANVAS_SMALL)
full_json = self._get_json(input_data.ordered_labels, input_data.ordered_centres, normalized_weights,
math.pi, 0, 1, norm_rays, colors, X_CANVAS_FULL, Y_CANVAS_FULL)
params = dict(bothHemisphereJson=full_json, rightHemisphereJson=right_json, leftHemisphereJson=left_json,
stepValue=step or max_ray, firstColor=self.DEFAULT_COLOR,
secondColor=self.OTHER_COLOR, minRay=min_ray, maxRay=max_ray)
return params, {}
def _fill_apriori_data(self, operations):
""" Gather apriori data from the operations. Also gather the datatype gid's"""
for metric in self.metrics:
self.apriori_data[metric] = numpy.zeros((self.apriori_x.size, self.apriori_y.size))
# An 2D array of GIDs which is used later to launch overlay for a DataType
self.datatypes_gids = [[None for _ in self.range2] for _ in self.range1]
for operation in operations:
self.log.debug("Gathering data from operation : %s" % operation.id)
range_values = eval(operation.range_values)
key_1 = range_values[self.range1_name]
index_x = self.range1.index(key_1)
key_2 = range_values[self.range2_name]
index_y = self.range2.index(key_2)
if operation.status == STATUS_STARTED:
raise LaunchException("Not all operations from this range are complete. Cannot view until then.")
operation_results = dao.get_results_for_operation(operation.id)
if operation_results:
datatype = operation_results[0]
self.datatypes_gids[index_x][index_y] = datatype.gid
if datatype.type == "DatatypeMeasure":
measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
else:
measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
else:
self.datatypes_gids[index_x][index_y] = None
measures = None
for metric in self.metrics:
if measures:
self.apriori_data[metric][index_x][index_y] = measures[0].metrics[metric]
else:
self.apriori_data[metric][index_x][index_y] = numpy.NaN
def launch(self,
file_type,
data_file,
data_file_part2,
should_center=False):
"""
Execute import operations:
"""
parser = GIFTIParser(self.storage_path, self.operation_id)
try:
surface = parser.parse(data_file,
data_file_part2,
file_type,
should_center=should_center)
validation_result = surface.validate()
if validation_result.warnings:
self.add_operation_additional_info(validation_result.summary())
return [surface]
except ParseException, excep:
logger = get_logger(__name__)
logger.exception(excep)
raise LaunchException(excep)
def launch(self, annotations, region_map=None, **kwarg):
if region_map is None:
region_map = dao.get_generic_entity(RegionMapping, annotations.connectivity.gid, '_connectivity')
if len(region_map) < 1:
raise LaunchException(
"Can not launch this viewer unless we have at least a RegionMapping for the current Connectivity!")
region_map = region_map[0]
boundary_url = region_map.surface.get_url_for_region_boundaries(region_map)
url_vertices_pick, url_normals_pick, url_triangles_pick = region_map.surface.get_urls_for_pick_rendering()
url_vertices, url_normals, _, url_triangles, url_region_map = \
region_map.surface.get_urls_for_rendering(True, region_map)
params = dict(title="Connectivity Annotations Visualizer",
baseUrl=TvbProfile.current.web.BASE_URL,
annotationsTreeUrl=self.paths2url(annotations, 'tree_json'),
urlTriangleToRegion=self.paths2url(region_map, "get_triangles_mapping"),
urlActivationPatterns=self.paths2url(annotations, "get_activation_patterns"),
minValue=0,
maxValue=annotations.connectivity.number_of_regions - 1,
urlColors=json.dumps(url_region_map),
urlVerticesPick=json.dumps(url_vertices_pick),
urlTrianglesPick=json.dumps(url_triangles_pick),
urlNormalsPick=json.dumps(url_normals_pick),
brainCenter=json.dumps(region_map.surface.center()),
urlVertices=json.dumps(url_vertices),
urlTriangles=json.dumps(url_triangles),
urlNormals=json.dumps(url_normals),
urlRegionBoundaries=boundary_url)
return self.build_display_result("annotations/annotations_view", params,
pages={"controlPage": "annotations/annotations_controls"})
def launch(self, projection_file, surface, sensors, dataset_name=DEFAULT_DATASET_NAME):
"""
Creates ProjectionMatrix entity from uploaded data.
:raises LaunchException: when
* no projection_file or sensors are specified
* the dataset is invalid
* number of sensors is different from the one in dataset
"""
if projection_file is None:
raise LaunchException("Please select MATLAB file which contains data to import")
if sensors is None:
raise LaunchException("No sensors selected. Please initiate upload again and select one.")
if surface is None:
raise LaunchException("No source selected. Please initiate upload again and select a source.")
expected_shape = surface.number_of_vertices
self.logger.debug("Reading projection matrix from uploaded file...")
if projection_file.endswith(".mat"):
eeg_projection_data = self.read_matlab_data(projection_file, dataset_name)
else:
eeg_projection_data = self.read_list_data(projection_file)
if eeg_projection_data is None or len(eeg_projection_data) == 0:
raise LaunchException("Invalid (empty) dataset...")
if eeg_projection_data.shape[0] != sensors.number_of_sensors:
raise LaunchException("Invalid Projection Matrix shape[0]: %d Expected: %d" % (eeg_projection_data.shape[0],
sensors.number_of_sensors))
if eeg_projection_data.shape[1] != expected_shape:
raise LaunchException("Invalid Projection Matrix shape[1]: %d Expected: %d" % (eeg_projection_data.shape[1],
expected_shape))
self.logger.debug("Creating Projection Matrix instance")
projection_matrix = build_projection_instance(sensors, self.storage_path)
projection_matrix.sources = surface
projection_matrix.sensors = sensors
if eeg_projection_data is not None:
projection_matrix.projection_data = eeg_projection_data
return [projection_matrix]
def prepare_operations(self,
user_id,
project_id,
algorithm,
category,
metadata,
visible=True,
**kwargs):
"""
Do all the necessary preparations for storing an operation. If it's the case of a
range of values create an operation group and multiple operations for each possible
instance from the range.
:param metadata: Initial MetaData with potential Burst identification inside.
"""
operations = []
available_args, group = self._prepare_group(project_id, kwargs)
if len(available_args) > TvbProfile.current.MAX_RANGE_NUMBER:
raise LaunchException(
"Too big range specified. You should limit the"
" resulting operations to %d" %
TvbProfile.current.MAX_RANGE_NUMBER)
else:
self.logger.debug("Launching a range with %d operations..." %
len(available_args))
group_id = None
if group is not None:
group_id = group.id
metadata, user_group = self._prepare_metadata(metadata, category,
group, kwargs)
self.logger.debug("Saving Operation(userId=" + str(user_id) +
",projectId=" + str(project_id) + "," +
str(metadata) + ",algorithmId=" + str(algorithm.id) +
", ops_group= " + str(group_id) + ")")
visible_operation = visible and category.display is False
meta_str = json.dumps(metadata)
for (one_set_of_args, range_vals) in available_args:
range_values = json.dumps(range_vals) if range_vals else None
operation = model.Operation(user_id,
project_id,
algorithm.id,
json.dumps(
one_set_of_args,
cls=MapAsJson.MapAsJsonEncoder),
meta_str,
op_group_id=group_id,
user_group=user_group,
range_values=range_values)
operation.visible = visible_operation
operations.append(operation)
operations = dao.store_entities(operations)
if group is not None:
burst_id = None
if DataTypeMetaData.KEY_BURST in metadata:
burst_id = metadata[DataTypeMetaData.KEY_BURST]
datatype_group = model.DataTypeGroup(
group,
operation_id=operations[0].id,
fk_parent_burst=burst_id,
state=metadata[DataTypeMetaData.KEY_STATE])
dao.store_entity(datatype_group)
return operations, group
def create_eeg_ts(self, data, sensors):
if sensors.number_of_sensors != data.shape[1]:
raise LaunchException(
"Data has %d channels but the sensors have %d" %
(data.shape[1], sensors.number_of_sensors))
return TimeSeriesEEG(storage_path=self.storage_path, sensors=sensors)
def launch(self, view_model):
# type: (TopographicViewerModel) -> dict
connectivities_idx = []
measures_ht = []
for measure in [
view_model.data_0, view_model.data_1, view_model.data_2
]:
if measure is not None:
measure_index = self.load_entity_by_gid(measure)
measures_ht.append(h5.load_from_index(measure_index))
conn_index = self.load_entity_by_gid(
measure_index.fk_connectivity_gid)
connectivities_idx.append(conn_index)
with h5.h5_file_for_index(connectivities_idx[0]) as conn_h5:
centres = conn_h5.centres.load()
sensor_locations = TopographyCalculations.normalize_sensors(centres)
sensor_number = len(sensor_locations)
arrays = []
titles = []
min_vals = []
max_vals = []
data_array = []
data_arrays = []
for i, measure in enumerate(measures_ht):
if connectivities_idx[i].number_of_regions != sensor_number:
raise Exception("Use the same connectivity!!!")
arrays.append(measure.array_data.tolist())
titles.append(measure.title)
min_vals.append(measure.array_data.min())
max_vals.append(measure.array_data.max())
color_bar_min = min(min_vals)
color_bar_max = max(max_vals)
for i, array_data in enumerate(arrays):
try:
data_array = TopographyCalculations.compute_topography_data(
array_data, sensor_locations)
# We always access the first element because only one connectivity can be used at one time
first_label = h5.load_from_index(
connectivities_idx[0]).hemispheres[0]
if first_label:
data_array = numpy.rot90(data_array, k=1, axes=(0, 1))
else:
data_array = numpy.rot90(data_array, k=-1, axes=(0, 1))
if numpy.any(numpy.isnan(array_data)):
titles[i] = titles[i] + " - Topography contains nan"
if not numpy.any(array_data):
titles[i] = titles[i] + " - Topography data is all zeros"
data_arrays.append(
ABCDisplayer.dump_with_precision(data_array.flat))
except KeyError as err:
self.log.exception(err)
raise LaunchException(
"The measure points location is not compatible with this viewer ",
err)
params = dict(matrix_datas=data_arrays,
matrix_shape=json.dumps(data_array.squeeze().shape),
titles=titles,
vmin=color_bar_min,
vmax=color_bar_max)
return self.build_display_result(
"topographic/view",
params,
pages={"controlPage": "topographic/controls"})
def launch(self,
uploaded,
surface_type,
zero_based_triangles=False,
should_center=False):
"""
Execute import operations: unpack ZIP and build Surface object as result.
:param uploaded: an archive containing the Surface data to be imported
:param surface_type: a string from the following\: \
"Skin Air", "Skull Skin", "Brain Skull", "Cortical Surface", "EEG Cap", "Face"
:returns: a subclass of `Surface` DataType
:raises LaunchException: when
* `uploaded` is missing
* `surface_type` is invalid
:raises RuntimeError: when triangles contain an invalid vertex index
"""
if uploaded is None:
raise LaunchException(
"Please select ZIP file which contains data to import")
self.logger.debug("Start to import surface: '%s' from file: %s" %
(surface_type, uploaded))
try:
zip_surface = ZipSurfaceParser(uploaded)
except IOError:
exception_str = "Did not find the specified ZIP at %s" % uploaded
raise LaunchException(exception_str)
# Detect and instantiate correct surface type
self.logger.debug("Create surface instance")
surface = self._make_surface(surface_type)
surface.storage_path = self.storage_path
surface.zero_based_triangles = zero_based_triangles
if should_center:
vertices = center_vertices(zip_surface.vertices)
else:
vertices = zip_surface.vertices
surface.vertices = vertices
if len(zip_surface.normals) != 0:
surface.vertex_normals = zip_surface.normals
if zero_based_triangles:
surface.triangles = zip_surface.triangles
else:
surface.triangles = zip_surface.triangles - 1
if zip_surface.bi_hemispheric:
self.logger.info("Hemispheres detected")
surface.hemisphere_mask = zip_surface.hemisphere_mask
surface.triangle_normals = None
# Now check if the triangles of the surface are valid
triangles_min_vertex = numpy.amin(surface.triangles)
if triangles_min_vertex < 0:
if triangles_min_vertex == -1 and not zero_based_triangles:
raise LaunchException(
"Triangles contain a negative vertex index. Maybe you have a ZERO based surface."
)
else:
raise LaunchException(
"Your triangles contain a negative vertex index: %d" %
triangles_min_vertex)
no_of_vertices = len(surface.vertices)
triangles_max_vertex = numpy.amax(surface.triangles)
if triangles_max_vertex >= no_of_vertices:
if triangles_max_vertex == no_of_vertices and zero_based_triangles:
raise LaunchException(
"Your triangles contain an invalid vertex index: %d. "
"Maybe your surface is NOT ZERO Based." %
triangles_max_vertex)
else:
raise LaunchException(
"Your triangles contain an invalid vertex index: %d." %
triangles_max_vertex)
validation_result = surface.validate()
if validation_result.warnings:
self.add_operation_additional_info(validation_result.summary())
self.logger.debug("Surface ready to be stored")
return surface
def launch(self, view_model):
# type: (RegionMappingImporterModel) -> [RegionMappingIndex]
"""
Creates region mapping from uploaded data.
:raises LaunchException: when
* a parameter is None or missing
* archive has more than one file
* uploaded files are empty
* number of vertices in imported file is different to the number of surface vertices
* imported file has negative values
* imported file has regions which are not in connectivity
"""
if view_model.mapping_file is None:
raise LaunchException(
"Please select mappings file which contains data to import")
if view_model.surface is None:
raise LaunchException(
"No surface selected. Please initiate upload again and select a brain surface."
)
if view_model.connectivity is None:
raise LaunchException(
"No connectivity selected. Please initiate upload again and select one."
)
self.logger.debug("Reading mappings from uploaded file")
if zipfile.is_zipfile(view_model.mapping_file):
tmp_folder = tempfile.mkdtemp(
prefix='region_mapping_zip_',
dir=TvbProfile.current.TVB_TEMP_FOLDER)
try:
files = FilesHelper().unpack_zip(view_model.mapping_file,
tmp_folder)
if len(files) > 1:
raise LaunchException(
"Please upload a ZIP file containing only one file.")
array_data = self.read_list_data(files[0], dtype=numpy.int32)
finally:
if os.path.exists(tmp_folder):
shutil.rmtree(tmp_folder)
else:
array_data = self.read_list_data(view_model.mapping_file,
dtype=numpy.int32)
# Now we do some checks before building final RegionMapping
if array_data is None or len(array_data) == 0:
raise LaunchException(
"Uploaded file does not contains any data. Please initiate upload with another file."
)
# Check if we have a mapping for each surface vertex.
surface_index = self.load_entity_by_gid(view_model.surface)
if len(array_data) != surface_index.number_of_vertices:
msg = "Imported file contains a different number of values than the number of surface vertices. " \
"Imported: %d values while surface has: %d vertices." % (
len(array_data), surface_index.number_of_vertices)
raise LaunchException(msg)
# Now check if the values from imported file correspond to connectivity regions
if array_data.min() < 0:
raise LaunchException(
"Imported file contains negative values. Please fix problem and re-import file"
)
connectivity_index = self.load_entity_by_gid(view_model.connectivity)
if array_data.max() >= connectivity_index.number_of_regions:
msg = "Imported file contains invalid regions. Found region: %d while selected connectivity has: %d " \
"regions defined (0 based)." % (array_data.max(), connectivity_index.number_of_regions)
raise LaunchException(msg)
self.logger.debug("Creating RegionMapping instance")
connectivity_ht = h5.load_from_index(connectivity_index)
surface_ht = h5.load_from_index(surface_index)
region_mapping = RegionMapping(surface=surface_ht,
connectivity=connectivity_ht,
array_data=array_data)
return h5.store_complete(region_mapping, self.storage_path)
def plot(self, figure, operation_group, metric, range1_name, range2_name):
"""
Do the plot for the given figure. Also need operation group, metric and ranges
in order to compute the data to be plotted.
"""
operations = dao.get_operations_in_group(operation_group.id)
# Data from which to interpolate larger 2-D space
apriori_x = numpy.array(self.range1)
apriori_y = numpy.array(self.range2)
apriori_data = numpy.zeros((apriori_x.size, apriori_y.size))
# An 2D array of GIDs which is used later to launch overlay for a DataType
datatypes_gids = [[None for _ in self.range2] for _ in self.range1]
for operation_ in operations:
range_values = eval(operation_.range_values)
key_1 = range_values[range1_name]
index_x = self.range1.index(key_1)
key_2 = range_values[range2_name]
index_y = self.range2.index(key_2)
if operation_.status != model.STATUS_FINISHED:
raise LaunchException("Not all operations from this range are complete. Cannot view until then.")
datatype = dao.get_results_for_operation(operation_.id)[0]
datatypes_gids[index_x][index_y] = datatype.gid
if datatype.type == "DatatypeMeasure":
measures = dao.get_generic_entity(DatatypeMeasure, datatype.id)
else:
measures = dao.get_generic_entity(DatatypeMeasure, datatype.gid, '_analyzed_datatype')
if measures:
apriori_data[index_x][index_y] = measures[0].metrics[metric]
else:
apriori_data[index_x][index_y] = 0
# Attempt order-3 interpolation.
kx = ky = 3
if len(self.range1) <= 3 or len(self.range2) <= 3:
# Number of points is too small, just do linear interpolation
kx = ky = 1
s = interpolate.RectBivariateSpline(apriori_x, apriori_y, apriori_data, kx=kx, ky=ky)
# Get data of higher resolution that we'll plot later on
posteriori_x = numpy.arange(self.range1[0], self.range1[-1],
(self.range1[-1] - self.range1[0]) / RESOLUTION[0])
posteriori_y = numpy.arange(self.range2[0], self.range2[-1],
(self.range2[-1] - self.range2[0]) / RESOLUTION[1])
posteriori_data = numpy.rot90(s(posteriori_x, posteriori_y))
self.interp_models[figure.number] = s
# Do actual plot.
axes = figure.gca()
img = axes.imshow(posteriori_data, extent=(min(self.range1), max(self.range1),
min(self.range2), max(self.range2)),
aspect='auto', interpolation='nearest')
axes.set_title("Interpolated values for metric %s" % (metric,))
figure.colorbar(img)
axes.set_xlabel(range1_name)
axes.set_ylabel(range2_name)
def format_coord(x, y):
return 'x=%1.4f, y=%1.4f' % (x, y)
axes.format_coord = format_coord
return datatypes_gids
def launch(self, view_model):
# type: (ZIPSurfaceImporterModel) -> [SurfaceIndex]
"""
Execute import operations: unpack ZIP and build Surface object as result
:raises LaunchException: when
* `uploaded` is missing
* `surface_type` is invalid
:raises RuntimeError: when triangles contain an invalid vertex index
"""
if view_model.uploaded is None:
raise LaunchException(
"Please select ZIP file which contains data to import")
self.logger.debug("Start to import surface: '%s' from file: %s" %
(view_model.surface_type, view_model.uploaded))
try:
zip_surface = ZipSurfaceParser(view_model.uploaded)
except IOError:
exception_str = "Did not find the specified ZIP at %s" % view_model.uploaded
raise LaunchException(exception_str)
# Detect and instantiate correct surface type
self.logger.debug("Create surface instance")
surface = self._make_surface(view_model.surface_type)
surface.zero_based_triangles = view_model.zero_based_triangles
if view_model.should_center:
vertices = center_vertices(zip_surface.vertices)
else:
vertices = zip_surface.vertices
surface.vertices = vertices
if len(zip_surface.normals) != 0:
surface.vertex_normals = zip_surface.normals
if view_model.zero_based_triangles:
surface.triangles = zip_surface.triangles
else:
surface.triangles = zip_surface.triangles - 1
if zip_surface.bi_hemispheric:
self.logger.info("Hemispheres detected")
surface.hemisphere_mask = zip_surface.hemisphere_mask
surface.compute_triangle_normals()
# Now check if the triangles of the surface are valid
triangles_min_vertex = numpy.amin(surface.triangles)
if triangles_min_vertex < 0:
if triangles_min_vertex == -1 and not view_model.zero_based_triangles:
raise LaunchException(
"Triangles contain a negative vertex index. Maybe you have a ZERO based surface."
)
else:
raise LaunchException(
"Your triangles contain a negative vertex index: %d" %
triangles_min_vertex)
no_of_vertices = len(surface.vertices)
triangles_max_vertex = numpy.amax(surface.triangles)
if triangles_max_vertex >= no_of_vertices:
if triangles_max_vertex == no_of_vertices and view_model.zero_based_triangles:
raise LaunchException(
"Your triangles contain an invalid vertex index: %d. "
"Maybe your surface is NOT ZERO Based." %
triangles_max_vertex)
else:
raise LaunchException(
"Your triangles contain an invalid vertex index: %d." %
triangles_max_vertex)
validation_result = surface.validate()
if validation_result.warnings:
self.add_operation_additional_info(validation_result.summary())
surface.configure()
self.logger.debug("Surface ready to be stored")
surf_idx = h5.store_complete(surface, self.storage_path)
self.generic_attributes.user_tag_1 = surface.surface_type
return surf_idx
def generate_preview(self, **kwargs):
"""
Should be implemented by all visualizers that can be used by portlets.
"""
raise LaunchException("%s used as Portlet but doesn't implement 'generate_preview'" % self.__class__)
def configure(self,
model,
model_parameters,
integrator,
integrator_parameters,
connectivity,
monitors,
monitors_parameters=None,
surface=None,
surface_parameters=None,
stimulus=None,
coupling=None,
coupling_parameters=None,
initial_conditions=None,
conduction_speed=None,
simulation_length=0,
simulation_state=None):
"""
Make preparations for the adapter launch.
"""
self.log.debug("available_couplings: %s..." %
str(self.available_couplings))
self.log.debug("coupling: %s..." % str(coupling))
self.log.debug("coupling_parameters: %s..." % str(coupling_parameters))
self.log.debug("%s: Initializing Model..." % str(self))
noise_framework.build_noise(model_parameters)
model_instance = self.available_models[str(model)](**model_parameters)
self._validate_model_parameters(model_instance, connectivity, surface)
self.log.debug("%s: Initializing Integration scheme..." % str(self))
noise_framework.build_noise(integrator_parameters)
integr = self.available_integrators[integrator](
**integrator_parameters)
self.log.debug("%s: Instantiating Monitors..." % str(self))
monitors_list = []
for monitor_name in monitors:
if (monitors_parameters is not None) and (str(monitor_name)
in monitors_parameters):
current_monitor_parameters = monitors_parameters[str(
monitor_name)]
HRFKernelEquation.build_equation_from_dict(
'hrf_kernel', current_monitor_parameters, True)
monitors_list.append(
self.available_monitors[str(monitor_name)](
**current_monitor_parameters))
else:
### We have monitors without any UI settable parameter.
monitors_list.append(
self.available_monitors[str(monitor_name)]())
if len(monitors) < 1:
raise LaunchException(
"Can not launch operation without monitors selected !!!")
self.log.debug("%s: Initializing Coupling..." % str(self))
coupling_inst = self.available_couplings[str(coupling)](
**coupling_parameters)
self.log.debug("Initializing Cortex...")
if self._is_surface_simulation(surface, surface_parameters):
cortex_entity = Cortex(use_storage=False).populate_cortex(
surface, surface_parameters)
if cortex_entity.region_mapping_data.connectivity.number_of_regions != connectivity.number_of_regions:
raise LaunchException(
"Incompatible RegionMapping -- Connectivity !!")
if cortex_entity.region_mapping_data.surface.number_of_vertices != surface.number_of_vertices:
raise LaunchException(
"Incompatible RegionMapping -- Surface !!")
select_loc_conn = cortex_entity.local_connectivity
if select_loc_conn is not None and select_loc_conn.surface.number_of_vertices != surface.number_of_vertices:
raise LaunchException(
"Incompatible LocalConnectivity -- Surface !!")
else:
cortex_entity = None
self.log.debug("%s: Instantiating requested simulator..." % str(self))
if conduction_speed not in (0.0, None):
connectivity.speed = numpy.array([conduction_speed])
else:
raise LaunchException("conduction speed cannot be 0 or missing")
self.algorithm = Simulator(connectivity=connectivity,
coupling=coupling_inst,
surface=cortex_entity,
stimulus=stimulus,
model=model_instance,
integrator=integr,
monitors=monitors_list,
initial_conditions=initial_conditions,
conduction_speed=conduction_speed)
self.simulation_length = simulation_length
self.log.debug("%s: Initializing storage..." % str(self))
try:
self.algorithm.preconfigure()
except ValueError as err:
raise LaunchException(
"Failed to configure simulator due to invalid Input Values. It could be because "
"of an incompatibility between different version of TVB code.",
err)
class OperationService:
"""
Class responsible for preparing an operation launch.
It will prepare parameters, and decide if the operation is to be executed
immediately, or to be sent on the cluster.
"""
ATT_UID = "uid"
def __init__(self):
self.logger = get_logger(self.__class__.__module__)
self.workflow_service = WorkflowService()
self.file_helper = FilesHelper()
##########################################################################################
######## Methods related to launching operations start here ##############################
##########################################################################################
def initiate_operation(self,
current_user,
project_id,
adapter_instance,
temporary_storage,
visible=True,
**kwargs):
"""
Gets the parameters of the computation from the previous inputs form,
and launches a computation (on the cluster or locally).
Invoke custom method on an Adapter Instance. Make sure when the
operation has finished that the correct results are stored into DB.
"""
if not isinstance(adapter_instance, ABCAdapter):
self.logger.warning("Inconsistent Adapter Class:" +
str(adapter_instance.__class__))
raise LaunchException("Developer Exception!!")
# Prepare Files parameters
files = {}
kw2 = copy(kwargs)
for i, j in kwargs.iteritems():
if isinstance(j, FieldStorage) or isinstance(j, Part):
files[i] = j
del kw2[i]
temp_files = {}
try:
for i, j in files.iteritems():
if j.file is None:
kw2[i] = None
continue
uq_name = utils.date2string(datetime.now(),
True) + '_' + str(i)
# We have to add original file name to end, in case file processing
# involves file extension reading
file_name = TEMPORARY_PREFIX + uq_name + '_' + j.filename
file_name = os.path.join(temporary_storage, file_name)
kw2[i] = file_name
temp_files[i] = file_name
with open(file_name, 'wb') as file_obj:
file_obj.write(j.file.read())
self.logger.debug("Will store file:" + file_name)
kwargs = kw2
except Exception, excep:
self._handle_exception(
excep, temp_files,
"Could not launch operation: invalid input files!")
### Store Operation entity.
algo = adapter_instance.stored_adapter
algo_category = dao.get_category_by_id(algo.fk_category)
operations = self.prepare_operations(current_user.id, project_id, algo,
algo_category, {}, visible,
**kwargs)[0]
if isinstance(adapter_instance, ABCSynchronous):
if len(operations) > 1:
raise LaunchException(
"Synchronous operations are not supporting ranges!")
if len(operations) < 1:
self.logger.warning("No operation was defined")
raise LaunchException("Invalid empty Operation!!!")
return self.initiate_prelaunch(operations[0], adapter_instance,
temp_files, **kwargs)
else:
return self._send_to_cluster(operations, adapter_instance,
current_user.username)
def launch(self, view_model):
# type: (ConnectivityAnnotationsViewModel) -> dict
annotations_index = self.load_entity_by_gid(
view_model.annotations_index)
if view_model.connectivity_index is None:
connectivity_index = self.load_entity_by_gid(
annotations_index.fk_connectivity_gid)
else:
connectivity_index = self.load_entity_by_gid(
view_model.connectivity_index)
if view_model.region_mapping_index is None:
region_map = dao.get_generic_entity(RegionMappingIndex,
connectivity_index.gid,
'fk_connectivity_gid')
if len(region_map) < 1:
raise LaunchException(
"Can not launch this viewer unless we have at least a RegionMapping for the current Connectivity!"
)
region_mapping_index = region_map[0]
else:
region_mapping_index = self.load_entity_by_gid(
view_model.region_mapping_index)
boundary_url = SurfaceURLGenerator.get_url_for_region_boundaries(
region_mapping_index.fk_surface_gid, region_mapping_index.gid,
self.stored_adapter.id)
surface_index = self.load_entity_by_gid(
region_mapping_index.fk_surface_gid)
surface_h5 = h5.h5_file_for_index(surface_index)
assert isinstance(surface_h5, SurfaceH5)
url_vertices_pick, url_normals_pick, url_triangles_pick = SurfaceURLGenerator.get_urls_for_pick_rendering(
surface_h5)
url_vertices, url_normals, _, url_triangles, url_region_map = SurfaceURLGenerator.get_urls_for_rendering(
surface_h5, region_mapping_index.gid)
params = dict(title="Connectivity Annotations Visualizer",
annotationsTreeUrl=URLGenerator.build_url(
self.stored_adapter.id, 'tree_json',
view_model.annotations_index),
urlTriangleToRegion=URLGenerator.build_url(
self.stored_adapter.id, "get_triangles_mapping",
region_mapping_index.gid),
urlActivationPatterns=URLGenerator.paths2url(
view_model.annotations_index,
"get_activation_patterns"),
minValue=0,
maxValue=connectivity_index.number_of_regions - 1,
urlColors=json.dumps(url_region_map),
urlVerticesPick=json.dumps(url_vertices_pick),
urlTrianglesPick=json.dumps(url_triangles_pick),
urlNormalsPick=json.dumps(url_normals_pick),
brainCenter=json.dumps(surface_h5.center()),
urlVertices=json.dumps(url_vertices),
urlTriangles=json.dumps(url_triangles),
urlNormals=json.dumps(url_normals),
urlRegionBoundaries=boundary_url)
return self.build_display_result(
"annotations/annotations_view",
params,
pages={"controlPage": "annotations/annotations_controls"})