def export_data(self, data, exporter_id, project, user_public_key=None):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:param user_public_key: public key file used for encrypting data before exporting
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException("Could not export null data. Please select data to be exported")
if exporter_id is None:
raise ExportException("Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException("Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if user_public_key is not None:
public_key_path, encryption_password = self.storage_interface.prepare_encryption(project.name)
if isinstance(user_public_key, (FieldStorage, Part)):
with open(public_key_path, 'wb') as file_obj:
self.storage_interface.copy_file(user_public_key.file, file_obj)
else:
shutil.copy2(user_public_key, public_key_path)
else:
public_key_path, encryption_password = None, None
if project is None:
raise ExportException("Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException("Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
export_data = None
try:
self.logger.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, project, public_key_path, encryption_password)
except Exception:
pass
return export_data
def export_data(self, data, exporter_id, project):
"""
Export provided data using given exporter
:param data: data type to be exported
:param exporter_id: identifier of the exporter to be used
:param project: project that contains data to be exported
:returns: a tuple with the following elements
1. name of the file to be shown to user
2. full path of the export file (available for download)
3. boolean which specify if file can be deleted after download
"""
if data is None:
raise InvalidExportDataException(
"Could not export null data. Please select data to be exported"
)
if exporter_id is None:
raise ExportException(
"Please select the exporter to be used for this operation")
if exporter_id not in self.all_exporters:
raise ExportException(
"Provided exporter identifier is not a valid one")
exporter = self.all_exporters[exporter_id]
if project is None:
raise ExportException(
"Please provide the project where data files are stored")
# Now we start the real export
if not exporter.accepts(data):
raise InvalidExportDataException(
"Current data can not be exported by specified exporter")
# Now compute and create folder where to store exported data
# This will imply to generate a folder which is unique for each export
data_export_folder = None
try:
data_export_folder = self._build_data_export_folder(data)
# Here is the real export
LOG.debug("Start export of data: %s" % data.type)
export_data = exporter.export(data, data_export_folder, project)
finally:
# In case export did not generated any file delete folder
if data_export_folder is not None and len(
os.listdir(data_export_folder)) == 0:
os.rmdir(data_export_folder)
return export_data
def export(self, data, export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self.get_export_file_name(data)
files_helper = FilesHelper()
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data")
zip_file = os.path.join(export_folder, download_file_name)
# Now process each data type from group and add it to ZIP file
operation_folders = []
for data_type in all_datatypes:
operation_folder = files_helper.get_operation_folder(
project.name, data_type.fk_from_operation)
operation_folders.append(operation_folder)
# Create ZIP archive
files_helper.zip_folders(zip_file, operation_folders,
self.OPERATION_FOLDER_PREFIX)
return download_file_name, zip_file, True
else:
data_file = self.copy_dt_to_export_folder(data, export_folder)
return None, data_file, True
def export(self, data, export_folder, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self.get_export_file_name(data)
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data")
zip_file = os.path.join(export_folder, download_file_name)
# Create ZIP archive
self.storage_interface.zip_folders(all_datatypes, project.name,
zip_file)
return download_file_name, zip_file, True
else:
data_file = self.copy_dt_to_export_folder(data, export_folder)
return None, data_file, True
def export(self, data, project):
"""
Exports data type:
1. If data is a normal data type, simply exports storage file (HDF format)
2. If data is a DataTypeGroup creates a zip with all files for all data types
"""
download_file_name = self._get_export_file_name(data)
if self.is_data_a_group(data):
all_datatypes = self._get_all_data_types_arr(data)
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data")
# Create ZIP archive
zip_file = self.storage_interface.export_datatypes_structure(
all_datatypes, data, download_file_name, project.name)
return download_file_name, zip_file, True
else:
data_path = h5.path_for_stored_index(data)
data_file = self.storage_interface.export_datatypes([data_path],
data, None)
return None, data_file, True
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
:param project)name: name of the project to be exported
:param export_folder: folder where to store export result( e.g zip file)
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
bursts_dict = {}
datatype_burst_mapping = {}
bursts_count = dao.get_bursts_for_project(project.id, count=True)
for start_idx in range(0, bursts_count, BURST_PAGE_SIZE):
bursts = dao.get_bursts_for_project(project.id,
page_start=start_idx,
page_end=start_idx +
BURST_PAGE_SIZE)
for burst in bursts:
self._build_burst_export_dict(burst, bursts_dict)
datatypes_count = dao.get_datatypes_for_project(project.id, count=True)
for start_idx in range(0, datatypes_count, DATAYPES_PAGE_SIZE):
datatypes = dao.get_datatypes_for_project(project.id,
page_start=start_idx,
page_end=start_idx +
DATAYPES_PAGE_SIZE)
for datatype in datatypes:
datatype_burst_mapping[datatype.gid] = datatype.fk_parent_burst
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name,
self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
bursts_file_name = os.path.join(project_folder, BURST_INFO_FILE)
burst_info = {
BURSTS_DICT_KEY: bursts_dict,
DT_BURST_MAP: datatype_burst_mapping
}
with open(bursts_file_name, 'w') as bursts_file:
bursts_file.write(json.dumps(burst_info))
# pack project content into a ZIP file
result_zip = files_helper.zip_folder(result_path, project_folder)
# remove these files, since we only want them in export archive
os.remove(bursts_file_name)
return result_zip
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
files_helper = FilesHelper()
project_folder = files_helper.get_project_folder(project)
project_datatypes = self._gather_project_datatypes(project, optimize_size)
to_be_exported_folders = []
considered_op_ids = []
if optimize_size:
## take only the DataType with visibility flag set ON
for dt in project_datatypes:
if dt[KEY_OPERATION_ID] not in considered_op_ids:
to_be_exported_folders.append({'folder': files_helper.get_project_folder(project,
str(dt[KEY_OPERATION_ID])),
'archive_path_prefix': str(dt[KEY_OPERATION_ID]) + os.sep})
considered_op_ids.append(dt[KEY_OPERATION_ID])
else:
to_be_exported_folders.append({'folder': project_folder,
'archive_path_prefix': '', 'exclude': ["TEMP"]})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name, self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
LOG.debug("Done preparing, now we will write folders " + str(len(to_be_exported_folders)))
LOG.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
LOG.debug("Done exporting files, now we will write the burst configurations...")
self._export_bursts(project, project_datatypes, zip_file)
LOG.debug("Done exporting burst configurations, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
## Make sure the Project.xml file gets copied:
if optimize_size:
LOG.debug("Done linked, now we write the project xml")
zip_file.write(files_helper.get_project_meta_file_path(project.name), files_helper.TVB_PROJECT_FILE)
LOG.debug("Done, closing")
return result_path
def export_project(self, project):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
folders_to_exclude = self._get_op_with_errors(project.id)
linked_paths, op = self._get_paths_of_linked_datatypes(project)
result_path = self.storage_interface.export_project(project, folders_to_exclude, linked_paths, op)
return result_path
def prepare_datatypes_for_export(data):
"""
Method used for exporting data type groups. This method returns a list of all datatype indexes needed to be
exported and a dictionary where keys are operation folder names and the values are lists containing the paths
that belong to one particular operation folder.
"""
all_datatypes = ProjectService.get_all_datatypes_from_data(data)
first_datatype = all_datatypes[0]
# We are exporting a group of datatype measures so we need to find the group of time series
if hasattr(first_datatype, 'fk_source_gid'):
ts = h5.load_entity_by_gid(first_datatype.fk_source_gid)
dt_metric_group = dao.get_datatypegroup_by_op_group_id(
ts.parent_operation.fk_operation_group)
datatype_measure_list = ProjectService.get_all_datatypes_from_data(
dt_metric_group)
all_datatypes = datatype_measure_list + all_datatypes
else:
ts_group = dao.get_datatype_measure_group_from_ts_from_pse(
first_datatype.gid, DatatypeMeasureIndex)
time_series_list = ProjectService.get_all_datatypes_from_data(
ts_group)
all_datatypes = all_datatypes + time_series_list
if all_datatypes is None or len(all_datatypes) == 0:
raise ExportException(
"Could not export a data type group with no data!")
op_file_dict = dict()
for dt in all_datatypes:
h5_path = h5.path_for_stored_index(dt)
op_folder = os.path.dirname(h5_path)
op_file_dict[op_folder] = [h5_path]
op = dao.get_operation_by_id(dt.fk_from_operation)
vms = h5.gather_references_of_view_model(op.view_model_gid,
os.path.dirname(h5_path),
only_view_models=True)
op_file_dict[op_folder].extend(vms[0])
return all_datatypes, op_file_dict
def export_project(self, project, optimize_size=False):
"""
Given a project root and the TVB storage_path, create a ZIP
ready for export.
:param project: project object which identifies project to be exported
"""
if project is None:
raise ExportException("Please provide project to be exported")
project_folder = self.files_helper.get_project_folder(project)
project_datatypes = dao.get_datatypes_in_project(
project.id, only_visible=optimize_size)
to_be_exported_folders = []
considered_op_ids = []
folders_to_exclude = self._get_op_with_errors(project.id)
if optimize_size:
# take only the DataType with visibility flag set ON
for dt in project_datatypes:
op_id = dt.fk_from_operation
if op_id not in considered_op_ids:
to_be_exported_folders.append({
'folder':
self.files_helper.get_project_folder(
project, str(op_id)),
'archive_path_prefix':
str(op_id) + os.sep,
'exclude':
folders_to_exclude
})
considered_op_ids.append(op_id)
else:
folders_to_exclude.append("TEMP")
to_be_exported_folders.append({
'folder': project_folder,
'archive_path_prefix': '',
'exclude': folders_to_exclude
})
# Compute path and name of the zip file
now = datetime.now()
date_str = now.strftime("%Y-%m-%d_%H-%M")
zip_file_name = "%s_%s.%s" % (date_str, project.name,
self.ZIP_FILE_EXTENSION)
export_folder = self._build_data_export_folder(project)
result_path = os.path.join(export_folder, zip_file_name)
with TvbZip(result_path, "w") as zip_file:
# Pack project [filtered] content into a ZIP file:
self.logger.debug("Done preparing, now we will write folders " +
str(len(to_be_exported_folders)))
self.logger.debug(str(to_be_exported_folders))
for pack in to_be_exported_folders:
zip_file.write_folder(**pack)
self.logger.debug(
"Done exporting files, now we will export linked DTs")
self._export_linked_datatypes(project, zip_file)
# Make sure the Project.xml file gets copied:
if optimize_size:
self.logger.debug("Done linked, now we write the project xml")
zip_file.write(
self.files_helper.get_project_meta_file_path(project.name),
self.files_helper.TVB_PROJECT_FILE)
self.logger.debug("Done, closing")
return result_path
