def initialise_default_schemas(session):
"""
Initialise all default MethodSchema standardised fields (parent schemas).
:param session: Session to add the created schemas to the database with.
:return: None
"""
# -----------------Location Offset Schema----------------------
location_offsets_schema = MethodSchema()
location_offsets_schema.schema_type = DataEntrySchema.__xmlrpc_class__
location_offsets_schema.name = "XYZ Location Offsets"
location_offsets_schema.template_schema = True
x_offset_field = MethodSchemaField()
x_offset_field.type = Double.__xmlrpc_class__
x_offset_field.units = "meters"
x_offset_field.name = "X Offset"
x_offset_field.placeholder = "eg. 23.4"
x_offset_field.default = 0
location_offsets_schema.custom_fields.append(x_offset_field)
y_offset_field = MethodSchemaField()
y_offset_field.type = Double.__xmlrpc_class__
y_offset_field.units = "meters"
y_offset_field.name = "Z Offset"
y_offset_field.placeholder = "eg. 23.4"
y_offset_field.default = 0
location_offsets_schema.custom_fields.append(y_offset_field)
z_offset_field = MethodSchemaField()
z_offset_field.type = Double.__xmlrpc_class__
z_offset_field.units = "meters"
z_offset_field.name = "Z Offset"
z_offset_field.placeholder = "eg. 23.4"
z_offset_field.default = 0
location_offsets_schema.custom_fields.append(z_offset_field)
session.add(location_offsets_schema)
session.flush()
# ----------Temperature schema--------------
temp_schema = MethodSchema()
temp_schema.name = "Temperature"
temp_schema.template_schema = True
temp_schema.schema_type = DataEntrySchema.__xmlrpc_class__
temp_field = MethodSchemaField()
temp_field.type = "decimal"
temp_field.units = "Celcius"
temp_field.name = "Temperature"
temp_schema.custom_fields.append(temp_field)
session.add(temp_schema)
# ----------Humidity schema--------------
humidity_schema = MethodSchema()
humidity_schema.name = "Humidity"
humidity_schema.template_schema = True
humidity_schema.schema_type = DataEntrySchema.__xmlrpc_class__
humidity_field = MethodSchemaField()
humidity_field.type = "decimal"
humidity_field.units = "%"
humidity_field.name = "Humidity"
humidity_schema.custom_fields.append(humidity_field)
session.add(humidity_schema)
# ----------Moisture schema--------------
moisture_schema = MethodSchema()
moisture_schema.name = "Moisture"
moisture_schema.template_schema = True
moisture_schema.schema_type = DataEntrySchema.__xmlrpc_class__
moisture_field = MethodSchemaField()
moisture_field.type = "decimal"
moisture_field.units = "%"
moisture_field.name = "Moisture"
moisture_schema.custom_fields.append(moisture_field)
session.add(moisture_schema)
# ----------Altitude schema--------------
altitude_schema = MethodSchema()
altitude_schema.name = "Altitude"
altitude_schema.template_schema = True
altitude_schema.schema_type = DataEntrySchema.__xmlrpc_class__
altitude_field = MethodSchemaField()
altitude_field.type = "decimal"
altitude_field.units = "Meters above Mean Sea Level (MSL)"
altitude_field.name = "Altitude"
altitude_schema.custom_fields.append(altitude_field)
session.add(altitude_schema)
# ----------Distance schema--------------
distance_schema = MethodSchema()
distance_schema.name = "Distance"
distance_schema.template_schema = True
distance_schema.schema_type = DataEntrySchema.__xmlrpc_class__
distance_field = MethodSchemaField()
distance_field.type = "decimal"
distance_field.units = "Meters"
distance_field.name = "Distance"
distance_schema.custom_fields.append(distance_field)
session.add(distance_schema)
# ----------Light Intensity schema--------------
luminosity_schema = MethodSchema()
luminosity_schema.name = "Luminosity"
luminosity_schema.template_schema = True
luminosity_schema.schema_type = DataEntrySchema.__xmlrpc_class__
luminosity_field = MethodSchemaField()
luminosity_field.type = "decimal"
luminosity_field.units = "candela (cd)"
luminosity_field.name = "Luminosity"
luminosity_schema.custom_fields.append(luminosity_field)
session.add(luminosity_schema)
# ----------Weight schema--------------
weight_schema = MethodSchema()
weight_schema.name = "Weight"
weight_schema.template_schema = True
weight_schema.schema_type = DataEntrySchema.__xmlrpc_class__
weight_field = MethodSchemaField()
weight_field.type = "decimal"
weight_field.units = "kg"
weight_field.name = "Weight"
weight_schema.custom_fields.append(weight_field)
session.add(weight_schema)
# ----------Density schema--------------
density_schema = MethodSchema()
density_schema.name = "Density"
density_schema.template_schema = True
density_schema.schema_type = DataEntrySchema.__xmlrpc_class__
density_field = MethodSchemaField()
density_field.type = "decimal"
density_field.units = "kg/m^3"
density_field.name = "Density"
density_schema.custom_fields.append(density_field)
session.add(density_schema)
# ------------Data Quality Assurance Schema----------
data_quality = MethodSchema()
data_quality.name = "Data Quality"
data_quality.template_schema = False
data_quality.schema_type = DataEntryMetadataSchema.__xmlrpc_class__
quality_field = MethodSchemaField()
quality_field.type = "decimal"
quality_field.name = "Value"
data_quality.custom_fields.append(quality_field)
description_field = MethodSchemaField()
description_field.type = "text_area"
description_field.name = "Description"
data_quality.custom_fields.append(description_field)
session.add(data_quality)
# ------------Dataset calibration/changes schema-------------------
dataset_calibration = MethodSchema()
dataset_calibration.name = "Dataset Calibration"
dataset_calibration.template_schema = False
dataset_calibration.schema_type = DatasetMetadataSchema.__xmlrpc_class__
date = MethodSchemaField()
date.type = "date"
date.name = "Date"
dataset_calibration.custom_fields.append(date)
# Textual representation of an array of changes.
changes = MethodSchemaField()
changes.type = "text_area"
changes.name = "Description"
dataset_calibration.custom_fields.append(changes)
session.add(dataset_calibration)
session.flush()
def initialise_method_templates(session):
"""
Initialise the default method templates.
:param session: Database connection to add the created templates to
:return: None
"""
blank_template = session.query(MethodTemplate).filter_by(name="Blank Template").first()
if not blank_template:
blank_method = Method()
# blank_method.method_description = "Test description"
session.add(blank_method) # Add an empty project as a blank template
blank_dataset = Dataset()
blank_dataset.name = "Test Title"
session.add(blank_dataset) # Add an empty project as a blank template
session.flush()
blank_template = MethodTemplate()
blank_template.template_id = blank_method.id
blank_template.dataset_id = blank_dataset.id
blank_template.category = "Blank (No pre-fill)"
blank_template.name = "Blank Template"
blank_template.description = (
"An empty template that allows you to start from scratch (only for advanced "
"users or if no other template is relevent)."
)
session.add(blank_template) # Add an empty project as a blank template
tree_template = session.query(MethodTemplate).filter_by(name="Artificial Tree").first()
if not tree_template:
tree_method = Method()
tree_method.method_name = "Artificial Sensor Tree"
tree_method.method_description = (
"Collection method for ingesting aggregated tree sensor data from an external file server."
)
tree_method.data_source = PullDataSource.__tablename__
tree_schema = MethodSchema()
tree_schema.name = "ArtificialTree"
tree_data_field = MethodSchemaField()
tree_data_field.type = "file"
tree_data_field.units = "text"
tree_data_field.name = "TreeData"
tree_data_field.description = "Aggregated data of all sensors for an artificial tree."
tree_schema.custom_fields = [tree_data_field]
tree_method.data_type = tree_schema
# blank_method.method_description = "Test description"
session.add(tree_method) # Add an empty project as a blank template
tree_dataset = Dataset()
tree_dataset.name = "Raw Artificial Tree Data"
tree_datasource = PullDataSource()
tree_datasource.uri = "http://emu.hpc.jcu.edu.au/tree/split/"
tree_datasource.filename_pattern = ""
tree_datasource.selected_sampling = PullDataSource.periodic_sampling.key
tree_datasource.periodic_sampling = "1"
tree_dataset.pull_data_source = tree_datasource
session.add(tree_dataset) # Add an empty project as a blank template
session.flush()
tree_template = MethodTemplate()
tree_template.template_id = tree_method.id
tree_template.dataset_id = tree_dataset.id
tree_template.category = "Artificial Tree"
tree_template.name = "Artificial Tree"
tree_template.description = "Template for setting up ingestion from an artificial tree."
session.add(tree_template) # Add an empty project as a blank template
sensor_template = session.query(MethodTemplate).filter_by(name="Artificial Tree Sensor").first()
if not sensor_template:
sensor_method = Method()
sensor_method.method_name = "Artificial Tree Sensor"
sensor_method.method_description = (
"Filter and index one sensor station from the aggregated artificial tree data."
)
sensor_method.data_source = DatasetDataSource.__tablename__
sensor_method.data_type = session.query(MethodSchema).filter_by(name="Temperature").first()
# blank_method.method_description = "Test description"
session.add(sensor_method) # Add an empty project as a blank template
sensor_dataset = Dataset()
sensor_dataset.name = "Artificial Tree Sensor"
sensor_datasource = DatasetDataSource()
sensor_datasource.custom_processing_parameters = (
"file_field=TreeData, temp_field=Temperature, sensor_id=28180E08030000BE"
)
sensor_dataset.dataset_data_source = sensor_datasource
session.add(sensor_dataset) # Add an empty project as a blank template
session.flush()
sensor_template = MethodTemplate()
sensor_template.template_id = sensor_method.id
sensor_template.dataset_id = sensor_dataset.id
sensor_template.category = "Artificial Tree"
sensor_template.name = "Artificial Tree Sensor"
sensor_template.description = "Templates setting up post-processing and indexing of one artificial tree sensor from the aggregated artificial tree data."
session.add(sensor_template) # Add an empty project as a blank template
placeholder_template_names = [
"DRO",
"Australian Wet Tropics",
"TERN Supersite",
"The Wallace Initiative",
"Tropical Futures",
]
templates = session.query(MethodTemplate).all()
print len(templates)
if len(templates) <= 1:
count = 0
for name in placeholder_template_names:
for i in range(random.randint(2, 5)):
template = MethodTemplate()
template.template_id = blank_template.id
template.dataset_id = blank_template.dataset_id
template.category = name
template.name = name + " Placeholder Template " + str(count) + " (Testing Only)"
count += 1
template.description = (
"An empty template that allows you to start from scratch (only for advanced "
"users or if no other template is relevent)."
)
session.add(template) # Add an empty project as a blank template
def setUp(self):
self.config = ConfigParser.SafeConfigParser()
self.config.read('../../development.ini')
settings = self.config._sections["app:main"]
engine = engine_from_config(settings, 'sqlalchemy.')
DBSession.configure(bind=engine)
Base.metadata.create_all(engine)
self.session = DBSession()
self.auth = CredentialsAuthentication(self.config.get("app:main", "ingesterapi.username"), self.config.get("app:main", "ingesterapi.password"))
self.ingester_api = IngesterAPIWrapper(self.config.get("app:main", "ingesterapi.url"), self.auth)
self.project = Project()
# self.project.description = "This is a test description for the DC24 provisioning interface"
# self.project.no_activity = True
# self.project.project_title = "This is a test title for the test DC24 project"
# self.project.data_manager = "A Person"
# self.project.project_lead = "Another Person"
# self.project.brief_description = "This is a test brief description"
# self.project.full_description = "This is a test full description"
# keyword1 = Keyword()
# keyword1.id = 0
# keyword1.project_id = self.project.id
# keyword1.keyword = "Test Keyword"
# self.project.keywords.append(keyword1)
# for1 = FieldOfResearch()
# for1.id = 0
# for1.project_id = self.project.id
# for1.field_of_research = "010101"
# self.project.fieldOfResearch.append(for1)
#
# seo1 = FieldOfResearch()
# seo1.id = 0
# seo1.project_id = self.project.id
# seo1.field_of_research = "010101"
# self.project.socioEconomicObjective.append(seo1)
# self.project.ecosystems_conservation_climate = True
# self.project.typeOfResearch = "applied"
# self.project.time_period_description = "Test time period description " + str(self.project.id)
# self.project.date_from = 12345
# self.project.date_to = 1234
# self.project.location_description = "Test location description"
test_location = Location()
test_location.name = "Test Location"
test_location.location = "POINT(135.8763427287297 -24.167471616893767)"
test_location.elevation = 12.3
self.project.information = Metadata()
self.project.information.locations.append(test_location)
self.project.information.retention_period = "5"
self.project.metadata.national_significance = False
method1 = Method()
method1.method_name = "Artificial tree sensor"
method1.method_description = "A custom developed sensor consisting of a calibrated temperature sensor and a humidity sensor (which also has an uncalibrated temperature sensor within it)"
method1.data_source = PullDataSource.__tablename__
temperature_schema = self.session.query(MethodSchema).filter_by(id=1).first()
method1.data_type = MethodSchema()
method1.data_type.name = "Test Schema"
method1.data_type.parents.append(temperature_schema) # This is the default template schema that is setup on first run within scripts\initialise_database.py
# The data entry location offset functionality has been changed
# offset_schema = MethodSchema()
# offset_schema.id = 1
# offset_schema.template_schema = True
# offset_schema.name = "XYZ Offset Schema"
# offset = LocationOffset()
#
# x_offset = MethodSchemaField()
# x_offset.id = 0
# x_offset.method_schema_id = offset_schema.id
# x_offset.type = "Double"
# x_offset.units = "m"
# offset_schema.custom_fields.append(x_offset)
#
# y_offset = MethodSchemaField()
# y_offset.id = 1
# y_offset.method_schema_id = offset_schema.id
# y_offset.type = "Double"
# y_offset.units = "m"
# offset_schema.custom_fields.append(y_offset)
#
# z_offset = MethodSchemaField()
# z_offset.id = 2
# z_offset.method_schema_id = offset_schema.id
# z_offset.type = "Double"
# z_offset.units = "m"
# offset_schema.custom_fields.append(z_offset)
#
# method1.data_type.parents.append(offset_schema)
custom_field = MethodSchemaField()
custom_field.name = "Distance"
custom_field.type = "file"
custom_field.units = "text/cvs"
method1.data_type.custom_fields.append(custom_field)
self.project.methods.append(method1)
self.session.add(method1)
self.session.flush()
dataset1 = Dataset()
dataset1.method_id = method1.id
dataset1.disabled = False
dataset1.description = "Test dataset"
data_source = PullDataSource()
data_source.uri = "http://localhost/test_ingestion"
data_source.mime_type = custom_field.units
data_source.selected_sampling = PullDataSource.periodic_sampling.key
data_source.file_field = custom_field.id
data_source.periodic_sampling = 1
dataset1.pull_data_source = data_source
dataset1.time_period_description = "Test dataset time description"
dataset1.date_from = 1234
dataset1.date_to = 1234
dataset1.location_description = "Test dataset location description"
dataset1.elevation = 12.5
# If project location is set:
# Allow user to provide offset only (set dataset location to project location)
# Else:
# Must set location (with optional offset)
# TODO: For locations in project: add as region to location
dataset_location = Location()
dataset_location.name = "Test Dataset Location"
dataset_location.location = "POINT(132.8763427287297 -24.167471616893767)"
dataset_location.elevation = 12.6
dataset1.dataset_locations.append(dataset_location)
location_offset = LocationOffset(0, 0, 5)
dataset1.location_offset = location_offset
self.project.datasets.append(dataset1)
self.session.add(self.project)
self.session.flush()