def test_update_cell_in_cell_datasets(self):
store.dropAll()
bds = BoundaryDataSet("id")
boundaries = [
'O23P12P34S56', 'P10P11P2', 'N0', 'N8O2P0', 'O6S0S1S2', 'Q'
]
for boundary in boundaries:
bds.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.insert(bds)
store.update_boundary_in_boundary_datasets(
"id", (Boundary(boundary_ID=BoundaryID('O23P12P34S56'))),
Data("test"))
stored_bds = store.query_by_boundary_in_boundary_datasets(
"id", (Boundary(boundary_ID=BoundaryID('O23P12P34S56'))))
num_bds = 0
num_boundaries = 0
for bds in stored_bds:
for boundary, data in bds.get_boundaries_and_data():
assert boundaries.__contains__(boundary.AUID_to_CUIDs())
assert data.content == Data("test").content
num_boundaries = num_boundaries + 1
num_bds = num_bds + 1
self.assertEqual(num_bds, 1)
self.assertEqual(num_boundaries, 1)
store.dropAll()
def test_update_boundary_dataset(self):
store.dropAll()
bds = BoundaryDataSet("id")
boundaries = [
'O23P12P34S56', 'P10P11P2', 'N0', 'N8O2P0', 'O6S0S1S2', 'Q'
]
for boundary in boundaries:
bds.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.insert(bds)
bds2 = BoundaryDataSet("id")
boundaries2 = ['N0', 'N8O2P0', 'O6S0S1S2', 'Q']
for boundary in boundaries2:
bds2.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.update_boundary_dataset(bds2)
stored_bds = store.query_by_boundary_dataset_id("id")
num_bds = 0
num_boundaries = 0
for bds in stored_bds:
for boundary in bds.get_boundaries():
assert boundaries2.__contains__(boundary.AUID_to_CUIDs())
num_boundaries = num_boundaries + 1
num_bds = num_bds + 1
self.assertEqual(num_bds, 1)
self.assertEqual(num_boundaries, len(boundaries2))
store.dropAll()
def test_update_cell_dataset(self):
store.dropAll()
cds = CellDataSet("id")
cells = ['P0', 'P1', 'P2', 'P3', 'P4', 'P5']
for cell in cells:
cds.add(CellID(cell), Data(""))
store.insert(cds)
cds2 = CellDataSet("id")
cells2 = ['O0', 'O1', 'O2', 'O3', 'O4']
for cell in cells2:
cds2.add(CellID(cell), Data(""))
store.update_cell_dataset(cds2)
stored_cds = store.query_by_cell_dataset_id("id")
num_cds = 0
num_cells = 0
for cds in stored_cds:
for cell in cds.get_cells():
assert cells2.__contains__(cell.value)
num_cells = num_cells + 1
num_cds = num_cds + 1
self.assertEqual(num_cds, 1)
self.assertEqual(num_cells, len(cells2))
store.dropAll()
def test_cell_dataset(self):
cell_data_set = {
'N01': (CellID('N01'), Data('test')),
'N02': (CellID('N02'), Data('test')),
'N03': (CellID('N03'), Data('test'))
}
c_dataset = CellDataSet('id', cell_data_set=cell_data_set)
self.assertEqual(c_dataset.cell_data_set, cell_data_set)
def test_get_cell_data(self):
cells = [CellID('P0'), CellID('N02'), CellID('N033')]
data = Data('test')
data2 = Data('test2')
cell_data_set = {
cells[0].value: (cells[0], data),
cells[1].value: (cells[1], data),
cells[2].value: (cells[2], data2)
}
c_dataset = CellDataSet('id', cell_data_set=cell_data_set)
self.assertEqual(c_dataset.get_cell_data(CellID('N033')), data2)
def to_internal_value(self, data):
cds = CellDataSet(id='')
for item in data:
if not re.match(r'^[A-Za-z0-9]+$', item['cellID']):
raise ValidationError('Incorrect format: ' + item['cellID'])
cds.add(CellID(item['cellID']), Data(item['data']))
return cds.cell_data_set
def get_Boundary_Data(self, boundary):
"""
:param boundary: document stored that contains the boundary_id and data associated
:return: OptimalBoundary and Data tuple
"""
return OptimalBoundary(boundary_ID=AUID(boundary["auid"])), Data(
boundary["data"])
def query_by_boundary_in_boundary_datasets(self, id, boundary):
"""
:param id: identifier of the BoundaryDataset
:param boundary: Boundary or OptimalBoundary. If it is not optimal, it is optimized before making the query.
:return: Boundary and data associated stored in the BoundaryDataset with that id.
"""
if boundary.is_optimal():
optimal_boundary = boundary
else:
optimal_boundary = boundary.optimize()
boundaries_datasets_founded = self.db.b_data_sets.find({"_id": id})
boundary_data_sets = []
for boundary_dataset in boundaries_datasets_founded:
bds = BoundaryDataSet(id=id)
boundaries_in_bds_founded = self.db.boundaries.find({
"boundary_dataset_id":
boundary_dataset["_id"],
"auid":
optimal_boundary.boundary_ID.value
})
for boundary in boundaries_in_bds_founded:
bds.add(OptimalBoundary(boundary_ID=AUID(boundary["auid"])),
Data(boundary["data"]))
boundary_data_sets.append(bds)
return boundary_data_sets
def test_boundary_dataset_add_list(self):
boundary1 = OptimalBoundary(cells=[
CellID('N'),
CellID('O0'),
CellID('P123'),
CellID('S34567')
])
boundary2 = OptimalBoundary(cells=[CellID('O35'), CellID('P234')])
boundary3 = OptimalBoundary(cells=[CellID('S034'), CellID('S57')])
boundary4 = OptimalBoundary(cells=[CellID('P33'), CellID('P44')])
data = Data('')
b_ds = {
boundary1.boundary_ID.value: (boundary1, data),
boundary2.boundary_ID.value: (boundary2, data),
}
b_dataset = BoundaryDataSet('id', b_ds)
b_dataset.add_list([(boundary3, data), (boundary4, data)])
b_ds2 = {
boundary1.boundary_ID.value: (boundary1, data),
boundary2.boundary_ID.value: (boundary2, data),
boundary3.boundary_ID.value: (boundary3, data),
boundary4.boundary_ID.value: (boundary4, data)
}
self.assertEqual(b_dataset.boundary_data_set, b_ds2)
def get_optimal_boundary_from_shp_file(self,
file,
with_ids,
refinement=None,
unic_data=False):
shapes = fiona.open(file)
cells = []
data = {}
first = True
for polygon in shapes:
if with_ids:
cells.append(CellID(polygon['properties']['id']))
else:
assert refinement is not None
cells.append(self.get_cell_ID(polygon, int(refinement)))
if not unic_data:
if 'data' in polygon['properties']:
data[polygon['properties']
['id']] = polygon['properties']['data']
else:
data[polygon['properties']['id']] = polygon['properties']
elif unic_data and first:
if 'data' in polygon['properties']:
data = polygon['properties']['data']
else:
data = polygon['properties']
return OptimalBoundary(cells=cells), Data(data)
def test_delete_boundary_in_boundary_datasets(self):
store.dropAll()
bds = BoundaryDataSet("id")
boundaries = [
'O23P12P34S56', 'P10P11P2', 'N0', 'N8O2P0', 'O6S0S1S2', 'Q'
]
for boundary in boundaries:
bds.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.insert(bds)
deleted_boundaries = store.delete_boundary_in_boundary_datasets(
"id", (Boundary(boundary_ID=BoundaryID('O23P12P34S56'))))
self.assertEqual(deleted_boundaries, 1)
stored_bds = store.query_by_boundary_in_boundary_datasets(
"id", (Boundary(boundary_ID=BoundaryID('O23P12P34S56'))))
num_bds = 0
for bds in stored_bds:
assert bds.get_boundaries().__len__() == 0
num_bds = num_bds + 1
assert num_bds == 1
self.assertEqual(num_bds, 1)
store.dropAll()
def to_internal_value(self, data):
bds = BoundaryDataSet(id='')
for item in data:
if not re.match(r'^[A-Za-z0-9]+$', item['boundary']):
raise ValidationError('Incorrect format: ' + item['boundary'])
bds.add(Boundary(boundary_ID=BoundaryID(item['boundary'])),
Data(item['data']))
return bds.boundary_data_set
def test_get_max_refinement(self):
cells = [CellID('P0'), CellID('N02'), CellID('N033')]
data = Data('test')
cell_data_set = {
cells[0].value: (cells[0], data),
cells[1].value: (cells[1], data),
cells[2].value: (cells[2], data)
}
c_dataset = CellDataSet('id', cell_data_set=cell_data_set)
self.assertEqual(c_dataset.get_max_refinement(), 3)
def fromJSON(self, cds, file=False):
if file:
cds = json.load(cds)
self.cell_data_set = {}
self.id = cds['id']
cell_list = cds['cell_data_set']
for cell in cell_list:
self.add(CellID(cell['cellID']), Data(cell['data']))
return self
def test_get_boundary_data(self):
boundary1 = OptimalBoundary(cells=[
CellID('N'),
CellID('O0'),
CellID('P123'),
CellID('S34567')
])
boundary2 = OptimalBoundary(cells=[CellID('O35'), CellID('P234')])
boundary3 = OptimalBoundary(cells=[CellID('S034'), CellID('S57')])
data = Data('data')
data2 = Data('data2')
b_ds = {
boundary1.boundary_ID.value: (boundary1, data),
boundary2.boundary_ID.value: (boundary2, data),
boundary3.boundary_ID.value: (boundary3, data2)
}
b_dataset = BoundaryDataSet('id', b_ds)
self.assertEqual(b_dataset.get_boundary_data(boundary3.boundary_ID),
data2)
def fromJSON(self, bds, file=False):
if file:
bds = json.load(bds)
self.boundary_data_set = {}
self.id = bds['id']
boundary_list = bds['boundary_data_set']
for boundary in boundary_list:
self.add(OptimalBoundary(boundary_ID=AUID(boundary['AUID'])),
Data(boundary['data']))
return self
def test_cell_dataset_add(self):
cells = [CellID('N01'), CellID('N02'), CellID('N03'), CellID('N04')]
data = Data('test')
cell_data_set = {
cells[0].value: (cells[0], data),
cells[1].value: (cells[1], data),
cells[2].value: (cells[2], data)
}
c_dataset = CellDataSet('id', cell_data_set=cell_data_set)
c_dataset.add(cells[3], data)
self.assertEqual(c_dataset.cells, cells)
def all_cell_datasets(self):
"""
:return: List of all stored CellDatasets
"""
cell_data_sets = []
cell_datasets_founded = self.db.c_data_sets.find()
for cell_dataset in cell_datasets_founded:
cds = CellDataSet(id=cell_dataset["_id"])
cells_in_cds_founded = self.db.cells.find({"cell_dataset_id": cell_dataset["_id"]})
for cell in cells_in_cds_founded:
cds.add(CellID(cell['cellID']), Data(cell["data"]))
cell_data_sets.append(cds)
return cell_data_sets
def test_update_cell_in_cell_datasets(self):
store.dropAll()
cds = CellDataSet("id")
cells = ['P0', 'P1', 'P2', 'P3', 'P4', 'P5']
for cell in cells:
cds.add(CellID(cell), Data(""))
store.insert(cds)
store.update_cell_in_cell_datasets("id", CellID('P0'), Data("test"))
stored_cds = store.query_by_cell_in_cell_datasets("id", CellID('P0'))
num_cds = 0
num_cells = 0
for cds in stored_cds:
for cell, data in cds.get_cells_and_data():
assert cells.__contains__(cell.value)
assert data.content == Data("test").content
num_cells = num_cells + 1
num_cds = num_cds + 1
self.assertEqual(num_cds, 1)
self.assertEqual(num_cells, 1)
store.dropAll()
def query_by_cell_dataset_id(self, id):
"""
:param id: identifier of the CellDataset
:return: List of tuples with stored cells and data associated stored in the CellDataset with that id.
"""
cell_datasets_founded = self.db.c_data_sets.find({"_id": id})
cell_data_sets = []
for cell_dataset in cell_datasets_founded:
cds = CellDataSet(id=id)
cells_in_cds_founded = self.db.cells.find({"cell_dataset_id": cell_dataset["_id"]})
for cell in cells_in_cds_founded:
cds.add(CellID(cell['cellID']), Data(cell["data"]))
cell_data_sets.append(cds)
return cell_data_sets
def test_get_cells_and_data(self):
cells = [CellID('N01'), CellID('N02'), CellID('N03')]
data = Data('test')
cell_data_set = {
cells[0].value: (cells[0], data),
cells[1].value: (cells[1], data),
cells[2].value: (cells[2], data)
}
c_dataset = CellDataSet('id', cell_data_set=cell_data_set)
cells_data = c_dataset.get_cells_and_data()
self.assertEqual(cells_data, [(cells[0], data), (cells[1], data),
(cells[2], data)])
def test_delete_cell(self):
store.dropAll()
cds = CellDataSet("id")
cells = ['P0', 'P1', 'P2', 'P3', 'P4', 'P5']
for cell in cells:
cds.add(CellID(cell), Data(""))
store.insert(cds)
deleted_cells = store.delete_cell(CellID('P0'))
self.assertEqual(deleted_cells, 1)
stored_cells = store.query_by_cell(CellID('P0'))
self.assertEqual(stored_cells.__len__(), 0)
store.dropAll()
def all_boundary_datasets(self):
"""
:return: List of all stored BoundaryDatasets
"""
boundary_data_sets = []
boundaries_datasets_founded = self.db.b_data_sets.find()
for boundary_dataset in boundaries_datasets_founded:
bds = BoundaryDataSet(id=boundary_dataset["_id"])
boundaries_in_bds_founded = self.db.boundaries.find(
{"boundary_dataset_id": boundary_dataset["_id"]})
for boundary in boundaries_in_bds_founded:
bds.add(OptimalBoundary(boundary_ID=AUID(boundary["auid"])),
Data(boundary["data"]))
boundary_data_sets.append(bds)
return boundary_data_sets
def get_cell_dataset_from_tif_file(self, file, id):
cds = CellDataSet(id=id)
dataset, band, refinement = self.tif_file_treatment(file)
for j in range(dataset.height):
print(str(j) + "/" + str((dataset.height)))
for i in range(dataset.width):
cell_rHealPix_coords = dataset.xy(j, i)
cell = self.dggs.get_cell_from_projected_point(
refinement, cell_rHealPix_coords)
data = Data(int(band[j, i]))
cds.add(cell, data)
if j == 10:
break
return cds
def test_query_by_cell(self):
store.dropAll()
cds = CellDataSet("id")
cells = ['P0', 'P1', 'P2', 'P3', 'P4', 'P5']
for cell in cells:
cds.add(CellID(cell), Data(""))
store.insert(cds)
stored_cells = store.query_by_cell(CellID('P0'))
num_cells = 0
for cell in stored_cells:
assert cells.__contains__(cell[0].value)
num_cells = num_cells + 1
self.assertEqual(num_cells, 1)
store.dropAll()
def query_by_boundary_dataset_id(self, id):
"""
:param id: identifier of the BoundaryDataset
:return: BoundaryDataset stored with that id.
"""
boundaries_datasets_founded = self.db.b_data_sets.find({"_id": id})
boundary_data_sets = []
for boundary_dataset in boundaries_datasets_founded:
bds = BoundaryDataSet(id=id)
boundaries_in_bds_founded = self.db.boundaries.find(
{"boundary_dataset_id": boundary_dataset["_id"]})
for boundary in boundaries_in_bds_founded:
bds.add(OptimalBoundary(boundary_ID=AUID(boundary["auid"])),
Data(boundary["data"]))
boundary_data_sets.append(bds)
return boundary_data_sets
def test_insert_and_all_cells(self):
store.dropAll()
cds = CellDataSet("id")
cells = ['P0', 'P1', 'P2', 'P3', 'P4', 'P5']
for cell in cells:
cds.add(CellID(cell), Data(""))
store.insert(cds)
stored_cells = store.all_cells()
num_cells = 0
for cell in stored_cells:
assert cells.__contains__(cell[0].value)
num_cells = num_cells + 1
self.assertEqual(num_cells, len(cells))
store.dropAll()
def test_delete_boundary_dataset(self):
store.dropAll()
bds = BoundaryDataSet("id")
boundaries = [
'O23P12P34S56', 'P10P11P2', 'N0', 'N8O2P0', 'O6S0S1S2', 'Q'
]
for boundary in boundaries:
bds.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.insert(bds)
deleted_bds = store.delete_boundary_dataset("id")
self.assertEqual(deleted_bds, 1)
stored_bds = store.query_by_boundary_dataset_id("id")
self.assertEqual(stored_bds.__len__(), 0)
store.dropAll()
def test_insert_and_all_boudnaries(self):
store.dropAll()
bds = BoundaryDataSet("id")
boundaries = [
'O23P12P34S56', 'P10P11P2', 'N0', 'N8O2P0', 'O6S0S1S2', 'Q'
]
for boundary in boundaries:
bds.add(Boundary(boundary_ID=BoundaryID(boundary)), Data(""))
store.insert(bds)
stored_boundaries = store.all_boundaries()
num_boundaries = 0
for boundary in stored_boundaries:
assert boundaries.__contains__(boundary[0].AUID_to_CUIDs())
num_boundaries = num_boundaries + 1
self.assertEqual(num_boundaries, len(boundaries))
store.dropAll()
def get_boundary_dataset_from_shp_file(self,
dir,
id,
with_ids,
refinement=None,
unic_data=False):
import os
import glob
os.chdir(dir)
boundaries = []
for file in glob.glob('*.shp'):
boundaries.append(fiona.open(file))
bds = BoundaryDataSet(id=id)
for boundary in boundaries:
data = {}
boundary_id = ''
first = True
for polygon in boundary:
if with_ids:
boundary_id = boundary_id + polygon['properties']['id']
else:
assert refinement is not None
cell = self.get_cell_ID(polygon, int(refinement))
boundary_id = boundary_id + cell.value
if not unic_data:
if 'data' in polygon['properties']:
data[polygon['properties']
['id']] = polygon['properties']['data']
else:
data[polygon['properties']
['id']] = polygon['properties']
elif unic_data and first:
if 'data' in polygon['properties']:
data = polygon['properties']['data']
else:
data = polygon['properties']
bds.add(boundary=Boundary(boundary_ID=BoundaryID(boundary_id)),
data=Data(data))
return bds