def get_or_create(self, uri, *args, **kwargs):
''' Create dataset and corresponding metadata
Parameters:
----------
uri : str
URI to file or stream openable by netCDF4.Dataset
Returns:
-------
dataset and flag
'''
# check if dataset already exists
uris = DatasetURI.objects.filter(uri=uri)
if len(uris) > 0:
print(uri + ': Already ingested')
return uris[0].dataset, False
# set source
platform = pti.get_gcmd_platform('buoys')
instrument = pti.get_gcmd_instrument('in situ/laboratory instruments')
pp = Platform.objects.get(category=platform['Category'],
series_entity=platform['Series_Entity'],
short_name=platform['Short_Name'],
long_name=platform['Long_Name'])
ii = Instrument.objects.get(category=instrument['Category'],
instrument_class=instrument['Class'],
type=instrument['Type'],
subtype=instrument['Subtype'],
short_name=instrument['Short_Name'],
long_name=instrument['Long_Name'])
source = Source.objects.get_or_create(platform=pp, instrument=ii)[0]
try:
nc_dataset = netCDF4.Dataset(uri)
except OSError:
nc_dataset = netCDF4.Dataset(uri + '#fillmismatch')
if not 'time_coverage_start' in nc_dataset.ncattrs(
) or not nc_dataset.time_coverage_start:
time_coverage_start = parse(nc_dataset.variables['time'].units.split()[2]) + \
datetime.timedelta(seconds=nc_dataset.variables['time'][:].data[0])
else:
time_coverage_start = parse(nc_dataset.time_coverage_start),
if not 'time_coverage_end' in nc_dataset.ncattrs(
) or not nc_dataset.time_coverage_end:
time_coverage_end = parse(nc_dataset.variables['time'].units.split()[2]) + \
datetime.timedelta(seconds=nc_dataset.variables['time'][:].data[-1])
else:
time_coverage_end = parse(nc_dataset.time_coverage_end),
longitude = nc_dataset.variables['longitude'][:].data
latitude = nc_dataset.variables['latitude'][:].data
geometry = LineString(np.column_stack((longitude, latitude)))
geolocation = GeographicLocation.objects.get_or_create(
geometry=geometry)[0]
# in case of a point..
#location = GEOSGeometry('POINT(%s %s)' % (longitude, latitude))
#geolocation = GeographicLocation.objects.get_or_create(
# geometry=location)[0]
entrytitle = nc_dataset.title
dc = DataCenter.objects.get(short_name='NO/MET')
iso_category = ISOTopicCategory.objects.get(
name='Climatology/Meteorology/Atmosphere')
if 'summary' in nc_dataset.ncattrs():
summary = nc_dataset.summary
else:
summary = nc_dataset.title
if 'entry_id' in nc_dataset.ncattrs():
entry_id = nc_dataset.entry_id
elif 'id' in nc_dataset.ncattrs():
entry_id = nc_dataset.id
elif 'station_name' in nc_dataset.ncattrs():
id0 = nc_dataset.station_name.replace(' ', '')
lastds = self.filter(entry_id__contains=id0).last()
if lastds:
next_buoy_num = int(lastds.entry_id.replace(id0, '')) + 1
else:
next_buoy_num = 1
entry_id = id0 + str(next_buoy_num)
ds = Dataset(entry_id=entry_id,
entry_title=entrytitle,
ISO_topic_category=iso_category,
data_center=dc,
summary=summary,
time_coverage_start=time_coverage_start,
time_coverage_end=time_coverage_end,
source=source,
geographic_location=geolocation)
ds.save()
ds_uri = DatasetURI.objects.get_or_create(uri=uri, dataset=ds)[0]
# Add dataset parameters
vars = nc_dataset.variables
time = vars.pop('time')
lat = vars.pop('latitude')
lon = vars.pop('longitude')
id = vars.pop('station_id', '')
for key in vars.keys():
if not 'standard_name' in vars[key].ncattrs():
continue
try:
par = Parameter.objects.get(
standard_name=vars[key].standard_name)
except Parameter.DoesNotExist as e:
warnings.warn('{}: {}'.format(vars[key].standard_name,
e.args[0]))
continue
dsp = DatasetParameter(dataset=ds, parameter=par)
dsp.save()
return ds, True
def test_snapping_choose_closest_point(self):
# Line with several points in less than PATH_SNAPPING_DISTANCE
PathFactory.create(
geom=LineString((0, 0), (9.8, 0), (9.9, 0), (10, 0)))
path_snapped = PathFactory.create(geom=LineString((10, 0.1), (10, 10)))
self.assertEqual(path_snapped.geom.coords, ((10, 0), (10, 10)))
def test_resolution_step_depends_on_geometry_size(self):
self.assertEqual(self.area['resolution']['step'], 25)
geom = LineString((100, 370), (100100, 370), srid=settings.SRID)
area = AltimetryHelper.elevation_area(geom)
self.assertEqual(area['resolution']['step'], 866)
def test_splitted_paths_do_not_become_visible(self):
PathFactory(geom=LineString((10, 0), (12, 0)), visible=False)
PathFactory(geom=LineString((11, 1), (11, -1)))
self.assertEqual(len(Path.objects.all()), 3)
def test_valid_geometry_can_be_saved(self):
p = PathFactory.build(geom=LineString((0, 0), (2, 0), (1, 1)))
p.save()
def test_snapping_3d(self):
PathFactory.create(geom=LineString(((0, 0, 0), (5, 0, 1000), (6, 0,
0))))
path_snapped = PathFactory.create(
geom=LineString((0, 0, 0), (5.2, 0, 0)))
self.assertEqual(path_snapped.geom.coords, ((0, 0, 0), (5, 0, 1000)))
def put(self, request, unique_id, version='v1'):
sequence = Sequence.objects.get(unique_id=unique_id)
if not sequence or sequence is None:
return Response({
'error': 'Sequence id is invalid',
'status': False
})
data = request.data
message = ''
if 'google_street_view' in data.keys():
google_street_view = data['google_street_view']
print(google_street_view)
sequence.google_street_view = google_street_view
message += 'Google Street View is updated \n'
sequence.save()
if 'strava' in data.keys():
strava = data['strava']
print(strava)
sequence.strava = strava
message += 'Strava is updated \n'
sequence.save()
if not 'mapillary_user_token' in data.keys():
if message != '':
return Response({'message': message, 'status': True})
else:
return Response({
'error': 'Mapillary token is missing',
'status': False
})
token = data['mapillary_user_token']
mapillary = Mapillary(token, source='mtpdu')
map_user_data = mapillary.get_mapillary_user()
if not map_user_data:
return Response({
'error': 'Mapillary token is invalid',
'status': False
})
if not 'mapillary_sequence_key' in data.keys():
if message != '':
return Response({'message': message, 'status': True})
else:
return Response({
'error': 'Sequence key is missing',
'status': False
})
seq_key = data['mapillary_sequence_key']
sequence_json = mapillary.get_sequence_by_key(seq_key)
print('sequence_json')
if not sequence_json:
return Response({'error': 'Sequence is empty', 'status': False})
properties = sequence_json['properties']
geometry = sequence_json['geometry']
seq_key = properties['key']
if properties['username'] != map_user_data['username']:
return Response({
'error': 'You are not owner of this sequence',
'status': False
})
sequences = Sequence.objects.filter(seq_key=seq_key)[:1]
if sequences.count() > 0:
for seq in sequences:
if seq.unique_id == unique_id:
continue
tour_seqs = TourSequence.objects.filter(sequence=seq)
if tour_seqs.count() > 0:
for t_s in tour_seqs:
t_s.delete()
seq.delete()
sequence.user = request.user
if 'camera_make' in properties:
camera_makes = CameraMake.objects.filter(
name=properties['camera_make'])
if camera_makes.count() > 0:
c = camera_makes[0]
else:
c = CameraMake()
c.name = properties['camera_make']
c.save()
sequence.camera_make = c
sequence.captured_at = properties['captured_at']
if 'created_at' in properties:
sequence.created_at = properties['created_at']
sequence.seq_key = properties['key']
if 'pano' in properties:
sequence.pano = properties['pano']
if 'user_key' in properties:
sequence.user_key = properties['user_key']
if 'username' in properties:
sequence.username = properties['username']
sequence.geometry_coordinates_ary = geometry['coordinates']
sequence.image_count = len(geometry['coordinates'])
sequence.coordinates_cas = properties['coordinateProperties']['cas']
sequence.coordinates_image = properties['coordinateProperties'][
'image_keys']
if 'private' in properties:
sequence.is_private = properties['private']
lineString = LineString()
firstPoint = None
if sequence.image_count == 0:
firstPoint = Point(sequence.geometry_coordinates_ary[0][0],
sequence.geometry_coordinates_ary[0][1])
lineString = LineString(firstPoint.coords, firstPoint.coords)
else:
for i in range(len(sequence.geometry_coordinates_ary)):
coor = sequence.geometry_coordinates_ary[i]
if i == 0:
firstPoint = Point(coor[0], coor[1])
continue
point = Point(coor[0], coor[1])
if i == 1:
lineString = LineString(firstPoint.coords, point.coords)
else:
lineString.append(point.coords)
sequence.geometry_coordinates = lineString
sequence.is_published = True
sequence.save()
sequence.distance = sequence.get_distance()
sequence.save()
print('1')
p = threading.Thread(target=get_images_by_sequence,
args=(
sequence,
'mtpdu',
))
p.start()
print('2')
return JsonResponse({
'status': 'success',
'message':
'Sequence successfully was imported. Sequence will be published in about 10 minutes.',
'unique_id': str(sequence.unique_id)
})
except Exception, e:
logger.error('Could not download ocean basemap ')
mp = None
if mp is not None :
break
if mp is None :
logger.debug('Error - cannot cannot fetch basemap')
return
try:
logger.debug('plotting tracks')
if self.animate:
try:
track = LineString(points).simplify(tolerance=.001)
if track is not None:
ln,lt = zip(*track)
mp.plot(ln,lt,'-',c='k',alpha=0.5,linewidth=2, zorder=1)
except TypeError, e:
logger.warn("%s\nCannot plot map track path to None", e)
else:
for track in maptracks:
if track is not None:
ln,lt = zip(*track)
mp.plot(ln,lt,'-',c='k',alpha=0.5,linewidth=2, zorder=1)
# if have a valid series, then plot the dots
if self.plotDotParmName and len(z) > 0:
if len(z) > 2000:
sz = len(z)
def get_wegvak(input: str):
if input:
data = json.loads(input)
return LineString(data)
return None
def test_1m(self):
path = Path.objects.create(geom=LineString((0, 0), (0, 1), (1, 1)))
self.assertEqual(len(path.geom_3d.coords), 3)
def test_51m(self):
path = Path.objects.create(
geom=LineString((0, 0), (0, self.step * 2 + 1), (0, self.step * 4 +
2)))
self.assertEqual(len(path.geom_3d.coords), 7)
def test_49(self):
path = Path.objects.create(
geom=LineString((0, 0), (0, self.step * 2 - 1)))
self.assertEqual(len(path.geom_3d.coords), 3)
def test_elevation_altimetry_limits(self):
geom = LineString((1.5, 2.5, 8), (2.5, 2.5, 10), srid=settings.SRID)
profile = AltimetryHelper.elevation_profile(geom)
limits = AltimetryHelper.altimetry_limits(profile)
self.assertEqual(limits[0], 1108)
self.assertEqual(limits[1], -92)
def test_line_locate_point(self):
pos_expr = functions.LineLocatePoint(
LineString((0, 0), (0, 3), srid=4326), Point(0, 1, srid=4326))
self.assertAlmostEqual(
State.objects.annotate(pos=pos_expr).first().pos, 0.3333333)
def test_path_merge_without_snap(self):
"""
A B C D A D
|---------| + |---------| --> |------------------|
Test five cases : 1 - A match with C : unification D-C-A-B
2 - A match with D : unification C-D-A-B
3 - B match with C : unification A-B-C-D
4 - B match with D : unification A-B-D-C
5 - no match : no unification
"""
path_AB = PathFactory.create(name="PATH_AB", geom=LineString((0, 0), (4, 0)))
path_CD = PathFactory.create(name="PATH_CD", geom=LineString((4, 0), (8, 0)))
original_AB_length = path_AB.length
original_CD_length = path_CD.length
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0.0, 0.0), (4.0, 0.0), (8.0, 0.0), srid=settings.SRID))
self.assertEqual(path_AB.length, original_AB_length + original_CD_length)
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((4, 0), (0, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((4, 0), (8, 0)))
original_AB_length = path_AB.length
original_CD_length = path_CD.length
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (4, 0), (8, 0), srid=settings.SRID))
self.assertEqual(path_AB.length, original_AB_length + original_CD_length)
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((4, 0), (0, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((8, 0), (4, 0)))
original_AB_length = path_AB.length
original_CD_length = path_CD.length
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (4, 0), (8, 0), srid=settings.SRID))
self.assertEqual(path_AB.length, original_AB_length + original_CD_length)
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((0, 0), (4, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((8, 0), (4, 0)))
original_AB_length = path_AB.length
original_CD_length = path_CD.length
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (4, 0), (8, 0), srid=settings.SRID))
self.assertEqual(path_AB.length, original_AB_length + original_CD_length)
path_AB = PathFactory.create(name="PATH_AB", geom=LineString((0, 0), (4, 0)))
path_CD = PathFactory.create(name="PATH_CD", geom=LineString((50, 0), (100, 0)))
original_AB_length = path_AB.length
self.assertEqual(path_AB.merge_path(path_CD), False)
self.assertEqual(path_AB.length, original_AB_length)
class WayFeatureFactory(FeatureFactory):
osm_version = 3
osm_type = 'way'
url = factory.Sequence(lambda n: 'way-%d' % n)
geometry = LineString([(42.1, 13.4), (42.2, 14)])
geojson = json.dumps({'properties': [{'osm:type': 'way'}]})
def test_path_merge_with_snap(self):
"""
A B C D A D
|---------| + |---------| --> |------------------|
Test five cases : 1 - A match with C : unification D-C-A-B
2 - A match with D : unification C-D-A-B
3 - B match with C : unification A-B-C-D
4 - B match with D : unification A-B-D-C
5 - no match : no unification
"""
path_AB = PathFactory.create(name="PATH_AB", geom=LineString((0, 0), (15, 0)))
path_CD = PathFactory.create(name="PATH_CD", geom=LineString((16, 0), (30, 0)))
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (15, 0), (16, 0), (30, 0), srid=settings.SRID))
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((15, 0), (0, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((16, 0), (30, 0)))
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (15, 0), (16, 0), (30, 0), srid=settings.SRID))
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((15, 0), (0, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((30, 0), (16, 0)))
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (15, 0), (16, 0), (30, 0), srid=settings.SRID))
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="path_AB", geom=LineString((0, 0), (15, 0)))
path_CD = PathFactory.create(name="path_CD", geom=LineString((30, 0), (16, 0)))
self.assertEqual(path_AB.merge_path(path_CD), True)
self.assertEqual(path_AB.geom, LineString((0, 0), (15, 0), (16, 0), (30, 0), srid=settings.SRID))
path_AB.delete()
path_CD.delete()
path_AB = PathFactory.create(name="PATH_AB", geom=LineString((0, 0), (5, 0)))
path_CD = PathFactory.create(name="PATH_CD", geom=LineString((50, 0), (100, 0)))
self.assertEqual(path_AB.merge_path(path_CD), False)
def test_deleted_pois_nds(self):
trek = TrekFactory.create(geom=LineString((0, 0), (4, 4)))
poi = POIFactory.create(geom=Point(2.4, 2.4))
self.assertCountEqual(trek.pois, [poi])
poi.delete()
self.assertCountEqual(trek.pois, [])
def test_snapping_3d_extremities(self):
PathFactory.create(geom=LineString(((0, 0, 0), (10, 0, 1000))))
path_snapped = PathFactory.create(
geom=LineString((8, 0, 0), (-50, -50, 0)))
self.assertEqual(path_snapped.geom.coords, ((8, 0, 0), (-50, -50, 0)))
def setUp(self):
self.trek = TrekFactory.create(published=True, geom=LineString(((700000, 6600000), (700100, 6600100)), srid=2154))
def test_proxy(self):
"Testing Lazy-Geometry support (using the GeometryProxy)."
# Testing on a Point
pnt = Point(0, 0)
nullcity = City(name='NullCity', point=pnt)
nullcity.save()
# Making sure TypeError is thrown when trying to set with an
# incompatible type.
for bad in [5, 2.0, LineString((0, 0), (1, 1))]:
with self.assertRaisesMessage(TypeError, 'Cannot set'):
nullcity.point = bad
# Now setting with a compatible GEOS Geometry, saving, and ensuring
# the save took, notice no SRID is explicitly set.
new = Point(5, 23)
nullcity.point = new
# Ensuring that the SRID is automatically set to that of the
# field after assignment, but before saving.
self.assertEqual(4326, nullcity.point.srid)
nullcity.save()
# Ensuring the point was saved correctly after saving
self.assertEqual(new, City.objects.get(name='NullCity').point)
# Setting the X and Y of the Point
nullcity.point.x = 23
nullcity.point.y = 5
# Checking assignments pre & post-save.
self.assertNotEqual(Point(23, 5, srid=4326),
City.objects.get(name='NullCity').point)
nullcity.save()
self.assertEqual(Point(23, 5, srid=4326),
City.objects.get(name='NullCity').point)
nullcity.delete()
# Testing on a Polygon
shell = LinearRing((0, 0), (0, 90), (100, 90), (100, 0), (0, 0))
inner = LinearRing((40, 40), (40, 60), (60, 60), (60, 40), (40, 40))
# Creating a State object using a built Polygon
ply = Polygon(shell, inner)
nullstate = State(name='NullState', poly=ply)
self.assertEqual(4326, nullstate.poly.srid) # SRID auto-set from None
nullstate.save()
ns = State.objects.get(name='NullState')
self.assertEqual(ply, ns.poly)
# Testing the `ogr` and `srs` lazy-geometry properties.
self.assertIsInstance(ns.poly.ogr, gdal.OGRGeometry)
self.assertEqual(ns.poly.wkb, ns.poly.ogr.wkb)
self.assertIsInstance(ns.poly.srs, gdal.SpatialReference)
self.assertEqual('WGS 84', ns.poly.srs.name)
# Changing the interior ring on the poly attribute.
new_inner = LinearRing((30, 30), (30, 70), (70, 70), (70, 30),
(30, 30))
ns.poly[1] = new_inner
ply[1] = new_inner
self.assertEqual(4326, ns.poly.srid)
ns.save()
self.assertEqual(ply, State.objects.get(name='NullState').poly)
ns.delete()
def test_save_with_same_geom(self):
geom = LineString(((700000, 6600000), (700100, 6600100)), srid=2154)
self.trek.geom = geom
self.trek.save()
retrieve_trek = Trek.objects.get(pk=self.trek.pk)
self.assertTrue(retrieve_trek.geom.equals_exact(geom, tolerance=0.00001))
def test_self_intersection_raises_integrity_error(self):
# Create path with self-intersection
p = PathFactory.build(geom=LineString((0, 0), (2, 0), (1, 1), (1, -1)))
self.assertRaises(IntegrityError, p.save)
def get_geometry(self, lons, lats):
lon_lat = zip([self.shift_longitude(float(x)) for x in lons],
[float(x) for x in lats])
geometry = LineString(lon_lat)
return geometry
def test_modify_self_intersection_raises_integrity_error(self):
p = PathFactory.build(geom=LineString((0, 0), (2, 0), (1, 1)))
p.save()
p.geom = LineString((0, 0), (2, 0), (1, 1), (1, -1))
self.assertRaises(IntegrityError, p.save)
def create_path():
PathFactory.create(geom=LineString((0, 0), (2, 0), (1, 1), (1, -1)))
def test_area_has_nice_ratio_if_square_enough(self):
geom = LineString((0, 0), (1000, 1000), srid=settings.SRID)
area = AltimetryHelper.elevation_area(geom)
self.assertEqual(area['size']['x'], 1300.0)
self.assertEqual(area['size']['y'], 1300.0)
def test_valid_geometry_can_be_saved(self):
PathFactory.create(geom=LineString((0, 0), (2, 0), (1, 1)))
def setUp(self):
self._fill_raster()
self.geom = LineString((100, 370), (1100, 370), srid=settings.SRID)
self.area = AltimetryHelper.elevation_area(self.geom)
def test_reverse_geom(self):
coords = [(-95.363151, 29.763374), (-95.448601, 29.713803)]
Track.objects.create(name='Foo', line=LineString(coords))
track = Track.objects.annotate(reverse_geom=functions.Reverse('line')).get(name='Foo')
coords.reverse()
self.assertEqual(tuple(coords), track.reverse_geom.coords)
