def post(self, request):
user = request.user
data = request.data
# Now, we post the data directly to OSM.
try:
responses = []
facilities_data = data['healthsites']
for facility_data in facilities_data:
# Verify data uploader and owner/collector
is_valid, message = \
verify_user(user, facility_data['osm_user'])
if not is_valid:
return HttpResponseForbidden(message)
# Validate data
validate_osm_data(facility_data)
for facility_data in facilities_data:
# Map Healthsites tags to OSM tags
mapping_file_path = ABS_PATH('api', 'fixtures', 'mapping.yml')
facility_data['tag'] = convert_to_osm_tag(
mapping_file_path, facility_data['tag'], 'node')
# Push data to OSM
response = create_osm_node(user, facility_data)
responses.append(response)
return Response(responses)
except Exception as e:
return HttpResponseBadRequest('%s' % e)
def setUp(self):
if settings.TESTING:
# move to media root for testing purposes
self.default_media_path = settings.MEDIA_ROOT
settings.MEDIA_ROOT = ABS_PATH('media_test')
app_config = django_apps.get_app_config('realtime')
app_config.ready()
Flood.objects.create(event_id=u'2015112518-3-rw',
time=datetime.datetime(2015, 11, 25, 18, 0, 0),
interval=3,
source=u'Peta Jakarta',
region=u'Jakarta',
hazard_layer=File(
open(self.data_path('hazard.zip'))))
# create test user
User = get_user_model()
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******',
location=Point(0, 0),
email_updates=False)
realtime_group = Group.objects.get(name=REST_GROUP)
self.user.groups.add(realtime_group)
self.user.save()
self.client.login(email='*****@*****.**', password='******')
def setUp(self):
if settings.TESTING:
# move to media root for testing purposes
self.default_media_path = settings.MEDIA_ROOT
settings.MEDIA_ROOT = ABS_PATH('media_test')
flood = Flood.objects.create(
event_id=u'2015112518-3-rw',
time=datetime.datetime(2015, 11, 25, 18, 0, 0),
interval=3,
source=u'Peta Jakarta',
region=u'Jakarta',
hazard_layer=File(open(self.data_path('hazard.zip'))),
hazard_path=flood_layer_uri)
FloodReport.objects.create(
flood=flood,
language='en',
impact_report=File(open(self.data_path('impact-table-en.pdf'))),
impact_map=File(open(self.data_path('impact-map-en.pdf'))),
)
# create test user
User = get_user_model()
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******',
location=Point(0, 0),
email_updates=False)
realtime_group = Group.objects.get(name=REST_GROUP)
self.user.groups.add(realtime_group)
self.user.save()
self.client.login(email='*****@*****.**', password='******')
def create_osm_node_by_data(data, user, duplication_check=True):
""" Create data based on data and user"""
# Now, we post the data directly to OSM.
try:
# Split osm and extension attribute
osm_attr, locality_attr = split_osm_and_extension_attr(data['tag'])
data['tag'] = osm_attr
validate_osm_data(data, duplication_check=duplication_check)
# Map Healthsites tags to OSM tags
mapping_file_path = ABS_PATH('api', 'fixtures', 'mapping.yml')
data['tag'] = convert_to_osm_tag(mapping_file_path, data['tag'],
'node')
# Push data to OSM
response = create_osm_node(user, data)
# create pending index
create_pending_update('node', response['id'], data['tag']['name'],
user, response['version'])
save_extensions('node', response['id'], locality_attr)
return response
except Exception as e:
create_pending_review(user, data, '%s' % e)
output = {
'error': '%s' % e,
'payload': data,
}
return output
def get_osm_schema():
"""Get schema based on osm tag definitions.
:return: Defined OSM schema.
:rtype: dict
"""
schema_template_file_path = ABS_PATH('api', 'fixtures', 'schema.json')
with open(schema_template_file_path) as json_file:
schema = json.load(json_file)
schema['facilities']['create']['fields'] = ALL_FIELDS
return schema
def upload_to_osm(self):
"""Push parsed localities/facilities/healthsites data to OSM instance.
"""
self._upload_status['total'] = len(self._parsed_data)
for row_number, data in enumerate(self._parsed_data):
upload_status = {'uploaded': True, 'message': 'Uploaded'}
# split osm and extension attributes
osm_attr, locality_attr = split_osm_and_extension_attr(data['tag'])
data['tag'] = osm_attr
# Map Healthsites tags to OSM tags
mapping_file_path = ABS_PATH('api', 'fixtures', 'mapping.yml')
data['tag'] = convert_to_osm_tag(mapping_file_path, data['tag'],
'node')
# Push data to OSM
user = get_object_or_404(User, username=data['osm_user'])
try:
response = create_osm_node(user, data)
# create pending index
create_pending_update('node', response['id'],
data['tag']['name'], user,
response['version'])
save_extensions('node', response['id'], locality_attr)
except: # noqa
upload_status.update({
'uploaded':
False,
'message':
'{0}: {1}'.format(unicode(sys.exc_info()[0].__name__),
unicode(sys.exc_info()[1]))
})
self._upload_status['status'][row_number + 1] = upload_status
self._upload_status['count'] = row_number + 1
self._upload_status['summary'] = (
'There is error when uploading the data. '
'Please see the status detail for more '
'information.' if not self.is_uploaded() else '')
# write status to progress file
f = open(self.progress_file, 'w+')
f.write(json.dumps(self._upload_status))
f.close()
return False not in ([
status['uploaded']
for status in self._upload_status['status'].values()
])
def setUp(self):
if settings.TESTING:
# move to media root for testing purposes
self.default_media_path = settings.MEDIA_ROOT
settings.MEDIA_ROOT = ABS_PATH('media_test')
with open(self.data_path('grid.xml')) as grid_file:
Earthquake.objects.create(
shake_id='20150619200628',
shake_grid=File(grid_file),
magnitude=4.6,
time=datetime.datetime(2015,
6,
19,
20,
6,
28,
tzinfo=pytz.timezone('Asia/Jakarta')),
depth=10,
location=Point(x=126.52, y=4.16, srid=4326),
location_description='Manado')
earthquake = Earthquake.objects.get(shake_id='20150619200628')
earthquake.save()
report_pdf = earthquake.shake_id + '-id.pdf'
report_png = earthquake.shake_id + '-id.png'
report_thumb = earthquake.shake_id + '-thumb-id.png'
report = EarthquakeReport()
report.language = 'id'
report.earthquake = earthquake
if os.path.exists(self.data_path(report_pdf)):
with open(self.data_path(report_pdf)) as pdf:
report.report_pdf = File(pdf)
report.save()
if os.path.exists(self.data_path(report_png)):
with open(self.data_path(report_png)) as png:
report.report_image = File(png)
report.save()
if os.path.exists(self.data_path(report_thumb)):
with open(self.data_path(report_thumb)) as thumb:
report.report_thumbnail = File(thumb)
report.save()
# create test user
User = get_user_model()
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******',
location=Point(0, 0),
email_updates=False)
realtime_group = Group.objects.get(name=REST_GROUP)
self.user.groups.add(realtime_group)
self.user.save()
self.client.login(email='*****@*****.**', password='******')
def get_attributes(self, obj):
""" Get attributes from model fields
and also the extension"""
show_all = False
try:
show_all = self.context.get('flat', None)
# check tag that want to be used
tag = self.context.get('tag_format', None)
if tag == 'hxl':
mapping_file_path = ABS_PATH('api', 'fixtures',
'hxl_tags.json')
self.key_mapping = json.loads(
open(mapping_file_path, 'r').read())
except (AttributeError, IOError):
pass
attributes = {}
for attribute in attributes_fields:
if getattr(obj, attribute) or show_all:
attributes[self.map_tag(attribute)] = getattr(obj, attribute)
# check attributes from extension
extension, created = LocalityOSMExtension.objects.get_or_create(
osm_id=obj.osm_id, osm_type=self.get_osm_type(obj))
for tag in extension.tag_set.all():
if tag.value:
attributes[self.map_tag(tag.name)] = tag.value
# check and create uuid
uuid_attribute = self.map_tag('uuid')
try:
attributes[uuid_attribute]
except KeyError:
attributes[uuid_attribute] = uuid.uuid4().hex
Tag.objects.create(extension=extension,
name='uuid',
value=attributes[uuid_attribute])
return attributes
def validate_data(self):
"""Validate parsed data based on healthsites rules.
:return: Flag indicating whether the data is valid or not.
:rtype: bool
"""
self._validation_status['total'] = len(self._parsed_data)
for row_number, data in enumerate(self._parsed_data):
is_valid = True
validation_status = {'is_valid': is_valid, 'message': 'Valid'}
# check 2nd row based on the hxl
if row_number == 0 and 'lat' in data and '#' in data['lat']:
mapping_file_path = ABS_PATH('api', 'fixtures',
'hxl_tags.json')
_file = open(mapping_file_path, 'r')
hxl_tag = json.loads(_file.read())
_file.close()
# get the newest hxl tag
all_tag = copy.deepcopy(data['tag'])
all_tag['lon'] = data['lon']
all_tag['lat'] = data['lat']
for tag, hxl in all_tag.items():
try:
hxl_tag[tag]
except KeyError:
hxl_tag[tag] = hxl
_file = open(mapping_file_path, 'w')
_file.write(json.dumps(hxl_tag, indent=4))
_file.close()
continue
# Set default user to data loader author
user = self.data_loader.author
# Split osm and extension attribute
osm_attr, locality_attr = split_osm_and_extension_attr(data['tag'])
data['tag'] = osm_attr
data['tag'].update(locality_attr)
# Verify data uploader and owner/collector if the API is being used
# for uploading data from other osm user.
if not data.get('osm_user'):
data['osm_user'] = user.username
if user.username != data.get('osm_user'):
is_valid, message = verify_user(user,
data['osm_user'],
ignore_uploader_staff=True)
validation_status.update({
'is_valid': is_valid,
'message': message
})
if is_valid:
try:
_ = get_object_or_404(User, username=data['osm_user'])
except Http404:
message = 'User %s is not exist.' % data['osm_user']
validation_status.update({
'is_valid': False,
'message': message
})
if is_valid:
try:
# Validate data
is_valid = validate_osm_data(data)
validation_status.update({
'is_valid': is_valid,
'message': 'OSM data is valid.'
})
except Exception as e:
validation_status.update({
'is_valid': False,
'message': '%s' % e
})
self._validation_status['status'][row_number +
1] = validation_status
if self.is_valid() and row_number + 1 == len(self._parsed_data):
# prepare for the next step, upload data
self._validation_status['count'] = row_number
else:
self._validation_status['count'] = row_number + 1
self._validation_status['summary'] = (
'Validation failed. Please see the status detail for more '
'information.' if not self.is_valid() else '')
# write status to progress file
f = open(self.progress_file, 'w+')
f.write(json.dumps(self._validation_status))
f.close()
del self._parsed_data[0]
return False not in ([
status['is_valid']
for status in self._validation_status['status'].values()
])
def post(self, request):
user = request.user
data = copy.deepcopy(request.data)
if user.username in settings.TEST_USERS:
raise Exception('Create osm : {}'.format(
json.dumps({
'payload': data,
'user': user.username
})))
# Now, we post the data directly to OSM.
try:
# Split osm and extension attribute
osm_attr, locality_attr = split_osm_and_extension_attr(data['tag'])
data['tag'] = osm_attr
# Verify data uploader and owner/collector if the API is being used
# for uploading data from other osm user.
if request.user.is_staff and request.GET.get('review', None):
data['osm_user'] = get_pending_review(
request.GET.get('review')).uploader.username
if data.get('osm_user'):
is_valid, message = verify_user(user, data['osm_user'])
if not is_valid:
return HttpResponseForbidden(message)
else:
try:
user = get_object_or_404(User,
username=data['osm_user'])
except Http404:
message = 'User %s is not exist.' % data['osm_user']
return HttpResponseForbidden(message)
duplication_check = request.GET.get('duplication-check', True)
if duplication_check == 'false':
duplication_check = False
validate_osm_data(data, duplication_check=duplication_check)
# Map Healthsites tags to OSM tags
mapping_file_path = ABS_PATH('api', 'fixtures', 'mapping.yml')
data['tag'] = convert_to_osm_tag(mapping_file_path, data['tag'],
'node')
# Push data to OSM
response = create_osm_node(user, data)
# create pending index
create_pending_update('node', response['id'], data['tag']['name'],
user, response['version'])
save_extensions('node', response['id'], locality_attr)
if request.GET.get('review', None):
delete_pending_review(request.GET.get('review', None))
return Response(response)
except Exception as e:
if not request.GET.get('review', None):
if user != request.user:
create_pending_review(user, request.data, '%s' % e)
else:
try:
update_pending_review(request.GET.get('review', None),
request.data, '%s' % e)
except Exception as e:
return HttpResponseBadRequest('%s' % e)
output = {
'error': '%s' % e,
'payload': request.data,
}
return HttpResponseBadRequest('%s' % json.dumps(output))
def post(self, request, osm_type, osm_id):
data = copy.deepcopy(request.data)
user = request.user
# delete uuid, because it is not editable
try:
del data['tag']['uuid']
except KeyError:
pass
# Now, we post the data directly to OSM.
try:
if osm_type == 'node':
osm_function = update_osm_node
elif osm_type == 'way':
osm_function = update_osm_way
else:
# For now, we only support Node
return HttpResponseBadRequest(
'%s is not supported as osm type' % osm_type)
locality = self.getLocalityOsm(osm_type, osm_id)
data['id'] = osm_id
data['type'] = osm_type
data['version'] = locality.changeset_version
# Verify data uploader and owner/collector if the API is being
# used for uploading data from other osm user.
if request.user.is_staff and request.GET.get('review', None):
data['osm_user'] = get_pending_review(
request.GET.get('review')).uploader.username
if data.get('osm_user'):
is_valid, message = verify_user(user, data['osm_user'])
if not is_valid:
return HttpResponseForbidden(message)
else:
try:
user = get_object_or_404(User,
username=data['osm_user'])
except Http404:
message = 'User %s is not exist.' % data['osm_user']
return HttpResponseForbidden(message)
# Validate data
validate_osm_data(data, duplication_check=False)
# Map Healthsites tags to OSM tags
mapping_file_path = ABS_PATH('api', 'fixtures', 'mapping.yml')
data['tag'] = convert_to_osm_tag(mapping_file_path, data['tag'],
osm_type)
# Push data to OSM
response = osm_function(user, data)
create_pending_update(osm_type, response['id'],
data['tag']['name'], user,
response['version'])
if request.GET.get('review', None):
delete_pending_review(request.GET.get('review', None))
return Response(response)
except KeyError as e:
return HttpResponseBadRequest('%s is needed' % e)
except Exception as e:
if not request.GET.get('review', None):
if user != request.user:
create_pending_review(user, request.data, '%s' % e)
else:
try:
update_pending_review(request.GET.get('review', None),
request.data, '%s' % e)
except Exception as e:
return HttpResponseBadRequest('%s' % e)
output = {
'error': '%s' % e,
'payload': request.data,
}
return HttpResponseBadRequest('%s' % json.dumps(output))
except (LocalityOSMNode.DoesNotExist, LocalityOSMNode.DoesNotExist):
raise Http404()
# Language code for this installation. All choices can be found here:
# http://www.i18nguy.com/unicode/language-identifiers.html
LANGUAGE_CODE = 'en-us'
# If you set this to False, Django will make some optimizations so as not
# to load the internationalization machinery.
USE_I18N = True
# If you set this to False, Django will not format dates, numbers and
# calendars according to the current locale.
USE_L10N = True
# If you set this to False, Django will not use timezone-aware datetimes.
USE_TZ = True
BASE_DIR = ABS_PATH('')
# Absolute filesystem path to the directory that will hold user-uploaded files.
# Example: "/var/www/example.com/media/"
MEDIA_ROOT = '/home/web/media'
# URL that handles the media served from MEDIA_ROOT. Make sure to use a
# trailing slash.
# Examples: "http://example.com/media/", "http://media.example.com/"
MEDIA_URL = '/media/'
# Absolute path to the directory static files should be collected to.
# Don't put anything in this directory yourself; store your static files
# in apps' "static/" subdirectories and in STATICFILES_DIRS.
# Example: "/var/www/example.com/static/"
STATIC_ROOT = '/home/web/static'