def import_and_extract_shapefile(feature_type, file_path, qgis_version=2, output_prefix='', inasafe_version=None, lang='en'): """Convert the OSM xml file to a shapefile. This is a multi-step process: * Create a temporary postgis database * Load the osm dataset into POSTGIS with osm2pgsql and our custom style file. * Save the data out again to a shapefile * Zip the shapefile ready for user to download :param feature_type: The feature to extract. :type feature_type: str :param file_path: Path to the OSM file name. :type file_path: str :param qgis_version: Get the QGIS version. Currently 1, 2 are accepted, default to 2. A different qml style file will be returned depending on the version :type qgis_version: int :param output_prefix: Base name for the shape file. Defaults to '' which will result in an output file of feature_type + '.shp'. Adding a prefix of e.g. 'test-' would result in a downloaded file name of 'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9]. :type output_prefix: str :param inasafe_version: The InaSAFE version, to get correct metadata. :type inasafe_version: str :param lang: The language desired for the labels in the legend. Example : 'en', 'fr', etc. Default is 'en'. :type lang: str :returns: Path to zipfile that was created. :rtype: str """ if not check_string(output_prefix): error = 'Invalid output prefix: %s' % output_prefix LOGGER.exception(error) raise Exception(error) output_prefix += feature_type work_dir = temp_dir(sub_dir=feature_type) directory_name = unique_filename(dir=work_dir) db_name = os.path.basename(directory_name) import_osm_file(db_name, feature_type, file_path) zip_file = extract_shapefile(feature_type, db_name, directory_name, qgis_version, output_prefix, inasafe_version, lang) drop_database(db_name) return zip_file
def test_home(self): """Test the home page works.""" try: return self.app.post('/', data=dict(), follow_redirects=True) except Exception, e: LOGGER.exception('Basic front page load failed.') raise e
def test_home(self): """Test the home page works.""" try: result = self.app.get('/', data=dict(), follow_redirects=True) code = result.status_code self.assertEquals(code, 200) except Exception as e: LOGGER.exception('Basic front page load failed.') raise e
def request_data(self, url_request): web_request = Request(url_request, None, self.headers) try: url_handle = urlopen(web_request, timeout=60) data = url_handle.read().decode('utf-8') return data except HTTPError as e: LOGGER.exception('Error with request') return e.msg
def test_home(self): """Test the home page works.""" try: result = self.app.get( '/', data=dict(), follow_redirects=True) code = result.status_code self.assertEquals(code, 200) except Exception as e: LOGGER.exception('Basic front page load failed.') raise e
def fetch_osm_with_post(file_path, url_path, post_data, returns_format='json'): """Fetch an osm map and store locally. :param url_path: The path (relative to the ftp root) from which the file should be retrieved. :type url_path: str :param file_path: The path on the filesystem to which the file should be saved. :type file_path: str :param post_data: Overpass data :type post_data: str :param returns_format: Format of the response, could be json or xml :type returns_format: str :returns: The path to the downloaded file. """ headers = {'User-Agent': 'HotOSM'} try: data = requests.post(url=url_path, data={'data': post_data}, headers=headers) if returns_format != 'xml': regex = '<remark> runtime error:' if re.search(regex, data.text): raise OverpassTimeoutException regex = '(elements|meta)' if not re.search(regex, data.text): raise OverpassDoesNotReturnData if os.path.exists(file_path): os.remove(file_path) file_handle = open(file_path, 'wb') file_handle.write(data.text.encode('utf-8')) file_handle.close() except HTTPError as e: if e.code == 400: LOGGER.exception('Bad request to Overpass') raise OverpassBadRequestException elif e.code == 419: raise OverpassConcurrentRequestException LOGGER.exception('Error with Overpass') raise e
def fetch_osm(file_path, url_path): """Fetch an osm map and store locally. :param url_path: The path (relative to the ftp root) from which the file should be retrieved. :type url_path: str :param file_path: The path on the filesystem to which the file should be saved. :type file_path: str :returns: The path to the downloaded file. """ LOGGER.debug('Getting URL: %s', url_path) headers = {'User-Agent': 'InaSAFE'} web_request = Request(url_path, None, headers) try: url_handle = urlopen(web_request, timeout=60) data = url_handle.read().decode('utf-8') regex = '<remark> runtime error:' if re.search(regex, data): raise OverpassTimeoutException regex = '(elements|meta)' if not re.search(regex, data): raise OverpassDoesNotReturnData if os.path.exists(file_path): os.remove(file_path) file_handle = open(file_path, 'wb') file_handle.write(data.encode('utf-8')) file_handle.close() except HTTPError as e: if e.code == 400: LOGGER.exception('Bad request to Overpass') raise OverpassBadRequestException elif e.code == 419: raise OverpassConcurrentRequestException LOGGER.exception('Error with Overpass') raise e
def user_status(): """Get nodes for user as a json doc. .. note:: User from reporter.js To use e.g.: http://localhost:5000/user?bbox=20.431909561157227, -34.02849543118406,20.45207977294922,-34.02227106658948& obj=building&username=timlinux """ username = request.args.get('username') bbox = request.args.get('bbox') try: coordinates = split_bbox(bbox) except ValueError: error = "Invalid bbox" coordinates = split_bbox(config.BBOX) LOGGER.exception(error + str(coordinates)) else: try: file_handle = get_osm_file(coordinates) except OverpassTimeoutException: error = "Bad request. Maybe the bbox is too big!" LOGGER.exception(error + str(coordinates)) except OverpassConcurrentRequestException: error = 'Please try again later, another query is running.' LOGGER.exception(error + str(coordinates)) except OverpassBadRequestException: error = "Bad request." LOGGER.exception(error + str(coordinates)) except URLError: error = "Bad request." LOGGER.exception(error + str(coordinates)) else: node_data = osm_nodes_by_user(file_handle, username) return jsonify(d=node_data)
def user_status(): """Get nodes for user as a json doc. .. note:: User from reporter.js To use e.g.: http://localhost:5000/user?bbox=20.431909561157227, -34.02849543118406,20.45207977294922,-34.02227106658948& obj=building&username=timlinux """ username = request.args.get('username') bbox = request.args.get('bbox') try: coordinates = split_bbox(bbox) except ValueError: error = "Invalid bbox" coordinates = split_bbox(config.BBOX) LOGGER.exception(error + str(coordinates)) else: try: file_handle = get_osm_file(coordinates) except OverpassTimeoutException: error = "Bad request. Maybe the bbox is too big!" LOGGER.exception(error + str(coordinates)) except OverpassConcurrentRequestException: error = 'Please try again later, another query is running.' LOGGER.exception(error + str(coordinates)) except OverpassBadRequestException: error = "Bad request." LOGGER.exception(error + str(coordinates)) except URLError: error = "Bad request." LOGGER.exception(error + str(coordinates)) else: node_data = osm_nodes_by_user(file_handle, username) return jsonify(d=node_data)
def fetch_osm(file_path, url_path): """Fetch an osm map and store locally. :param url_path: The path (relative to the ftp root) from which the file should be retrieved. :type url_path: str :param file_path: The path on the filesystem to which the file should be saved. :type file_path: str :returns: The path to the downloaded file. """ LOGGER.debug('Getting URL: %s', url_path) headers = {'User-Agent': 'InaSAFE'} web_request = Request(url_path, None, headers) try: url_handle = urlopen(web_request, timeout=60) data = url_handle.read().decode('utf-8') regex = '<remark> runtime error:' if re.search(regex, data): raise OverpassTimeoutException file_handle = open(file_path, 'wb') file_handle.write(data.encode('utf-8')) file_handle.close() except HTTPError as e: if e.code == 400: LOGGER.exception('Bad request to Overpass') raise OverpassBadRequestException elif e.code == 419: raise OverpassConcurrentRequestException LOGGER.exception('Error with Overpass') raise e
def osm_object_contributions(osm_file, tag_name): """Compile a summary of user contributions for the selected osm data type. :param osm_file: A file object reading from a .osm file. :type osm_file: file, FileIO :param tag_name: The tag name we want to filter on. :type tag_name: str :returns: A list of dicts where items in the list are sorted from highest contributor (based on number of ways) down to lowest. Each element in the list is a dict in the form: { 'user': <user>, 'ways': <way count>, 'nodes': <node count>, 'timeline': <timelinedict>, 'best': <most ways in a single day>, 'worst': <least ways in single day>, 'average': <average ways across active days>, 'crew': <bool> } where crew is used to designate users who are part of an active data gathering campaign. The timeline dict will contain a collection of dates and the total number of ways created on that date e.g. { u'2010-12-09': 10, u'2012-07-10': 14 } :rtype: list """ parser = OsmParser(tag_name=tag_name) try: xml.sax.parse(osm_file, parser) except xml.sax.SAXParseException: LOGGER.exception('Failed to parse OSM xml.') raise way_count_dict = parser.wayCountDict node_count_dict = parser.nodeCountDict timelines = parser.userDayCountDict # Convert to a list of dicts so we can sort it. crew_list = config.CREW user_list = [] for key, value in way_count_dict.items(): crew_flag = False if key in crew_list: crew_flag = True start_date, end_date = date_range(timelines[key]) start_date = time.strftime('%d-%m-%Y', start_date.timetuple()) end_date = time.strftime('%d-%m-%Y', end_date.timetuple()) user_timeline = timelines[key] record = { 'name': key, 'ways': value, 'nodes': node_count_dict[key], 'timeline': interpolated_timeline(user_timeline), 'start': start_date, 'end': end_date, 'activeDays': len(user_timeline), 'best': best_active_day(user_timeline), 'worst': worst_active_day(user_timeline), 'average': average_for_active_days(user_timeline), 'crew': crew_flag } user_list.append(record) # Sort it sorted_user_list = sorted( user_list, key=lambda d: ( -d['ways'], d['nodes'], d['name'], d['timeline'], d['start'], d['end'], d['activeDays'], d['best'], d['worst'], d['average'], d['crew'])) return sorted_user_list
def osm_object_contributions(osm_file, tag_name, date_start=None, date_end=None): """Compile a summary of user contributions for the selected osm data type. :param osm_file: A file object reading from a .osm file. :type osm_file: file, FileIO :param tag_name: The tag name we want to filter on. :type tag_name: str :param date_start: The start date we want to filter :type date_start: float :param date_end: The end date we want to filter :type date_end: float :returns: A list of dicts where items in the list are sorted from highest contributor (based on number of ways) down to lowest. Each element in the list is a dict in the form: { 'user': <user>, 'ways': <way count>, 'nodes': <node count>, 'timeline': <timelinedict>, 'best': <most ways in a single day>, 'worst': <least ways in single day>, 'average': <average ways across active days>, 'crew': <bool> } where crew is used to designate users who are part of an active data gathering campaign. The timeline dict will contain a collection of dates and the total number of ways created on that date e.g. { u'2010-12-09': 10, u'2012-07-10': 14 } :rtype: list """ parser = OsmParser(start_date=date_start, end_date=date_end) try: xml.sax.parse(osm_file, parser) except xml.sax.SAXParseException: LOGGER.exception('Failed to parse OSM xml.') raise way_count_dict = parser.wayCountDict node_count_dict = parser.nodeCountDict timelines = parser.userDayCountDict # Convert to a list of dicts so we can sort it. crew_list = config.CREW user_list = [] for key, value in way_count_dict.items(): start_date, end_date = date_range(timelines[key]) start_date = time.strftime('%d-%m-%Y', start_date.timetuple()) end_date = time.strftime('%d-%m-%Y', end_date.timetuple()) user_timeline = timelines[key] node_count = 0 if key in node_count_dict: node_count = node_count_dict[key] record = { 'name': key, 'ways': value, 'nodes': node_count, 'timeline': interpolated_timeline(user_timeline), 'start': start_date, 'end': end_date, 'activeDays': len(user_timeline), 'best': best_active_day(user_timeline), 'worst': worst_active_day(user_timeline), 'average': average_for_active_days(user_timeline) } user_list.append(record) for key, value in node_count_dict.items(): start_date, end_date = date_range(timelines[key]) start_date = time.strftime('%d-%m-%Y', start_date.timetuple()) end_date = time.strftime('%d-%m-%Y', end_date.timetuple()) user_timeline = timelines[key] record = { 'name': key, 'ways': 0, 'nodes': value, 'timeline': interpolated_timeline(user_timeline), 'start': start_date, 'end': end_date, 'activeDays': len(user_timeline), 'best': best_active_day(user_timeline), 'worst': worst_active_day(user_timeline), 'average': average_for_active_days(user_timeline) } user_list.append(record) # Sort it sorted_user_list = sorted( user_list, key=lambda d: (-d['ways'], d['nodes'], d['name'], d['timeline'], d['start'], d[ 'end'], d['activeDays'], d['best'], d['worst'], d['average'])) return sorted_user_list
def import_and_extract_shapefile( feature_type, file_path, qgis_version=2, output_prefix='', inasafe_version=None, lang='en'): """Convert the OSM xml file to a shapefile. This is a multi-step process: * Create a temporary postgis database * Load the osm dataset into POSTGIS with osm2pgsql and our custom style file. * Save the data out again to a shapefile * Zip the shapefile ready for user to download :param feature_type: The feature to extract. :type feature_type: str :param file_path: Path to the OSM file name. :type file_path: str :param qgis_version: Get the QGIS version. Currently 1, 2 are accepted, default to 2. A different qml style file will be returned depending on the version :type qgis_version: int :param output_prefix: Base name for the shape file. Defaults to '' which will result in an output file of feature_type + '.shp'. Adding a prefix of e.g. 'test-' would result in a downloaded file name of 'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9]. :type output_prefix: str :param inasafe_version: The InaSAFE version, to get correct metadata. :type inasafe_version: str :param lang: The language desired for the labels in the legend. Example : 'en', 'fr', etc. Default is 'en'. :type lang: str :returns: Path to zipfile that was created. :rtype: str """ if not check_string(output_prefix): error = 'Invalid output prefix: %s' % output_prefix LOGGER.exception(error) raise Exception(error) output_prefix += feature_type work_dir = temp_dir(sub_dir=feature_type) directory_name = unique_filename(dir=work_dir) db_name = os.path.basename(directory_name) import_osm_file(db_name, feature_type, file_path) zip_file = extract_shapefile( feature_type, db_name, directory_name, qgis_version, output_prefix, inasafe_version, lang) drop_database(db_name) return zip_file