示例#1
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    response = views.list_resources(resource_type)

    if request.url_rule.rule == '/json':
        json_dict = {'total': response['total'], 'resources': []}
        for r in response['resources']:
            ghc_url = '%s/resource/%s' % (GHC_SITE_URL, r.identifier)
            json_dict['resources'].append({
                'resource_type':
                r.resource_type,
                'title':
                r.title,
                'url':
                r.url,
                'ghc_url':
                ghc_url,
                'ghc_json':
                '%s/json' % ghc_url,
                'ghc_csv':
                '%s/csv' % ghc_url,
                'last_check':
                r.last_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                'status':
                r.last_run.success,
                'response_time':
                round(r.average_response_time, 2),
                'reliability':
                round(r.reliability, 2)
            })
        return jsonify(json_dict)
    elif request.url_rule.rule == '/csv':
        output = StringIO()
        writer = csv.writer(output)
        header = [
            'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
            'last_check', 'status', 'response_time', 'reliability'
        ]
        writer.writerow(header)
        for r in response['resources']:
            ghc_url = '%s%s' % (GHC_SITE_URL,
                                url_for('get_resource_by_id',
                                        identifier=r.identifier))
            writer.writerow([
                r.resource_type, r.title, r.url, ghc_url,
                '%s/json' % ghc_url,
                '%s/csv' % ghc_url,
                r.last_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                r.last_run.success,
                round(r.average_response_time, 2),
                round(r.reliability, 2)
            ])
        return output.getvalue()
def run_test_resource(resource_type, url):
    """tests a CSW service and provides run metrics"""

    if resource_type not in RESOURCE_TYPES.keys():
        msg = gettext('Invalid resource type')
        msg2 = '%s: %s' % (msg, resource_type)
        LOGGER.error(msg2)
        raise RuntimeError(msg2)

    title = None
    start_time = datetime.datetime.utcnow()
    message = None

    try:
        if resource_type == 'OGC:WMS':
            ows = WebMapService(url)
        elif resource_type == 'OGC:WFS':
            ows = WebFeatureService(url)
        elif resource_type == 'OGC:WCS':
            ows = WebCoverageService(url)
        elif resource_type == 'OGC:WPS':
            ows = WebProcessingService(url)
        elif resource_type == 'OGC:CSW':
            ows = CatalogueServiceWeb(url)
        elif resource_type == 'OGC:SOS':
            ows = SensorObservationService(url)
        elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
            ows = urlopen(url)
            if resource_type == 'WWW:LINK':
                import re
                try:
                    title_re = re.compile("<title>(.+?)</title>")
                    title = title_re.search(ows.read()).group(1)
                except:
                    title = url
            elif resource_type == 'urn:geoss:waf':
                title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
        elif resource_type == 'FTP':
            ows = urlopen(url)
            title = urlparse(url).hostname
        success = True
        if resource_type.startswith('OGC:'):
            title = ows.identification.title
        if title is None:
            title = '%s %s %s' % (resource_type, gettext('for'), url)
    except Exception as err:
        msg = str(err)
        LOGGER.exception(msg)
        message = msg
        success = False

    end_time = datetime.datetime.utcnow()

    delta = end_time - start_time
    response_time = '%s.%s' % (delta.seconds, delta.microseconds)

    return [title, success, response_time, message, start_time]
示例#3
0
def home():
    """homepage"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    response = views.list_resources(resource_type)
    return render_template('home.html', response=response)
示例#4
0
def home():
    """homepage"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    response = views.list_resources(resource_type)
    return render_template('home.html', response=response)
示例#5
0
def resources():
    """lists resources with optional filter"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    tag = request.args.get('tag')

    query = request.args.get('q')

    response = views.list_resources(resource_type, query, tag)
    return render_template('resources.html', response=response)
示例#6
0
def resources():
    """lists resources with optional filter"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    tag = request.args.get('tag')

    query = request.args.get('q')

    response = views.list_resources(resource_type, query, tag)
    return render_template('resources.html', response=response)
示例#7
0
def run_test_resource(resource_type, url):
    """tests a CSW service and provides run metrics"""

    if resource_type not in RESOURCE_TYPES.keys():
        msg = 'Invalid resource type: %s' % resource_type
        LOGGER.error(msg)
        raise RuntimeError(msg)

    title = None
    start_time = datetime.datetime.utcnow()
    message = None

    try:
        if resource_type == 'OGC:WMS':
            ows = WebMapService(url)
        elif resource_type == 'OGC:WFS':
            ows = WebFeatureService(url)
        elif resource_type == 'OGC:WCS':
            ows = WebCoverageService(url)
        elif resource_type == 'OGC:WPS':
            ows = WebProcessingService(url)
        elif resource_type == 'OGC:CSW':
            ows = CatalogueServiceWeb(url)
        elif resource_type == 'OGC:SOS':
            ows = SensorObservationService(url)
        elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
            ows = urlopen(url)
            if resource_type == 'WWW:LINK':
                import re
                try:
                    title_re = re.compile("<title>(.+?)</title>")
                    title = title_re.search(ows.read()).group(1)
                except:
                    title = url
            elif resource_type == 'urn:geoss:waf':
                title = 'WAF for %s' % urlparse(url).hostname
        elif resource_type == 'FTP':
            ows = urlopen(url)
            title = urlparse(url).hostname
        success = True
        if resource_type.startswith('OGC:'):
            title = ows.identification.title
        if title is None:
            title = '%s for %s' % (resource_type, url)
    except Exception, err:
        msg = str(err)
        LOGGER.exception(msg)
        message = msg
        success = False
示例#8
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    response = views.list_resources(resource_type)
    json_dict = {'resources': []}
    for r in response['resources']:
        json_dict['resources'].append({
            'resource_type': r.resource_type,
            'title': r.title,
            'url': r.url
        })
    return jsonify(json_dict)
示例#9
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    response = views.list_resources(resource_type)
    json_dict = {'resources': []}
    for r in response['resources']:
        json_dict['resources'].append({
            'resource_type': r.resource_type,
            'title': r.title,
            'url': r.url
        })
    return jsonify(json_dict)
示例#10
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    query = request.args.get('q')

    response = views.list_resources(resource_type, query)

    if request.url_rule.rule == '/json':
        json_dict = {'total': response['total'], 'resources': []}
        for r in response['resources']:
            ghc_url = '%s/resource/%s' % (CONFIG['GHC_SITE_URL'], r.identifier)
            json_dict['resources'].append({
                'resource_type':
                r.resource_type,
                'title':
                r.title,
                'url':
                r.url,
                'ghc_url':
                ghc_url,
                'ghc_json':
                '%s/json' % ghc_url,
                'ghc_csv':
                '%s/csv' % ghc_url,
                'first_run':
                r.first_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                'last_run':
                r.last_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                'status':
                r.last_run.success,
                'min_response_time':
                round(r.min_response_time, 2),
                'average_response_time':
                round(r.average_response_time, 2),
                'max_response_time':
                round(r.max_response_time, 2),
                'reliability':
                round(r.reliability, 2),
                'last_report':
                r.last_run.report,
                'recipients':
                r.dump_recipients()
            })
        return jsonify(json_dict)
    elif request.url_rule.rule == '/csv':
        output = StringIO()
        writer = csv.writer(output)
        header = [
            'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
            'first_run', 'last_run', 'status', 'min_response_time',
            'average_response_time', 'max_response_time', 'reliability',
            'recipients'
        ]
        writer.writerow(header)
        for r in response['resources']:
            ghc_url = '%s%s' % (CONFIG['GHC_SITE_URL'],
                                url_for('get_resource_by_id',
                                        identifier=r.identifier))
            # serialize recipients into a string
            recipients = r.dump_recipients()
            recipients_str = json.dumps(recipients)

            writer.writerow([
                r.resource_type, r.title, r.url, ghc_url,
                '%s/json' % ghc_url,
                '%s/csv' % ghc_url,
                r.first_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                r.last_run.checked_datetime.strftime('%Y-%m-%dT%H:%M:%SZ'),
                r.last_run.success,
                round(r.average_response_time, 2),
                round(r.reliability, 2), recipients_str
            ])
        return output.getvalue(), 200, {'Content-type': 'text/csv'}
示例#11
0
def sniff_test_resource(config, resource_type, url):
    """tests a Resource endpoint for general compliance"""

    out = []
    tag_list = []
    if resource_type not in RESOURCE_TYPES.keys():
        msg = gettext('Invalid resource type')
        msg2 = '%s: %s' % (msg, resource_type)
        LOGGER.error(msg2)
        raise RuntimeError(msg2)

    title = None
    start_time = datetime.utcnow()
    message = None
    resource_type_map = {'OGC:WMS': [partial(WebMapService, version='1.3.0'),
                                     partial(WebMapService, version='1.1.1')],
                         'OGC:WMTS': [WebMapTileService],
                         'OSGeo:TMS': [TileMapService],
                         'OGC:WFS': [WebFeatureService],
                         'OGC:WCS': [WebCoverageService],
                         'OGC:WPS': [WebProcessingService],
                         'OGC:CSW': [CatalogueServiceWeb],
                         'OGC:SOS': [SensorObservationService],
                         'OGC:WFS3': [urlopen],
                         'ESRI:FS': [urlopen],
                         'OGC:STA': [urlopen],
                         'WWW:LINK': [urlopen],
                         'FTP': [urlopen],
                         'GHC:Report': [urlopen],
                         'OSGeo:GeoNode': [geonode_get_ows],
                         }
    try:
        ows = None
        try:
            ows_handlers = resource_type_map[resource_type]
        except KeyError:
            LOGGER.error("No handler for %s type", resource_type)
            raise
        for ows_handler in ows_handlers:
            try:
                ows = ows_handler(url)
                break
            except Exception as err:
                LOGGER.warning("Cannot use %s on %s: %s",
                               ows_handler, url, err, exc_info=err)
        if ows is None:
            message = ("Cannot get {} service instance "
                       "for {}".format(resource_type, url))
            raise ValueError(message)

        if resource_type == 'WWW:LINK':
            content_type = ows.info().getheader('Content-Type')

            # Check content if the response is not an image
            if 'image/' not in content_type:
                content = ows.read()
                import re
                try:
                    title_re = re.compile("<title>(.+?)</title>")
                    title = title_re.search(content).group(1)
                except Exception:
                    title = url

                # Optional check for any OGC-Exceptions in Response
                if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
                    exception_text = None
                    try:
                        except_re = re.compile(
                            "ServiceException>|ExceptionReport>")
                        exception_text = except_re.search(content).group(0)
                    except Exception:
                        # No Exception in Response text
                        pass

                    if exception_text:
                        # Found OGC-Exception in Response text
                        raise Exception(
                            "Exception in response: %s" % exception_text)

                del content

        elif resource_type == 'urn:geoss:waf':
            title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
        elif resource_type == 'FTP':
            title = urlparse(url).hostname
        elif resource_type == 'OSGeo:GeoNode':
            endpoints = ows
            end_time = datetime.utcnow()
            delta = end_time - start_time
            response_time = '%s.%s' % (delta.seconds, delta.microseconds)
            base_tags = geonode_make_tags(url)

            for epoint in endpoints:
                row = sniff_test_resource(config,
                                          epoint['type'],
                                          epoint['url'])
                if row:
                    _tags = row[0][-1]
                    _tags.extend(base_tags)
                    row[0][-1] = _tags
                    out.append(row[0])

        elif resource_type.startswith(('OGC:', 'OSGeo', 'ESRI')):
            if resource_type == 'OGC:STA':
                title = 'OGC STA'
            elif resource_type == 'OGC:WFS3':
                title = 'OGC API Features (OAFeat)'
            elif resource_type == 'ESRI:FS':
                title = 'ESRI ArcGIS FS'
            else:
                title = ows.identification.title
        if title is None:
            title = '%s %s %s' % (resource_type, gettext('for'), url)
        success = True
    except Exception as err:
        title = 'Untitled'
        msg = 'Getting metadata failed: %s' % str(err)
        LOGGER.error(msg, exc_info=err)
        message = msg
        success = False

    end_time = datetime.utcnow()

    delta = end_time - start_time
    response_time = '%s.%s' % (delta.seconds, delta.microseconds)
    # if out is not populated yet, that means it should be populated now
    if not out:
        out.append([resource_type,
                    url,
                    title,
                    success,
                    response_time,
                    message,
                    start_time,
                    tag_list])
    return out
示例#12
0
def sniff_test_resource(config, resource_type, url):
    """tests a Resource endpoint for general compliance"""

    out = []
    tag_list = []
    if resource_type not in RESOURCE_TYPES.keys():
        msg = gettext('Invalid resource type')
        msg2 = '%s: %s' % (msg, resource_type)
        LOGGER.error(msg2)
        raise RuntimeError(msg2)

    title = None
    start_time = datetime.utcnow()
    message = None
    resource_type_map = {'OGC:WMS': [partial(WebMapService, version='1.3.0'),
                                     partial(WebMapService, version='1.1.1')],
                         'OGC:WMTS': [WebMapTileService],
                         'OSGeo:TMS': [TileMapService],
                         'OGC:WFS': [WebFeatureService],
                         'OGC:WCS': [WebCoverageService],
                         'OGC:WPS': [WebProcessingService],
                         'OGC:CSW': [CatalogueServiceWeb],
                         'OGC:SOS': [SensorObservationService],
                         'OGC:STA': [urlopen],
                         'WWW:LINK': [urlopen],
                         'FTP': [urlopen],
                         'OSGeo:GeoNode': [geonode_get_ows],
                         }
    try:
        ows = None
        try:
            ows_handlers = resource_type_map[resource_type]
        except KeyError:
            LOGGER.error("No handler for %s type", resource_type)
            raise
        for ows_handler in ows_handlers:
            try:
                ows = ows_handler(url)
                break
            except Exception, err:
                LOGGER.warning("Cannot use %s on %s: %s",
                               ows_handler, url, err, exc_info=err)
        if ows is None:
            message = ("Cannot get {} service instance "
                       "for {}".format(resource_type, url))
            raise ValueError(message)

        if resource_type == 'WWW:LINK':
            content_type = ows.info().getheader('Content-Type')

            # Check content if the response is not an image
            if 'image/' not in content_type:
                content = ows.read()
                import re
                try:
                    title_re = re.compile("<title>(.+?)</title>")
                    title = title_re.search(content).group(1)
                except Exception:
                    title = url

                # Optional check for any OGC-Exceptions in Response
                if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
                    exception_text = None
                    try:
                        except_re = re.compile(
                            "ServiceException>|ExceptionReport>")
                        exception_text = except_re.search(content).group(0)
                    except Exception:
                        # No Exception in Response text
                        pass

                    if exception_text:
                        # Found OGC-Exception in Response text
                        raise Exception(
                            "Exception in response: %s" % exception_text)

                del content

        elif resource_type == 'urn:geoss:waf':
            title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
        elif resource_type == 'FTP':
            title = urlparse(url).hostname
        elif resource_type == 'OSGeo:GeoNode':
            endpoints = ows
            end_time = datetime.utcnow()
            delta = end_time - start_time
            response_time = '%s.%s' % (delta.seconds, delta.microseconds)
            base_tags = geonode_make_tags(url)

            for epoint in endpoints:
                row = sniff_test_resource(config,
                                          epoint['type'],
                                          epoint['url'])
                if row:
                    _tags = row[0][-1]
                    _tags.extend(base_tags)
                    row[0][-1] = _tags
                    out.append(row[0])

        elif resource_type.startswith(('OGC:', 'OSGeo')):
            if resource_type == 'OGC:STA':
                title = 'OGC STA'
            else:
                title = ows.identification.title
        if title is None:
            title = '%s %s %s' % (resource_type, gettext('for'), url)
        title = title.decode('utf-8')
        success = True
示例#13
0
def sniff_test_resource(config, resource_type, url):
    """tests a service and provides run metrics"""

    if resource_type not in RESOURCE_TYPES.keys():
        msg = gettext('Invalid resource type')
        msg2 = '%s: %s' % (msg, resource_type)
        LOGGER.error(msg2)
        raise RuntimeError(msg2)

    title = None
    start_time = datetime.datetime.utcnow()
    message = None

    try:
        if resource_type == 'OGC:WMS':
            ows = WebMapService(url)
        elif resource_type == 'OGC:WMTS':
            ows = WebMapTileService(url)
        elif resource_type == 'OSGeo:TMS':
            ows = TileMapService(url)
        elif resource_type == 'OGC:WFS':
            ows = WebFeatureService(url)
        elif resource_type == 'OGC:WCS':
            ows = WebCoverageService(url, version='1.0.0')
        elif resource_type == 'OGC:WPS':
            ows = WebProcessingService(url)
        elif resource_type == 'OGC:CSW':
            ows = CatalogueServiceWeb(url)
        elif resource_type == 'OGC:SOS':
            ows = SensorObservationService(url)
        elif resource_type == 'OGC:STA':
            ows = urlopen(url)
        elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
            ows = urlopen(url)
            if resource_type == 'WWW:LINK':
                content_type = ows.info().getheader('Content-Type')

                # Check content if the response is not an image
                if 'image/' not in content_type:
                    content = ows.read()
                    import re
                    try:
                        title_re = re.compile("<title>(.+?)</title>")
                        title = title_re.search(content).group(1)
                    except:
                        title = url

                    # Optional check for any OGC-Exceptions in Response
                    if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
                        exception_text = None
                        try:
                            except_re = re.compile(
                                "ServiceException>|ExceptionReport>")
                            exception_text = except_re.search(content).group(0)
                        except:
                            # No Exception in Response text
                            pass

                        if exception_text:
                            # Found OGC-Exception in Response text
                            raise Exception("Exception in response: %s" %
                                            exception_text)

                    del content

            elif resource_type == 'urn:geoss:waf':
                title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
        elif resource_type == 'FTP':
            ows = urlopen(url)
            title = urlparse(url).hostname
        success = True
        if resource_type.startswith(('OGC:', 'OSGeo')):
            if resource_type == 'OGC:STA':
                title = 'OGC STA'
            else:
                title = ows.identification.title
        if title is None:
            title = '%s %s %s' % (resource_type, gettext('for'), url)
    except Exception as err:
        msg = str(err)
        LOGGER.exception(msg)
        message = msg
        success = False

    end_time = datetime.datetime.utcnow()

    delta = end_time - start_time
    response_time = '%s.%s' % (delta.seconds, delta.microseconds)

    return [title, success, response_time, message, start_time]
示例#14
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    query = request.args.get('q')

    response = views.list_resources(resource_type, query)

    if request.url_rule.rule == '/json':
        json_dict = {'total': response['total'], 'resources': []}
        for r in response['resources']:
            try:
                ghc_url = '%s/resource/%s' % \
                          (CONFIG['GHC_SITE_URL'], r.identifier)
                last_run_report = '-'
                if r.last_run:
                    last_run_report = r.last_run.report

                json_dict['resources'].append({
                    'resource_type': r.resource_type,
                    'title': r.title,
                    'url': r.url,
                    'ghc_url': ghc_url,
                    'ghc_json': '%s/json' % ghc_url,
                    'ghc_csv': '%s/csv' % ghc_url,
                    'first_run': format_checked_datetime(r.first_run),
                    'last_run': format_checked_datetime(r.last_run),
                    'status': format_run_status(r.last_run),
                    'min_response_time': round(r.min_response_time, 2),
                    'average_response_time': round(r.average_response_time, 2),
                    'max_response_time': round(r.max_response_time, 2),
                    'reliability': round(r.reliability, 2),
                    'last_report': format_obj_value(last_run_report)
                })
            except Exception as e:
                LOGGER.warning(
                    'JSON error resource id=%d: %s' % (r.identifier, str(e)))

        return jsonify(json_dict)
    elif request.url_rule.rule == '/csv':
        output = StringIO()
        writer = csv.writer(output)
        header = [
            'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
            'first_run', 'last_run', 'status', 'min_response_time',
            'average_response_time', 'max_response_time', 'reliability'
        ]
        writer.writerow(header)
        for r in response['resources']:
            try:
                ghc_url = '%s%s' % (CONFIG['GHC_SITE_URL'],
                                    url_for('get_resource_by_id',
                                            identifier=r.identifier))

                writer.writerow([
                    r.resource_type,
                    r.title,
                    r.url,
                    ghc_url,
                    '%s/json' % ghc_url,
                    '%s/csv' % ghc_url,
                    format_checked_datetime(r.first_run),
                    format_checked_datetime(r.last_run),
                    format_run_status(r.last_run),
                    round(r.min_response_time, 2),
                    round(r.average_response_time, 2),
                    round(r.max_response_time, 2),
                    round(r.reliability, 2)
                ])
            except Exception as e:
                LOGGER.warning(
                    'CSV error resource id=%d: %s' % (r.identifier, str(e)))

        return output.getvalue(), 200, {'Content-type': 'text/csv'}
示例#15
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    query = request.args.get('q')

    response = views.list_resources(resource_type, query)

    if request.url_rule.rule == '/json':
        json_dict = {'total': response['total'], 'resources': []}
        for r in response['resources']:
            try:
                ghc_url = '%s/resource/%s' % \
                          (CONFIG['GHC_SITE_URL'], r.identifier)
                last_run_report = '-'
                if r.last_run:
                    last_run_report = r.last_run.report

                json_dict['resources'].append({
                    'resource_type': r.resource_type,
                    'title': r.title.encode('utf-8'),
                    'url': r.url,
                    'ghc_url': ghc_url,
                    'ghc_json': '%s/json' % ghc_url,
                    'ghc_csv': '%s/csv' % ghc_url,
                    'first_run': format_checked_datetime(r.first_run),
                    'last_run': format_checked_datetime(r.last_run),
                    'status': format_run_status(r.last_run),
                    'min_response_time': round(r.min_response_time, 2),
                    'average_response_time': round(r.average_response_time, 2),
                    'max_response_time': round(r.max_response_time, 2),
                    'reliability': round(r.reliability, 2),
                    'last_report': format_obj_value(last_run_report)
                })
            except Exception as e:
                LOGGER.warning(
                    'JSON error resource id=%d: %s' % (r.identifier, str(e)))

        return jsonify(json_dict)
    elif request.url_rule.rule == '/csv':
        output = StringIO()
        writer = csv.writer(output)
        header = [
            'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
            'first_run', 'last_run', 'status', 'min_response_time',
            'average_response_time', 'max_response_time', 'reliability'
        ]
        writer.writerow(header)
        for r in response['resources']:
            try:
                ghc_url = '%s%s' % (CONFIG['GHC_SITE_URL'],
                                    url_for('get_resource_by_id',
                                            identifier=r.identifier))

                writer.writerow([
                    r.resource_type,
                    r.title.encode('utf-8'),
                    r.url,
                    ghc_url,
                    '%s/json' % ghc_url,
                    '%s/csv' % ghc_url,
                    format_checked_datetime(r.first_run),
                    format_checked_datetime(r.last_run),
                    format_run_status(r.last_run),
                    round(r.min_response_time, 2),
                    round(r.average_response_time, 2),
                    round(r.max_response_time, 2),
                    round(r.reliability, 2)
                ])
            except Exception as e:
                LOGGER.warning(
                    'CSV error resource id=%d: %s' % (r.identifier, str(e)))

        return output.getvalue(), 200, {'Content-type': 'text/csv'}
示例#16
0
def export():
    """export resource list as JSON"""

    resource_type = None

    if request.args.get('resource_type') in RESOURCE_TYPES.keys():
        resource_type = request.args['resource_type']

    query = request.args.get('q')

    response = views.list_resources(resource_type, query)

    if request.url_rule.rule == '/json':
        json_dict = {'total': response['total'], 'resources': []}
        for r in response['resources']:
            ghc_url = '%s/resource/%s' % (GHC_SITE_URL, r.identifier)
            json_dict['resources'].append({
                'resource_type': r.resource_type,
                'title': r.title,
                'url': r.url,
                'ghc_url': ghc_url,
                'ghc_json': '%s/json' % ghc_url,
                'ghc_csv': '%s/csv' % ghc_url,
                'first_run': r.first_run.checked_datetime.strftime(
                    '%Y-%m-%dT%H:%M:%SZ'),
                'last_run': r.last_run.checked_datetime.strftime(
                    '%Y-%m-%dT%H:%M:%SZ'),
                'status': r.last_run.success,
                'min_response_time': round(r.min_response_time, 2),
                'average_response_time': round(r.average_response_time, 2),
                'max_response_time': round(r.max_response_time, 2),
                'reliability': round(r.reliability, 2)
            })
        return jsonify(json_dict)
    elif request.url_rule.rule == '/csv':
        output = StringIO()
        writer = csv.writer(output)
        header = [
            'resource_type', 'title', 'url', 'ghc_url', 'ghc_json', 'ghc_csv',
            'first_run', 'last_run', 'status', 'min_response_time',
            'average_response_time', 'max_response_time', 'reliability'
        ]
        writer.writerow(header)
        for r in response['resources']:
            ghc_url = '%s%s' % (GHC_SITE_URL,
                                url_for('get_resource_by_id',
                                        identifier=r.identifier))
            writer.writerow([
                r.resource_type,
                r.title,
                r.url,
                ghc_url,
                '%s/json' % ghc_url,
                '%s/csv' % ghc_url,
                r.first_run.checked_datetime.strftime(
                    '%Y-%m-%dT%H:%M:%SZ'),
                r.last_run.checked_datetime.strftime(
                    '%Y-%m-%dT%H:%M:%SZ'),
                r.last_run.success,
                round(r.average_response_time, 2),
                round(r.reliability, 2)
            ])
        return output.getvalue(), 200, {'Content-type': 'text/csv'}
示例#17
0
def run_test_resource(config, resource_type, url):
    """tests a CSW service and provides run metrics"""

    if resource_type not in RESOURCE_TYPES.keys():
        msg = gettext('Invalid resource type')
        msg2 = '%s: %s' % (msg, resource_type)
        LOGGER.error(msg2)
        raise RuntimeError(msg2)

    title = None
    start_time = datetime.datetime.utcnow()
    message = None

    try:
        if resource_type == 'OGC:WMS':
            ows = WebMapService(url)
        elif resource_type == 'OGC:WMTS':
            ows = WebMapTileService(url)
        elif resource_type == 'OSGeo:TMS':
            ows = TileMapService(url)
        elif resource_type == 'OGC:WFS':
            ows = WebFeatureService(url)
        elif resource_type == 'OGC:WCS':
            ows = WebCoverageService(url, version='1.0.0')
        elif resource_type == 'OGC:WPS':
            ows = WebProcessingService(url)
        elif resource_type == 'OGC:CSW':
            ows = CatalogueServiceWeb(url)
        elif resource_type == 'OGC:SOS':
            ows = SensorObservationService(url)
        elif resource_type in ['WWW:LINK', 'urn:geoss:waf']:
            ows = urlopen(url)
            if resource_type == 'WWW:LINK':
                content_type = ows.info().getheader('Content-Type')

                # Check content if the response is not an image
                if 'image/' not in content_type:
                    content = ows.read()
                    import re
                    try:
                        title_re = re.compile("<title>(.+?)</title>")
                        title = title_re.search(content).group(1)
                    except:
                        title = url

                    # Optional check for any OGC-Exceptions in Response
                    if config and config['GHC_WWW_LINK_EXCEPTION_CHECK']:
                        exception_text = None
                        try:
                            except_re = re.compile(
                                "ServiceException>|ExceptionReport>")
                            exception_text = except_re.search(content).group(0)
                        except:
                            # No Exception in Response text
                            pass

                        if exception_text:
                            # Found OGC-Exception in Response text
                            raise Exception(
                                "Exception in response: %s" % exception_text)

                    del content

            elif resource_type == 'urn:geoss:waf':
                title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname)
        elif resource_type == 'FTP':
            ows = urlopen(url)
            title = urlparse(url).hostname
        success = True
        if resource_type.startswith(('OGC:', 'OSGeo')):
            title = ows.identification.title
        if title is None:
            title = '%s %s %s' % (resource_type, gettext('for'), url)
    except Exception as err:
        msg = str(err)
        LOGGER.exception(msg)
        message = msg
        success = False

    end_time = datetime.datetime.utcnow()

    delta = end_time - start_time
    response_time = '%s.%s' % (delta.seconds, delta.microseconds)

    return [title, success, response_time, message, start_time]