예제 #1
0
def stats_map(request, origin=None, source=None, date=None):
    stats = None
    if settings.STATS_SAVE_MEMORY:
        cache, stats = get_from_cache('default','stats_tilerequests')
    if settings.STATS_SAVE_FILE and not stats:
        stats = get_from_file(settings.STATS_REQUEST_FILE, filetype='json')
    dates = []
    if stats:
        if 'by_date' in stats:
            dates = stats['by_date'].keys()
    #print stats['by_date_location'].keys()
    context_dict = {
        'date': date,
        'origins': getTileOrigins(),
        'sources': getTileSources(),
        'dates': dates
    }

    try:
        context_dict['origin'] = TileOrigin.objects.get(name=origin)
    except:
        context_dict['origin'] = None


    try:
        context_dict['source'] = TileSource.objects.get(name=source)
    except:
        context_dict['source'] = None


    return render_to_response(
        "cache/stats_map_3.html",
        RequestContext(request, context_dict))
예제 #2
0
def sources_list(request):
    #cache, stats = get_from_cache('default','stats_tilerequests')
    context_dict = {
        'sources': getTileSources()
    }
    return render_to_response(
        "cache/sources_list.html",
        RequestContext(request, context_dict))
예제 #3
0
def sources_json(request):
    now = datetime.datetime.now()
    dt = now
    #######
    stats = None
    if settings.STATS_SAVE_MEMORY:
        cache, stats = get_from_cache(
            settings.CACHES['default']['LOCATION'],
            settings.CACHES['default'],
            'default',
            'stats_tilerequests',
            GEVENT_MONKEY_PATCH=settings.TILEJET_GEVENT_MONKEY_PATCH)
    if settings.STATS_SAVE_FILE and not stats:
        stats = get_from_file(settings.STATS_REQUEST_FILE, filetype='json')
    sources = []
    #for source in TileSource.objects.all().order_by('name'):
    for source in getTileSources():
        link_geojson = settings.SITEURL+'cache/stats/export/geojson/15/source/'+source['name']+'.geojson'
        link_proxy_internal = settings.SITEURL+'proxy/?url='+(source['url']).replace("{ext}","png")
        link_proxy_external = ""
        if source['type'] in [TYPE_TMS, TYPE_TMS_FLIPPED]:
            link_proxy_external = settings.SITEURL+'cache/proxy/tms/origin/'+source['origin']+'/source/'+source['name']+'/{z}/{x}/{y}.png' 
        elif source['type'] == TYPE_BING:
            link_proxy_external = settings.SITEURL+'cache/proxy/bing/origin/'+source['origin']+'/source/'+source['name']+'{u}.png'
        if stats:
            sources.append({
                'name': source['name'],
                'type': source['type_title'],
                'origin': source['origin'],
                'url': source['url'],
                'requests_all': getValue(stats['by_source'], source['name'],0),
                'requests_year': getValue(getValue(stats['by_year_source'],dt.strftime('%Y')),source['name'], 0),
                'requests_month': getValue(getValue(stats['by_month_source'],dt.strftime('%Y-%m')),source['name'], 0),
                'requests_today': getValue(getValue(stats['by_date_source'],dt.strftime('%Y-%m-%d')),source['name'], 0),
                'link_proxy': link_proxy_internal,
                'link_id': 'http://www.openstreetmap.org/edit#?background=custom:'+link_proxy_external,
                'link_geojson': link_geojson,
                'link_geojsonio': 'http://geojson.io/#data=data:text/x-url,'+link_geojson
            })
        else:
            sources.append({
                'name': source['name'],
                'type': source['type_title'],
                'origin': source['origin'],
                'url': source['url'],
                'requests_all': -1,
                'requests_year': -1,
                'requests_month': -1,
                'requests_today': -1,
                'link_proxy': link_proxy_internal,
                'link_id': 'http://www.openstreetmap.org/edit#?background=custom:'+link_proxy_external,
                'link_geojson': link_geojson,
                'link_geojsonio': 'http://geojson.io/#data=data:text/x-url,'+link_geojson
            })

    return HttpResponse(json.dumps(sources),
                        content_type="application/json"
                        )
예제 #4
0
def proxy_tms(request, origin=None, slug=None, z=None, x=None, y=None, u=None, ext=None):

    #starttime = time.clock()
    # Check Existing Tile Sourcesi
    match_tilesource = None
    tilesources = getTileSources(proxy=True)
    for tilesource in tilesources:
        if tilesource['name'] == slug:
            match_tilesource = tilesource
            break

    if match_tilesource:
        if match_tilesource['origin'] != origin:
            print "Origin is not correct.  Tilesource is unique, but origin need to match too."
            print tilesource['origin']
            return None
        else:
            tile = _requestTile(
                request,
                tileservice=None,
                tilesource=match_tilesource,
                tileorigin=match_tilesource['origin'],
                z=z,x=x,y=y,u=u,ext=ext)
            #print "Time Elapsed: "+str(time.clock()-starttime)
            return tile


    # Check Existing Tile Origins to see if we need to create a new tile source
    match_tileorigin = None
    if origin:
        tileorigins = getTileOrigins(proxy=True)
        for tileorigin in tileorigins:
            if tileorigin.name == origin:
                match_tileorigin = tileorigin
                break

    if match_tileorigin:
        to = match_tileorigin
        if to.multiple:
            ts_url = to.url.replace('{slug}', slug)
            if TileSource.objects.filter(url=ts_url).count() > 0:
                print "Error: This souldn't happen.  You should have matched the tilesource earlier so you don't duplicate"
                return None
            exts = string_to_list(to.extensions)
            ts_pattern = url_to_pattern(ts_url, extensions=exts)
            ts = TileSource(auto=True,url=ts_url,pattern=ts_pattern,name=slug,type=to.type,extensions=exts,origin=to)
            ts.save()
            reloadTileSources(proxy=False)
            reloadTileSources(proxy=True)
            return _requestTile(request,tileservice=None,tilesource=tilesource,z=z,x=x,y=y,u=u,ext=ext)
        else:
            ts = TileSource(auto=True,url=to.url,pattern=to.pattern,name=to.name,type=to.type,extensions=to.extensions)
            ts.save()
            reloadTileSources(proxy=False)
            reloadTileSources(proxy=True)
            return _requestTile(request,tileservice=None,tilesource=tilesource,z=z,x=x,y=y,u=u,ext=ext)
    else:
        return None
예제 #5
0
def tile_tms(request, slug=None, z=None, x=None, y=None, u=None, ext=None):
    match_tileservice = None
    tileservices = getTileServices()
    for tileservice in tileservices:
        if tileservice['name'] == slug:
            match_tileservice = tileservice
            break

    if match_tileservice:
        match_tilesource = None
        tilesources = getTileSources()
        for tilesource in tilesources:
            if tilesource['name'] == tileservice['source']:
                match_tilesource = tilesource
                break

        if match_tilesource:
            return _requestTile(request,tileservice=match_tileservice,tilesource=match_tilesource,z=z,x=x,y=y,u=u,ext=ext)
        else:
            return HttpResponse(RequestContext(request, {}), status=404)
    else:
        return HttpResponse(RequestContext(request, {}), status=404)
예제 #6
0
def taskRequestTile(ts, iz, ix, iy, ext, verbose=True):

    now = datetime.datetime.now()
    # Load Logging Info
    #log_root = settings.LOG_REQUEST_ROOT
    log_format = settings.LOG_REQUEST_FORMAT
    #if log_root and log_format:
    #    if not os.path.exists(log_root):
    #        os.makedirs(log_root)

    #if settings.LOG_ERRORS_ROOT
    #    if not os.path.exists(log_root):
    #        os.makedirs(log_root)

    indirect_file = settings.LOG_INDIRECT_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_indirect.tsv"
    # Find TileSource
    tilesource = None
    tilesources = getTileSources(proxy=True)
    for candidate in tilesources:
        if candidate['id'] == ts:
            tilesource = candidate
            break

    if not tilesource:
        error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
        with open(error_file,'a') as f:
            line = "Error: Could not find tilesource for primary key "+str(ts)+"."
            f.write(line+"\n")
        return

    #Y is always in regualar TMS before being added to task queue
    iyf = flip_y(ix,iy,iz)
    #iy, iyf = getYValues(None,tilesource,ix,iy,iz)

    tile_bbox = tms_to_bbox(ix,iy,iz)

    #Check if requested tile is within source's extents
    returnBlankTile = False
    returnErrorTile = False
    intersects = True
    if tilesource['extents']:
        intersects = bbox_intersects_source(tilesource,ix,iyf,iz)
        if not intersects:
           returnBlankTile = True

    validZoom = 0
    #Check if inside source zoom levels
    if tilesource['minZoom'] or tilesource['maxZoom']:
        if (tilesource['minZoom'] and iz < tilesource['minZoom']):
            validZoom = -1
        elif (tilesource['maxZoom'] and iz > tilesource['maxZoom']):
           validZoom = 1

        if validZoom != 0:
            #returnBlank = True
            returnErrorTile = True

    if returnBlankTile or returnErrorTile:
        return

    tile = None
    if iz >= settings.TILEJET['cache']['memory']['minZoom'] and iz <= settings.TILEJET['cache']['memory']['maxZoom']:
        #key = "{layer},{z},{x},{y},{ext}".format(layer=tilesource.name,x=ix,y=iy,z=iz,ext=ext)
        key = ",".join([tilesource['name'],str(iz),str(ix),str(iy),ext])
        tilecache, tile = getTileFromCache(
            settings.CACHES['tiles']['LOCATION'],
            settings.CACHES['tiles'],
            'tiles',
            key,
            True,
            GEVENT_MONKEY_PATCH=True)


        if not tilecache:
            error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
            with open(error_file,'a') as f:
                line = "Error: Could not connect to cache (tiles)."
                f.write(line+"\n")
            return

        if tile:
            if verbose:
                print "task / cache hit for "+key
        else:
            if verbose:
                print "task / cache miss for "+key

            with open(indirect_file,'a') as f:
                line = log_format.format(
                    status='indirect',
                    tileorigin=tilesource['origin'],
                    tilesource=tilesource['name'],
                    z=iz,x=ix,y=iy,
                    ip='-',
                    datetime=now.isoformat())
                f.write(line+"\n")

            from urllib2 import HTTPError
            try:
                if tilesource['type'] == TYPE_TMS:
                    tile = requestTileFromSource(tilesource,ix,iy,iz,ext,True)
                elif tilesource['type'] == TYPE_TMS_FLIPPED:
                    tile = requestTileFromSource(tilesource,ix,iyf,iz,ext,True)
            except HTTPError, err:
                error_file = settings.LOG_ERRORS_ROOT+os.sep+"requests_tiles_"+now.strftime('%Y-%m-%d')+"_errors.txt"
                with open(error_file,'a') as f:
                    line = "Error: HTTPError.  Could not get tile ("+key+") from source."
                    f.write(line+"\n")
                return
            except:
예제 #7
0
def info(request):
    #stats_tr = stats_tilerequest()
    #cache, stats_tr = get_from_cache(
    #    settings.CACHES['default']['LOCATION'],
    #    settings.CACHES['default'],
    #    'default',
    #    'stats_tilerequests',
    #    GEVENT_MONKEY_PATCH=settings.TILEJET_GEVENT_MONKEY_PATCH)

    caches = []
    c = settings.TILEJET['cache']['memory']

    stats_c = stats_cache()
    if stats_c:
        size = int(stats_c['bytes'])
        maxsize = int(stats_c['limit_maxbytes'])
        size_percentage = format(((100.0 * size) / maxsize),'.4f')+"%" 
        items = int(stats_c['curr_items'])

        caches.append({
            'name': 'memory',
            'enabled': c['enabled'],
            'description': c['description'],
            'type': c['type'],
            'size': formatMemorySize(size, original='B'),
            'maxsize': formatMemorySize(maxsize, original='B'),
            'size_percentage': size_percentage,
            'items': items,
            'minzoom': c['minZoom'],
            'maxzoom': c['maxZoom'],
            'expiration': c['expiration'],
            'link_memcached': '/cache/stats/export/cache.json'
        })

    heuristics = []
    h = settings.TILEJET['heuristic']['down']
    heuristics.append({
        'name': 'down',
        'enabled': h['enabled'],
        'description': h['description']
    })
    h = settings.TILEJET['heuristic']['up']
    heuristics.append({
        'name': 'up',
        'enabled': h['enabled'],
        'description': h['description']
    })
    h = settings.TILEJET['heuristic']['nearby']
    heuristics.append({
        'name': 'nearby',
        'enabled': h['enabled'],
        'description': h['description']
    })

    # Build Queues List
    queues =  []
    try:
        import celery
        for key, raw_queues in celery.current_app.control.inspect().active_queues().items():
            for q in raw_queues:
                queues.append({
                    'name': getValue(q, u'name', fallback=''),
                    'routing_key': getValue(q, u'routing_key', fallback=''),
                    'durable': getValue(q, u'durable', fallback=False),
                    'ttl': getValue(q[u'queue_arguments'], u'x-message-ttl', fallback=-1)
                })

        #import pyrabbit.api
        #pyrabbit_client = pyrabbit.api.Client(settings.BROKER_DOMAIN+':'+settings.BROKER_PORT, settings.BROKER_USER, settings.BROKER_PASSWORD)
        for q in queues:
            q['messages'] = 0
    except:
        print "Could not generate queues.  Is celery or RabbitMQ offline?"

    # Build Schedules Tasks
    scheduled = []
    try:
        import celery
        s = beat.Scheduler(app=celery.current_app)
        scheduled = s.schedule.keys()
    except:
        print "Could not build scheduled tasks.  Is celery beat running?"

    #topics = []
    #try:
    #    from kafka import KafkaClient
    #    kafka = KafkaClient(settings.TILEJET_GEOWATCH_HOST)
    #    for topic in kafka.topics:
    #        topic2 = {
    #            'name': topic,
    #            'partitions': len(kafka.topic_partitions.get(topic, []))
    #        }
    #        topics.append(topic2)
    #except:
    #    print "Could not generate topics.  Is Kafka offline?"
    client = provision_geowatch_client()
    topics = client.list_topics()

    context_dict = {
        'origins': getTileOrigins(),
        'sources': getTileSources(),
        'caches': caches,
        'heuristics': heuristics,
        'queues': queues,
        'topics': topics,
        'scheduled': scheduled,
        'stats': settings.TILEJET_LIST_STATS,
        'hosts': settings.PROXY_ALLOWED_HOSTS
    }
    return render_to_response(
        "cache/info.html",
        RequestContext(request, context_dict))
예제 #8
0
def proxy(request):
    PROXY_ALLOWED_HOSTS = getattr(settings, 'PROXY_ALLOWED_HOSTS', ())

    host = None

    if 'url' not in request.GET:
        return HttpResponse("The proxy service requires a URL-encoded URL as a parameter.",
                            status=400,
                            content_type="text/plain"
                            )

    raw_url = request.GET['url']
    url = urlsplit(raw_url)
    locator = url.path
    if url.query != "":
        locator += '?' + url.query
    if url.fragment != "":
        locator += '#' + url.fragment

    if not settings.DEBUG:
        if not validate_host(url.hostname, PROXY_ALLOWED_HOSTS):
            return HttpResponse("DEBUG is set to False but the host of the path provided to the proxy service"
                                " is not in the PROXY_ALLOWED_HOSTS setting.",
                                status=403,
                                content_type="text/plain"
                                )
    headers = {}

    if settings.SESSION_COOKIE_NAME in request.COOKIES and is_safe_url(url=raw_url, host=host):
        headers["Cookie"] = request.META["HTTP_COOKIE"]

    if request.method in ("POST", "PUT") and "CONTENT_TYPE" in request.META:
        headers["Content-Type"] = request.META["CONTENT_TYPE"]

    print "Raw URL: "+ raw_url
    match_regex = None
    match_tilesource = None

    # Try to match against existing tile sources
    #tilesources = TileSource.objects.exclude(pattern__isnull=True).exclude(pattern__exact='')
    tilesources = getTileSources(proxy=True)
    for tilesource in tilesources:
        match = match_pattern_url(tilesource.pattern, raw_url)
        if match:
            match_regex = match
            match_tilesource = tilesource
            break

    if match_tilesource and match_regex:
        return proxy_tilesource(request, match_tilesource, match_regex)
    #else:
    #    return HttpResponse('No matching tilesource found.',RequestContext(request, {}), status=404)

    # Try to match against existing origins that can automatically create tile sources (auto=true)
    match_tileorigin = None
    #tileorigins = TileOrigin.objects.exclude(pattern__isnull=True).exclude(pattern__exact='').filter(auto=True)
    tileorigins = getTileOrigins(proxy=True)
    for tileorigin in tileorigins:
        match = match_pattern_url(tileorigin.pattern, raw_url)
        if match:
            match_regex = match
            match_tileorigin = tileorigin
            break

    if match_tileorigin and match_regex:
        to = match_tileorigin
        if to.multiple:
            slug = getRegexValue(match_regex, 'slug')
            ts_url = to.url.replace('{slug}', slug)
            #print "ts_url: "+ts_url
            if TileSource.objects.filter(url=ts_url).count() > 0:
                print "Error: This souldn't happen.  You should have matched the tilesource earlier so you don't duplicate"
                return None
            exts = string_to_list(to.extensions)
            ts_pattern = url_to_pattern(ts_url, extensions=exts)
            ts = TileSource(auto=True,url=ts_url,pattern=ts_pattern,name=slug,type=to.type,extensions=exts,origin=to)
            ts.save()
            reloadTileSources(proxy=False)
            reloadTileSources(proxy=True)
            return proxy_tilesource(request, ts, match_regex)
        else:
            ts = TileSource(auto=True,url=to.url,pattern=to.pattern,name=to.name,type=to.type,extensions=to.extensions)
            ts.save()
            reloadTileSources(proxy=False)
            reloadTileSources(proxy=True)
            return proxy_tilesource(request, ts, match_regex)
    else:
        return HttpResponse('No matching tile origin or tile source found.',RequestContext(request, {}), status=404)