def jpg(self, cam=1, latesttime=None):
"""Return Image"""
web.modified(self.lastupdate('jpg'))
sql = []
sql.append("""SELECT * FROM icecast.station_image
WHERE station_id = $stationid
AND camera_no = $camerano
AND image_datetime = (SELECT MAX(image_datetime)
FROM icecast.station_image
WHERE station_id = $stationid
AND camera_no = $camerano""")
variables = {
'stationid': self.row['station_id'],
'camerano': cam,
}
if latesttime:
sql.append("AND image_datetime <= $latesttime")
variables['latesttime'] = latesttime
sql.append(")")
try:
return db.query(' '.join(sql), variables)[0].image_content
except IndexError:
web.header("Content-Type", "text/html")
raise web.HTTPError("404 not found", {},
"No camera image available.")
def serve_file(path):
'''Serve static file from the given path. Sets headers: Modified, Content-Type, Encoding'''
import mimetypes
from datetime import datetime
if not os.path.exists(path):
raise web.notfound()
mdate = datetime.fromtimestamp(os.stat(path).st_mtime)
web.modified(date=mdate, etag='')
fh = open(path, 'rb')
s = fh.read()
fh.close()
mime, enc = mimetypes.guess_type(path, strict=False)
if mime:
web.header('Content-Type', mime)
if enc:
web.header('Content-Encoding', enc)
return s
def render_cached_feed(events, format_type='atom'):
last_modif_time = events[0].time_taken
web.debug(last_modif_time)
#http conditional get
if not config.http_conditional_get or web.modified(last_modif_time):
web.lastmodified(last_modif_time)
web.header("Content-Type", "application/atom+xml")
web.debug(len(events))
print render.events_atom_feed(config, last_modif_time, events)
def GET(self, category, key, value, size):
i = web.input(default="true")
key = key.lower()
def notfound():
if config.default_image and i.default.lower(
) != "false" and not i.default.startswith('http://'):
return read_file(config.default_image)
elif i.default.startswith('http://'):
raise web.seeother(i.default)
else:
raise web.notfound("")
def redirect(id):
size_part = size and ("-" + size) or ""
url = "/%s/id/%s%s.jpg" % (category, id, size_part)
query = web.ctx.env.get('QUERY_STRING')
if query:
url += '?' + query
raise web.found(url)
if key == 'isbn':
value = value.replace("-", "").strip() # strip hyphens from ISBN
# Disabling ratelimit as iptables is taking care of botnets.
#value = self.ratelimit_query(category, key, value)
value = self.query(category, key, value)
elif key != 'id':
value = self.query(category, key, value)
d = value and db.details(value)
if not d:
return notfound()
# set cache-for-ever headers only when requested with ID
if key == 'id':
etag = "%s-%s" % (d.id, size.lower())
if not web.modified(trim_microsecond(d.created), etag=etag):
raise web.notmodified()
web.header('Cache-Control', 'public')
web.expires(
100 * 365 * 24 *
3600) # this image is not going to expire in next 100 years.
else:
web.header('Cache-Control', 'public')
web.expires(
10 * 60
) # Allow the client to cache the image for 10 mins to avoid further requests
web.header('Content-Type', 'image/jpeg')
try:
return read_image(d, size)
except IOError:
raise web.notfound()
def GET(self):
i = web.input(v=None)
v = doc.md5()
if v != i.v:
raise web.seeother(web.changequery(v=v))
if web.modified(etag=v):
oneyear = 365 * 24 * 3600
web.header("Content-Type", content_type)
web.header("Cache-Control", "Public, max-age=%d" % oneyear)
web.lastmodified(doc.last_modified)
web.expires(oneyear)
return delegate.RawText(doc.get_text())
def GET(self):
i = web.input(v=None)
v = doc.md5()
if v != i.v:
raise web.seeother(web.changequery(v=v))
if web.modified(etag=v):
oneyear = 365 * 24 * 3600
web.header("Content-Type", content_type)
web.header("Cache-Control", "Public, max-age=%d" % oneyear)
web.lastmodified(doc.last_modified)
web.expires(oneyear)
return delegate.RawText(doc.get_text())
def GET(self, category, key, value, size):
i = web.input(default="true")
key = key.lower()
def notfound():
if config.default_image and i.default.lower() != "false" and not i.default.startswith('http://'):
return read_file(config.default_image)
elif i.default.startswith('http://'):
raise web.seeother(i.default)
else:
raise web.notfound("")
def redirect(id):
size_part = size and ("-" + size) or ""
url = "/%s/id/%s%s.jpg" % (category, id, size_part)
query = web.ctx.env.get('QUERY_STRING')
if query:
url += '?' + query
raise web.found(url)
if key == 'isbn':
value = value.replace("-", "").strip() # strip hyphens from ISBN
# Disabling ratelimit as iptables is taking care of botnets.
#value = self.ratelimit_query(category, key, value)
value = self.query(category, key, value)
elif key != 'id':
value = self.query(category, key, value)
d = value and self.get_details(value, size.lower())
if not d:
return notfound()
# set cache-for-ever headers only when requested with ID
if key == 'id':
etag = "%s-%s" % (d.id, size.lower())
if not web.modified(trim_microsecond(d.created), etag=etag):
raise web.notmodified()
web.header('Cache-Control', 'public')
web.expires(100 * 365 * 24 * 3600) # this image is not going to expire in next 100 years.
else:
web.header('Cache-Control', 'public')
web.expires(10*60) # Allow the client to cache the image for 10 mins to avoid further requests
web.header('Content-Type', 'image/jpeg')
try:
return read_image(d, size)
except IOError:
raise web.notfound()
def GET(self, path, filename):
try:
etag = str(
os.path.getmtime(dirname + 'files/' + path + '/' + filename))
last_modified = datetime.datetime.fromtimestamp(
os.path.getmtime(dirname + 'files/' + path + '/' + filename))
if web.modified(last_modified, etag):
f = open(dirname + 'files/' + path + '/' + filename, 'rb')
web.header('Content-type', mimetypes.guess_type(filename)[0])
return f.read()
else:
return web.notmodified()
except (FileNotFoundError, OSError):
return web.notfound()
def GET(self, category, key, value, size):
i = web.input(default="true")
key = key.lower()
def notfound():
if config.default_image and i.default.lower() != "false" and not i.default.startswith('http://'):
return read_file(config.default_image)
elif i.default.startswith('http://'):
raise web.seeother(i.default)
else:
raise web.notfound("")
def redirect(id):
size_part = size and ("-" + size) or ""
url = "/%s/id/%s%s.jpg" % (category, id, size_part)
query = web.ctx.env.get('QUERY_STRING')
if query:
url += '?' + query
raise web.found(url)
if key != 'id':
value = _query(category, key, value)
if value is None:
return notfound()
else:
return redirect(value)
else:
d = db.details(value)
if not d:
return notfound()
etag = "%s-%s" % (d.id, size.lower())
if not web.modified(trim_microsecond(d.created), etag=etag):
raise web.notmodified()
web.header('Cache-Control', 'public')
web.expires(100 * 365 * 24 * 3600) # this image is not going to expire in next 100 years.
web.header('Content-Type', 'image/jpeg')
try:
return read_image(d, size)
except IOError:
raise web.notfound()
def GET(self, category, key, value, size):
i = web.input(default="true")
key = key.lower()
d = _query(category, key, value)
if d:
if key == 'id':
etag = "%s-%s" % (d.id, size.lower())
if not web.modified(trim_microsecond(d.created), etag=etag):
raise web.notmodified()
web.header('Cache-Control', 'public')
web.expires(100 * 365 * 24 * 3600) # this image is not going to expire in next 100 years.
web.header('Content-Type', 'image/jpeg')
return read_image(d, size)
elif config.default_image and i.default.lower() != "false" and not i.default.startswith('http://'):
return read_file(config.default_image)
elif i.default.startswith('http://'):
raise web.seeother(i.default)
else:
raise web.notfound("")
def GET(self, category, key, value, size):
i = web.input(default="true")
key = key.lower()
def is_valid_url(url):
return url.startswith("http://") or url.startswith("https://")
def notfound():
if key in ["id", "olid"] and config.get("upstream_base_url"):
# this is only used in development
base = web.rstrips(config.upstream_base_url, "/")
raise web.redirect(base + web.ctx.fullpath)
elif config.default_image and i.default.lower() != "false" and not is_valid_url(i.default):
return read_file(config.default_image)
elif is_valid_url(i.default):
raise web.seeother(i.default)
else:
raise web.notfound("")
def redirect(id):
size_part = size and ("-" + size) or ""
url = "/%s/id/%s%s.jpg" % (category, id, size_part)
query = web.ctx.env.get('QUERY_STRING')
if query:
url += '?' + query
raise web.found(url)
if key == 'isbn':
value = value.replace("-", "").strip() # strip hyphens from ISBN
# Disabling ratelimit as iptables is taking care of botnets.
#value = self.ratelimit_query(category, key, value)
value = self.query(category, key, value)
# Redirect isbn requests to archive.org.
# This will heavily reduce the load on coverstore server.
# The max_coveritem_index config parameter specifies the latest
# olcovers items uploaded to archive.org.
if value and self.is_cover_in_cluster(value):
url = zipview_url_from_id(int(value), size)
raise web.found(url)
elif key == 'ia':
url = self.get_ia_cover_url(value, size)
if url:
raise web.found(url)
else:
value = None # notfound or redirect to default. handled later.
elif key != 'id':
value = self.query(category, key, value)
if value and safeint(value) in config.blocked_covers:
raise web.notfound()
# redirect to archive.org cluster for large size and original images whenever possible
if value and (size == "L" or size == "") and self.is_cover_in_cluster(value):
url = zipview_url_from_id(int(value), size)
raise web.found(url)
d = value and self.get_details(value, size.lower())
if not d:
return notfound()
# set cache-for-ever headers only when requested with ID
if key == 'id':
etag = "%s-%s" % (d.id, size.lower())
if not web.modified(trim_microsecond(d.created), etag=etag):
raise web.notmodified()
web.header('Cache-Control', 'public')
web.expires(100 * 365 * 24 * 3600) # this image is not going to expire in next 100 years.
else:
web.header('Cache-Control', 'public')
web.expires(10*60) # Allow the client to cache the image for 10 mins to avoid further requests
web.header('Content-Type', 'image/jpeg')
try:
return read_image(d, size)
except IOError:
raise web.notfound()
class Service(object):
"""Cache service
This service implements the content delivering caching mechanism.
All requests handled by this service produces cache manipulation
on the Google Datastore (with a Memcache top layer).
- origin: Set the origin url (type: String; mandatory)
- forceTTL: Does not honor CacheControl value, replacing cache TTL by this value (type: Integer)
- maxTTL: When the CacheControl value is honored (forceTTL not set), the cache TTL
value cannot be greater than this value (otherwise, it is overriden). (type: Integer)
- ignoreQueryString: Tell if the trailing HTTP query string is not taken into account
to generate the cache object key in Datastore. In other terms, if this value is set
to True, /url/path/obj.js?v=42 and /url/path/obj.js referer to the same object. (type: Boolean; default: False)
- forwardPost: If it is True, POST requests will be forwarded, instead of being redirected (type: Boolean; default: True)
- allowFlushFrom: Specify client IP which are allowed to make DELETE requests to flush
cache object explicitly. (type: List)
- disableIfModifiedSince: Disable IMS request during object refresh. (type: Boolean; default: False)
- prefetch: (EXPERIMENTAL) Prefetch content from HTML or other pages. (type: Boolean; default: False)
- headerBlacklist: Set list of origin headers to remove. (type: List)
"""
origin = None
forceTTL = None
maxTTL = None
ignoreQueryString = False
forwardPost = True
# Set your client IP address to authorize cache entry deletion
allowFlushFrom = ['127.0.0.1']
prefetch = False
disableIfModifiedSince = False
# undocumented: remove querystring before forwarding
# the request to the origin
stripForwardedQueryString = False
# These headers won't be forwarded
headerBlacklist = []
_headerBlacklist = [
'date',
'last-modified',
'via',
'expires',
'etag'
]
def __init__(self):
self.name = self.__class__.__name__
self.cache = type(self.name, (Cache,), {})
# Register the dynamic object globally
# if not, pickle cannot find it for serialization
globals()[self.name] = self.cache
def GET(self, request, _beforeWriteCache=None):
if self.ignoreQueryString is False:
request += web.ctx.query
try:
cache = self.readCache(request)
if cache is None:
cache = self.writeCache(request, _beforeWriteCache=_beforeWriteCache)
except runtime.DeadlineExceededError:
raise web.SeeOther(self.origin + request, absolute=True)
except CacheExpired, cache:
cache = self.writeCache(request, cache=cache(), _beforeWriteCache=_beforeWriteCache)
if not web.modified(cache.lastModified):
raise web.HTTPError(status='304 Not Modified')
web.header('Expires', web.httpdate(cache.expires))
for h in cache.headers:
print h
return cache.data
def data(self,
earliesttime=None,
latesttime=None,
lastget=None,
raise_exception=True):
"""
Return data for an individual station after earliesttime &
before latesttime.
"""
## TODO: Better latesttime & earliesttime checking. Limits?
log.debug("starting station request...")
if raise_exception:
web.modified(self.lastupdate('obs'))
sql = []
sql.append("""select vc.mes_datetime as data_time,
s.sensor_id as sensor_id,
v.value_num/s.scaling as data_value,
s.symbol as data_symbol,
s.data_number as data_number,
st.vmdb_id as vmdb_id,
x.xmltagname as xmltagname,
x.xsitype as xsitype,
di.datex_id as datex_id,
st.station_name as station_name
from icecast.value v, icecast.value_control vc,
icecast.sensor s, icecast.application_symbol app_s,
icecast.station st, exportws.xmltags x,
exportws.sensorindex di
where v.vc_id = vc.vc_id
and v.value_status>= 0
and v.vc_id in""")
variables = {
'stationid': self.row['station_id'],
}
if earliesttime or latesttime:
sql.append("""( select vc_id
from icecast.value_control
where station_id = $stationid""")
if earliesttime:
log.debug("extracting from %s" % earliesttime)
sql.append("and mes_datetime >= $earliesttime")
variables['earliesttime'] = earliesttime
if latesttime:
log.debug("extracting since %s" % latesttime)
sql.append("and mes_datetime <= $latesttime")
variables['latesttime'] = latesttime
sql.append(")")
elif lastget:
log.debug("extracting data since %s" % lastget)
# We can't use > over >= in the sql as entry_datetime is stored
# to sub-second accuracy in postgres. (mes_datetime isn't!!)
# We'll just add a second to lastget instead.
sql.append("""( select vc_id
from icecast.value_control
where station_id = $stationid
and entry_datetime >= $nextget
)""")
variables['nextget'] = lastget + timedelta(seconds=1)
else:
sql.append("""( select max(vc_id)
from icecast.value_control
where station_id = $stationid
)""")
sql.append("""and v.sensor_id = s.sensor_id
and s.symbol = app_s.symbol
and s.symbol = x.data_symbol
and s.symbol = di.data_symbol
and s.data_number = di.data_number
and app_s.application_key = 'ExportObservationXML'
and vc.station_id = st.station_id
order by vc.mes_datetime; """)
result = db.query(' '.join(sql), vars=variables)
if not result and raise_exception:
# FIXME: Should this be here?
web.header('Content-Type', 'text/html; charset=utf-8')
raise web.HTTPError("404 not found", {}, "No data available.")
return Data(result, {
self.row['vmdb_id']: self,
})
def data(self, earliesttime=None, latesttime=None, lastget=None,
raise_exception=True):
"""
Return data for an individual station after earliesttime &
before latesttime.
"""
## TODO: Better latesttime & earliesttime checking. Limits?
log.debug("starting station request...")
if raise_exception:
web.modified(self.lastupdate('obs'))
sql = []
sql.append("""SELECT v.obs_creationtime as data_time,
v.db_insertiontime as insertion_time,
si.symbol as sensor_id,
st.station_name,
st.stn_id,
st.vmdb_id,
st.owning_region_id,
st.lat,
st.lon,
di.data_symbol,
di.data_number as data_number,
nvalue as data_value,
v.qc_check_total AS qctotal,
qc_check_failed as qcfailed,
x.xmltagname as xmltagname,
x.xsitype as xsitype,
di.datex_id as datex_id
FROM qm.data_value v,
qm.sensor_identity si,
qm.station_identity st,
exportws.xmltags x,
exportws.sensorindex di
WHERE True""")
variables = {'stationid': self.row['stn_id'], }
if earliesttime or latesttime:
if earliesttime:
log.debug("extracting from %s" % earliesttime)
sql.append("and obs_creationtime >= $earliesttime")
variables['earliesttime'] = earliesttime
if latesttime:
log.debug("extracting since %s" % latesttime)
sql.append("and obs_creationtime <= $latesttime")
variables['latesttime'] = latesttime
elif lastget:
log.debug("extracting data since %s" % lastget)
# We can't use > over >= in the sql as entry_datetime is stored
# to sub-second accuracy in postgres. (mes_datetime isn't!!)
# We'll just add a second to lastget instead.
sql.append("""and db_insertiontime >= $nextget""")
variables['nextget'] = lastget + timedelta(seconds=1)
else:
sql.append("""and v.obs_creationtime = st.last_updated""")
sql.append("""and v.stn_id = $stationid
and v.stn_id = st.stn_id
and v.sensor_id = si.sensor_id
and st.vmdb_id IS NOT NULL
and si.symbol = di.dqm_symbol
and di.data_symbol = x.data_symbol
order by v.obs_creationtime; """)
result = db.query(' '.join(sql), vars=variables)
if not result and raise_exception:
# FIXME: Should this be here?
web.header('Content-Type', 'text/html; charset=utf-8')
raise web.HTTPError("404 not found", {}, "No data available.")
return Data(result, {self.row['vmdb_id']: self,})
