def riak_map_reduce(self):
mapreduce = RiakMapReduce(self.client)
# Hack: We replace the two methods that hit the network with
# deferToThread wrappers to prevent accidental sync calls in
# other code.
run = mapreduce.run
stream = mapreduce.stream
mapreduce.run = lambda *a, **kw: deferToThread(run, *a, **kw)
mapreduce.stream = lambda *a, **kw: deferToThread(stream, *a, **kw)
return mapreduce
def riak_map_reduce(self):
mapreduce = RiakMapReduce(self.client)
# Hack: We replace the two methods that hit the network with
# deferToThread wrappers to prevent accidental sync calls in
# other code.
run = mapreduce.run
stream = mapreduce.stream
mapreduce.run = lambda *a, **kw: deferToThread(run, *a, **kw)
mapreduce.stream = lambda *a, **kw: deferToThread(stream, *a, **kw)
return mapreduce
def riak_map_reduce(self):
return RiakMapReduce(self.client)
def query2(client, start, duration=HOURS24):
"""
Count number of loopdata records with speed over 100
"""
# lookup detector ids for Foster NB station
detectorids = lookup_detectors(FOSTER_NB_STATIONID)
# build list of key filters for any key starting with detector id
# for Foster NB station
# starts_with 1361 OR starts_with 1362 OR starts_with 1363
foster = None
for detector in detectorids:
f = RiakKeyFilter().starts_with(detector)
if not foster:
foster = f
else:
foster = foster | f
# filter records where volume is zero
volume = RiakKeyFilter().tokenize("-", 4)
volume += RiakKeyFilter().neq("0")
# key: <detector id>-<epoch>-<speed>-<volume>
# build key filters for epoch being between start and end times
start_epoch, end_epoch = timerange_convert(start, duration)
timerange = RiakKeyFilter().tokenize("-", 2)
timerange += RiakKeyFilter().string_to_int()
timerange += RiakKeyFilter().between(start_epoch, end_epoch)
mr = RiakMapReduce(client)
mr.add_bucket('loopdata')
mr.add_key_filters(volume & timerange & foster)
# return 1 for each record with speed greater than 100
# and reduce into total count
mr.map("""
function(record) {
var data = Riak.mapValuesJson(record)[0];
return [parseInt(data.volume)];
}
""")
mr.reduce_sum()
response = mr.run(timeout=HOURS24 * 1000)
if response:
return response[0]
return None
def query1(client):
"""
Name: Speed Over 100
Desc: Count number of loopdata records with speed over 100
"""
mr = RiakMapReduce(client)
mr.add_bucket('loopdata')
# key: <detector id>-<epoch>-<speed>-<volume>
# tokenize key and grab speed, filter greater than 100
mr.add_key_filter("tokenize", "-", 3)
mr.add_key_filter("string_to_int")
mr.add_key_filter("greater_than", 100)
# return 1 for each record with speed greater than 100
# and reduce into total count
mr.map("function(record) { return [1]; }")
mr.reduce_sum()
response = mr.run(timeout=HOURS24 * 1000)
if response:
return response[0]
return None
def query3(client, start, duration=HOURS2):
"""
Count number of loopdata records with speed over 100
"""
# get length of Foster NB station
stations_bucket = Bucket(client, 'stations')
results = stations_bucket.get(FOSTER_NB_STATIONID)
length = results.data['length']
# lookup detector ids for Foster NB station
detectorids = lookup_detectors(FOSTER_NB_STATIONID)
# build list of key filters for any key starting with detector id
# for Foster NB station
# starts_with 1361 OR starts_with 1362 OR starts_with 1363
foster = None
for detector in detectorids:
f = RiakKeyFilter().starts_with(detector)
if not foster:
foster = f
else:
foster = foster | f
# filter records where volume is zero
volume = RiakKeyFilter().tokenize("-", 4)
volume += RiakKeyFilter().neq("0")
# key: <detector id>-<epoch>-<speed>-<volume>
# build key filters for epoch being between start and end times
start_epoch, end_epoch = timerange_convert(start, duration)
timerange = RiakKeyFilter().tokenize("-", 2)
timerange += RiakKeyFilter().string_to_int()
timerange += RiakKeyFilter().between(start_epoch, end_epoch)
mr = RiakMapReduce(client)
mr.add_bucket('loopdata')
mr.add_key_filters(volume & timerange & foster)
# return speed for each loopdata record matching key filters above
mr.map("""
function(record) {
var data = Riak.mapValuesJson(record)[0];
return [parseInt(data.speed)];
}
""")
mr.reduce("""
function(values) {
return values;
}
""")
# compute peak travel time from all mapped records
response = mr.run(timeout=HOURS24 * 1000)
if not response:
return None
avg_speed = sum(response) / float(len(response))
peak_travel_time = (float(length) / avg_speed) * 3600
return peak_travel_time