def stats_data(req, stats_type):
""" n and n_type will always be given. format may be None and will
default to 'html'. Also, either start/stop or left_start/left_stop/shift
will be present - if the latter, start and stop will be computed as left_start/left_stop
shifted by the value pointed to by shift.
"""
req_input = Bunch.fromkeys(
('utc_start', 'utc_stop', 'user_start', 'user_stop', 'n', 'n_type',
'format', 'left-start', 'left-stop', 'right-start', 'right-stop',
'shift', 'side', 'custom_range'))
for name in req_input:
req_input[name] = req.GET.get(name, '') or req.POST.get(name, '')
# Now, this may look odd but for some reason UTC timestamps submitted
# in the form of '2012-10-31T21:47:11.592868+00:00' get translated
# by Django into '2012-10-31T21:47:11.592868+00:00' so the plus sign disappears.
# We bring it back taking an advantage of the fact that ' 00:00' alone is never
# a proper string of characters in a UTC timestamp.
for name in ('utc_start', 'utc_stop'):
req_input[name] = req_input[name].replace(' 00:00', '+00:00')
req_input['orig_{}'.format(name)] = req_input[name]
try:
req_input.n = int(req_input.n)
except ValueError:
req_input.n = 0
req_input.format = req_input.format or 'html'
is_custom = False
if req_input.shift:
duration = skip_by_duration[req_input.shift]
format = user_format[duration]
shift_info = shift(parse(req_input.utc_start), req_input.user_start,
req.zato.user_profile, req_input.shift, duration,
format)
for date_type in ('utc', 'user'):
for direction in ('start', 'stop'):
full_name = '{}_{}'.format(date_type, direction)
req_input[full_name] = shift_info[full_name]
elif req_input.custom_range:
is_custom = True
req_input['utc_start'] = utc.fromutc(
from_user_to_utc(req_input.user_start,
req.zato.user_profile)).isoformat()
req_input['utc_stop'] = utc.fromutc(
from_user_to_utc(req_input.user_stop,
req.zato.user_profile)).isoformat()
req_input['user_start'] = req_input.user_start
req_input['user_stop'] = req_input.user_stop
return globals()['_stats_data_{}'.format(
req_input.format)](req.zato.user_profile, req_input, req.zato.client,
req.zato.cluster, stats_type, is_custom)
def stats_data(req, stats_type):
""" n and n_type will always be given. format may be None and will
default to 'html'. Also, either start/stop or left_start/left_stop/shift
will be present - if the latter, start and stop will be computed as left_start/left_stop
shifted by the value pointed to by shift.
"""
req_input = Bunch.fromkeys(('utc_start', 'utc_stop', 'user_start', 'user_stop',
'n', 'n_type', 'format', 'left-start', 'left-stop', 'right-start', 'right-stop',
'shift', 'side', 'custom_range'))
for name in req_input:
req_input[name] = req.GET.get(name, '') or req.POST.get(name, '')
# Now, this may look odd but for some reason UTC timestamps submitted
# in the form of '2012-10-31T21:47:11.592868+00:00' get translated
# by Django into '2012-10-31T21:47:11.592868+00:00' so the plus sign disappears.
# We bring it back taking an advantage of the fact that ' 00:00' alone is never
# a proper string of characters in a UTC timestamp.
for name in('utc_start', 'utc_stop'):
req_input[name] = req_input[name].replace(' 00:00', '+00:00')
req_input['orig_{}'.format(name)] = req_input[name]
try:
req_input.n = int(req_input.n)
except ValueError:
req_input.n = 0
req_input.format = req_input.format or 'html'
is_custom = False
if req_input.shift:
duration = skip_by_duration[req_input.shift]
format = user_format[duration]
shift_info = shift(parse(req_input.utc_start), req_input.user_start, req.zato.user_profile, req_input.shift, duration, format)
for date_type in('utc', 'user'):
for direction in('start', 'stop'):
full_name = '{}_{}'.format(date_type, direction)
req_input[full_name] = shift_info[full_name]
elif req_input.custom_range:
is_custom = True
req_input['utc_start'] = utc.fromutc(from_user_to_utc(req_input.user_start, req.zato.user_profile)).isoformat()
req_input['utc_stop'] = utc.fromutc(from_user_to_utc(req_input.user_stop, req.zato.user_profile)).isoformat()
req_input['user_start'] = req_input.user_start
req_input['user_stop'] = req_input.user_stop
return globals()['_stats_data_{}'.format(req_input.format)](req.zato.user_profile,
req_input, req.zato.client, req.zato.cluster, stats_type, is_custom)
def game_update(game_id, new_state=None):
with app.app_context():
game = Game.query.filter_by(id=game_id).first()
if game is not None:
now = datetime.utcnow()
job_id = 'game_' + str(game_id)
if new_state == 'start':
print('start game')
game.gameStarted = True
if game.state == 'stop':
game.roundEndData = now + timedelta(seconds=game.period)
elif game.state == 'pause':
game.roundEndData = now + timedelta(seconds=game.left_time)
game.state = 'start'
if game.startData is None:
game.startData = now
elif new_state == 'stop':
print('stop game')
game.roundEndData = now
try:
app.apscheduler.remove_job(job_id)
except JobLookupError:
pass
elif new_state == 'pause':
print('pause game')
game.update(now)
game.state = 'pause'
game.gameStarted = False
try:
app.apscheduler.remove_job(job_id)
except JobLookupError:
pass
timer_value = game.update(now)
if game.gameStarted and timer_value > 0:
# schedule next update
delay = timer_value
if timer_value > 20:
delay = 20
next_date = now + timedelta(seconds=delay)
app.apscheduler.add_job(id=job_id,
func=game_update,
trigger='date',
run_date=utc.fromutc(next_date),
kwargs={'game_id': game_id})
text_data = json.dumps({
'type': 'game_state',
'time': round(timer_value),
'timer_run': game.gameStarted,
'game_state': game.state
})
app.redis.publish(game.name, text_data)
db.session.commit()
def map_items(items):
batch_list = []
for batch in items:
normalized_cells = []
for c in batch['cellTowers']:
cell_radio = c['radioType']
if not cell_radio:
cell_radio = batch['radioType']
cell = {}
cell['radio'] = cell_radio
cell['mcc'] = c['mobileCountryCode']
cell['mnc'] = c['mobileNetworkCode']
cell['lac'] = c['locationAreaCode']
cell['cid'] = c['cellId']
cell['psc'] = c['psc']
cell['asu'] = c['asu']
cell['signal'] = c['signalStrength']
cell['ta'] = c['timingAdvance']
normalized_cells.append(cell)
normalized_wifi = []
for w in batch['wifiAccessPoints']:
wifi = {}
wifi['key'] = w['macAddress']
wifi['frequency'] = w['frequency']
wifi['channel'] = w['channel']
wifi['signal'] = w['signalStrength']
wifi['snr'] = w['signalToNoiseRatio']
normalized_wifi.append(wifi)
if batch['timestamp'] == 0:
batch['timestamp'] = time.time() * 1000.0
dt = utc.fromutc(
datetime.utcfromtimestamp(batch['timestamp'] /
1000.0).replace(tzinfo=utc))
normalized_batch = {
'lat': batch['latitude'],
'lon': batch['longitude'],
'time': dt,
'accuracy': batch['accuracy'],
'altitude': batch['altitude'],
'altitude_accuracy': batch['altitudeAccuracy'],
'radio': batch['radioType'],
'heading': batch['heading'],
'speed': batch['speed'],
'cell': normalized_cells,
'wifi': normalized_wifi,
}
batch_list.append(normalized_batch)
return batch_list
def prepare_measure_data(self, request_data):
batch_list = []
for batch in request_data['items']:
normalized_cells = []
for c in batch['cellTowers']:
cell_radio = c['radioType']
if not cell_radio:
cell_radio = batch['radioType']
cell = {}
cell['radio'] = cell_radio
cell['mcc'] = c['mobileCountryCode']
cell['mnc'] = c['mobileNetworkCode']
cell['lac'] = c['locationAreaCode']
cell['cid'] = c['cellId']
cell['psc'] = c['psc']
cell['asu'] = c['asu']
cell['signal'] = c['signalStrength']
cell['ta'] = c['timingAdvance']
normalized_cells.append(cell)
normalized_wifi = []
for w in batch['wifiAccessPoints']:
wifi = {}
wifi['key'] = w['macAddress']
wifi['frequency'] = w['frequency']
wifi['channel'] = w['channel']
wifi['signal'] = w['signalStrength']
wifi['snr'] = w['signalToNoiseRatio']
normalized_wifi.append(wifi)
timestamp = batch['timestamp']
if timestamp == 0:
timestamp = time.time() * 1000.0
dt = utc.fromutc(datetime.utcfromtimestamp(
timestamp / 1000.0).replace(tzinfo=utc))
normalized_batch = {'lat': batch['latitude'],
'lon': batch['longitude'],
'time': dt,
'accuracy': batch['accuracy'],
'altitude': batch['altitude'],
'altitude_accuracy': batch['altitudeAccuracy'],
'heading': batch['heading'],
'speed': batch['speed'],
'cell': normalized_cells,
'wifi': normalized_wifi,
}
batch_list.append(normalized_batch)
return batch_list
def get_default_date(date_type, user_profile, format):
""" Returns default start and stop date in UTC and user's timezone depending
on the stats type and duration requested.
"""
def get_today(_user_profile, _format):
""" user_start is today's midnight but it needs to be in user's TZ. user_stop is current time simply,
in user's timezone again.
"""
user_now = now(timezone(_user_profile.timezone)).replace(tzinfo=None)
user_today_midnight = datetime(user_now.year, user_now.month, user_now.day)
utc_start = from_local_to_utc(user_today_midnight, _user_profile.timezone)
utc_stop = from_local_to_utc(user_now, _user_profile.timezone)
user_start = from_utc_to_user(utc_start, _user_profile, _format)
user_stop = None
return utc_start, utc_stop, user_start, user_stop
if date_type == 'last_hour':
# stop is what current time is now so return it in UTC and user's TZ
# along with start which will be equal to stop - 1 hour.
utc_stop = utc.fromutc(utcnow())
utc_start = utc.fromutc(utc_stop + relativedelta(hours=-1))
user_start = from_utc_to_user(utc_start, user_profile)
user_stop = from_utc_to_user(utc_stop, user_profile)
label = 'Last hour'
step = 'hour'
elif date_type == 'today':
utc_start, utc_stop, user_start, user_stop = get_today(user_profile, format)
label = 'Today'
step = 'day'
elif date_type == 'yesterday':
# Yesterday's start is today's start - 1 day
today_utc_start, today_utc_stop, today_user_start, user_stop = get_today(user_profile, format)
utc_start = today_utc_start + relativedelta(days=-1)
utc_stop = utc_start + relativedelta(days=1)
user_start = from_utc_to_user(utc_start, user_profile, format)
label = 'Yesterday'
step = 'day'
elif date_type == 'this_week':
# This week extends from Monday midnight to right now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_prev_monday = user_now + relativedelta(weekday=MO(-1))
user_prev_monday = datetime(year=user_prev_monday.year, month=user_prev_monday.month, day=user_prev_monday.day)
utc_start = from_local_to_utc(user_prev_monday, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = from_utc_to_user(utc_stop, user_profile, format)
label = 'This week'
step = 'week'
elif date_type == 'this_month':
# From midnight the first day of month up until now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_1st_of_month = datetime(year=user_now.year, month=user_now.month, day=1)
utc_start = from_local_to_utc(user_1st_of_month, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = None
label = 'This month'
step = 'month'
elif date_type == 'this_year':
# From midnight the first day of year up until now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_new_year = datetime(year=user_now.year, month=1, day=1)
utc_start = from_local_to_utc(user_new_year, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = None
label = 'This year'
step = 'year'
else:
raise ValueError('Unrecognized date_type:[{}]'.format(date_type))
return DateInfo(utc_start.isoformat(), utc_stop.isoformat(), user_start, user_stop, label, step)
def get_default_date(date_type, user_profile, format):
""" Returns default start and stop date in UTC and user's timezone depending
on the stats type and duration requested.
"""
def get_today(_user_profile, _format):
""" user_start is today's midnight but it needs to be in user's TZ. user_stop is current time simply,
in user's timezone again.
"""
user_now = now(timezone(_user_profile.timezone)).replace(tzinfo=None)
user_today_midnight = datetime(user_now.year, user_now.month,
user_now.day)
utc_start = from_local_to_utc(user_today_midnight,
_user_profile.timezone)
utc_stop = from_local_to_utc(user_now, _user_profile.timezone)
user_start = from_utc_to_user(utc_start, _user_profile, _format)
user_stop = None
return utc_start, utc_stop, user_start, user_stop
if date_type == 'last_hour':
# stop is what current time is now so return it in UTC and user's TZ
# along with start which will be equal to stop - 1 hour.
utc_stop = utc.fromutc(utcnow())
utc_start = utc.fromutc(utc_stop + relativedelta(hours=-1))
user_start = from_utc_to_user(utc_start, user_profile)
user_stop = from_utc_to_user(utc_stop, user_profile)
label = 'Last hour'
step = 'hour'
elif date_type == 'today':
utc_start, utc_stop, user_start, user_stop = get_today(
user_profile, format)
label = 'Today'
step = 'day'
elif date_type == 'yesterday':
# Yesterday's start is today's start - 1 day
today_utc_start, today_utc_stop, today_user_start, user_stop = get_today(
user_profile, format)
utc_start = today_utc_start + relativedelta(days=-1)
utc_stop = utc_start + relativedelta(days=1)
user_start = from_utc_to_user(utc_start, user_profile, format)
label = 'Yesterday'
step = 'day'
elif date_type == 'this_week':
# This week extends from Monday midnight to right now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_prev_monday = user_now + relativedelta(weekday=MO(-1))
user_prev_monday = datetime(year=user_prev_monday.year,
month=user_prev_monday.month,
day=user_prev_monday.day)
utc_start = from_local_to_utc(user_prev_monday, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = from_utc_to_user(utc_stop, user_profile, format)
label = 'This week'
step = 'week'
elif date_type == 'this_month':
# From midnight the first day of month up until now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_1st_of_month = datetime(year=user_now.year,
month=user_now.month,
day=1)
utc_start = from_local_to_utc(user_1st_of_month, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = None
label = 'This month'
step = 'month'
elif date_type == 'this_year':
# From midnight the first day of year up until now
user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None)
user_new_year = datetime(year=user_now.year, month=1, day=1)
utc_start = from_local_to_utc(user_new_year, user_profile.timezone)
utc_stop = from_local_to_utc(user_now, user_profile.timezone)
user_start = from_utc_to_user(utc_start, user_profile, format)
user_stop = None
label = 'This year'
step = 'year'
else:
raise ValueError('Unrecognized date_type:[{}]'.format(date_type))
return DateInfo(utc_start.isoformat(), utc_stop.isoformat(), user_start,
user_stop, label, step)
def process_upload(nickname, email, items):
if isinstance(nickname, str): # pragma: no cover
nickname = nickname.decode('utf-8', 'ignore')
if isinstance(email, str): # pragma: no cover
email = email.decode('utf-8', 'ignore')
batch_list = []
for batch in items:
normalized_cells = []
for c in batch['cellTowers']:
cell = {}
cell['radio'] = batch['radioType']
cell['mcc'] = c['mobileCountryCode']
cell['mnc'] = c['mobileNetworkCode']
cell['lac'] = c['locationAreaCode']
cell['cid'] = c['cellId']
cell['psc'] = c['psc']
cell['asu'] = c['asu']
cell['signal'] = c['signalStrength']
cell['ta'] = c['timingAdvance']
normalized_cells.append(cell)
normalized_wifi = []
for w in batch['wifiAccessPoints']:
wifi = {}
wifi['key'] = w['macAddress']
wifi['frequency'] = w['frequency']
wifi['channel'] = w['channel']
wifi['signal'] = w['signalStrength']
wifi['signalToNoiseRatio'] = w['signalToNoiseRatio']
normalized_wifi.append(wifi)
if batch['timestamp'] == 0:
batch['timestamp'] = time.time() * 1000.0
dt = utc.fromutc(datetime.utcfromtimestamp(
batch['timestamp'] / 1000.0).replace(tzinfo=utc))
ts = dt.isoformat()
normalized_batch = {'lat': batch['latitude'],
'lon': batch['longitude'],
'time': ts,
'accuracy': batch['accuracy'],
'altitude': batch['altitude'],
'altitude_accuracy': batch['altitudeAccuracy'],
'radio': batch['radioType'],
'heading': batch['heading'],
'speed': batch['speed'],
'cell': normalized_cells,
'wifi': normalized_wifi,
}
batch_list.append(normalized_batch)
# Run the SubmitSchema validator against the normalized submit
# data.
schema = SubmitSchema()
body = {'items': batch_list}
errors = []
validated = {}
verify_schema(schema, body, errors, validated)
if errors: # pragma: no cover
# Short circuit on any error in schema validation
return errors
for i in range(0, len(batch_list), 100):
batch_items = dumps(batch_list[i:i + 100])
# insert measures, expire the task if it wasn't processed
# after six hours to avoid queue overload
try:
insert_measures.apply_async(
kwargs={
'email': email,
'items': batch_items,
'nickname': nickname,
},
expires=21600)
except ConnectionError: # pragma: no cover
return SENTINEL
return errors
def process_upload(nickname, email, items):
if isinstance(nickname, str): # pragma: no cover
nickname = nickname.decode('utf-8', 'ignore')
if isinstance(email, str): # pragma: no cover
email = email.decode('utf-8', 'ignore')
batch_list = []
for batch in items:
normalized_cells = []
for c in batch['cellTowers']:
cell = {}
cell['radio'] = batch['radioType']
cell['mcc'] = c['mobileCountryCode']
cell['mnc'] = c['mobileNetworkCode']
cell['lac'] = c['locationAreaCode']
cell['cid'] = c['cellId']
cell['psc'] = c['psc']
cell['asu'] = c['asu']
cell['signal'] = c['signalStrength']
cell['ta'] = c['timingAdvance']
normalized_cells.append(cell)
normalized_wifi = []
for w in batch['wifiAccessPoints']:
wifi = {}
wifi['key'] = w['macAddress']
wifi['frequency'] = w['frequency']
wifi['channel'] = w['channel']
wifi['signal'] = w['signalStrength']
wifi['signalToNoiseRatio'] = w['signalToNoiseRatio']
normalized_wifi.append(wifi)
if batch['timestamp'] == 0:
batch['timestamp'] = time.time() * 1000.0
dt = utc.fromutc(
datetime.utcfromtimestamp(batch['timestamp'] /
1000.0).replace(tzinfo=utc))
ts = dt.isoformat()
normalized_batch = {
'lat': batch['latitude'],
'lon': batch['longitude'],
'time': ts,
'accuracy': batch['accuracy'],
'altitude': batch['altitude'],
'altitude_accuracy': batch['altitudeAccuracy'],
'radio': batch['radioType'],
'heading': batch['heading'],
'speed': batch['speed'],
'cell': normalized_cells,
'wifi': normalized_wifi,
}
batch_list.append(normalized_batch)
# Run the SubmitSchema validator against the normalized submit
# data.
schema = SubmitSchema()
body = {'items': batch_list}
errors = []
validated = {}
verify_schema(schema, body, errors, validated)
if errors: # pragma: no cover
# Short circuit on any error in schema validation
return errors
for i in range(0, len(batch_list), 100):
batch_items = dumps(batch_list[i:i + 100])
# insert measures, expire the task if it wasn't processed
# after six hours to avoid queue overload
try:
insert_measures.apply_async(kwargs={
'email': email,
'items': batch_items,
'nickname': nickname,
},
expires=21600)
except ConnectionError: # pragma: no cover
return SENTINEL
return errors
