def stats_data(req, stats_type): """ n and n_type will always be given. format may be None and will default to 'html'. Also, either start/stop or left_start/left_stop/shift will be present - if the latter, start and stop will be computed as left_start/left_stop shifted by the value pointed to by shift. """ req_input = Bunch.fromkeys( ('utc_start', 'utc_stop', 'user_start', 'user_stop', 'n', 'n_type', 'format', 'left-start', 'left-stop', 'right-start', 'right-stop', 'shift', 'side', 'custom_range')) for name in req_input: req_input[name] = req.GET.get(name, '') or req.POST.get(name, '') # Now, this may look odd but for some reason UTC timestamps submitted # in the form of '2012-10-31T21:47:11.592868+00:00' get translated # by Django into '2012-10-31T21:47:11.592868+00:00' so the plus sign disappears. # We bring it back taking an advantage of the fact that ' 00:00' alone is never # a proper string of characters in a UTC timestamp. for name in ('utc_start', 'utc_stop'): req_input[name] = req_input[name].replace(' 00:00', '+00:00') req_input['orig_{}'.format(name)] = req_input[name] try: req_input.n = int(req_input.n) except ValueError: req_input.n = 0 req_input.format = req_input.format or 'html' is_custom = False if req_input.shift: duration = skip_by_duration[req_input.shift] format = user_format[duration] shift_info = shift(parse(req_input.utc_start), req_input.user_start, req.zato.user_profile, req_input.shift, duration, format) for date_type in ('utc', 'user'): for direction in ('start', 'stop'): full_name = '{}_{}'.format(date_type, direction) req_input[full_name] = shift_info[full_name] elif req_input.custom_range: is_custom = True req_input['utc_start'] = utc.fromutc( from_user_to_utc(req_input.user_start, req.zato.user_profile)).isoformat() req_input['utc_stop'] = utc.fromutc( from_user_to_utc(req_input.user_stop, req.zato.user_profile)).isoformat() req_input['user_start'] = req_input.user_start req_input['user_stop'] = req_input.user_stop return globals()['_stats_data_{}'.format( req_input.format)](req.zato.user_profile, req_input, req.zato.client, req.zato.cluster, stats_type, is_custom)
def stats_data(req, stats_type): """ n and n_type will always be given. format may be None and will default to 'html'. Also, either start/stop or left_start/left_stop/shift will be present - if the latter, start and stop will be computed as left_start/left_stop shifted by the value pointed to by shift. """ req_input = Bunch.fromkeys(('utc_start', 'utc_stop', 'user_start', 'user_stop', 'n', 'n_type', 'format', 'left-start', 'left-stop', 'right-start', 'right-stop', 'shift', 'side', 'custom_range')) for name in req_input: req_input[name] = req.GET.get(name, '') or req.POST.get(name, '') # Now, this may look odd but for some reason UTC timestamps submitted # in the form of '2012-10-31T21:47:11.592868+00:00' get translated # by Django into '2012-10-31T21:47:11.592868+00:00' so the plus sign disappears. # We bring it back taking an advantage of the fact that ' 00:00' alone is never # a proper string of characters in a UTC timestamp. for name in('utc_start', 'utc_stop'): req_input[name] = req_input[name].replace(' 00:00', '+00:00') req_input['orig_{}'.format(name)] = req_input[name] try: req_input.n = int(req_input.n) except ValueError: req_input.n = 0 req_input.format = req_input.format or 'html' is_custom = False if req_input.shift: duration = skip_by_duration[req_input.shift] format = user_format[duration] shift_info = shift(parse(req_input.utc_start), req_input.user_start, req.zato.user_profile, req_input.shift, duration, format) for date_type in('utc', 'user'): for direction in('start', 'stop'): full_name = '{}_{}'.format(date_type, direction) req_input[full_name] = shift_info[full_name] elif req_input.custom_range: is_custom = True req_input['utc_start'] = utc.fromutc(from_user_to_utc(req_input.user_start, req.zato.user_profile)).isoformat() req_input['utc_stop'] = utc.fromutc(from_user_to_utc(req_input.user_stop, req.zato.user_profile)).isoformat() req_input['user_start'] = req_input.user_start req_input['user_stop'] = req_input.user_stop return globals()['_stats_data_{}'.format(req_input.format)](req.zato.user_profile, req_input, req.zato.client, req.zato.cluster, stats_type, is_custom)
def game_update(game_id, new_state=None): with app.app_context(): game = Game.query.filter_by(id=game_id).first() if game is not None: now = datetime.utcnow() job_id = 'game_' + str(game_id) if new_state == 'start': print('start game') game.gameStarted = True if game.state == 'stop': game.roundEndData = now + timedelta(seconds=game.period) elif game.state == 'pause': game.roundEndData = now + timedelta(seconds=game.left_time) game.state = 'start' if game.startData is None: game.startData = now elif new_state == 'stop': print('stop game') game.roundEndData = now try: app.apscheduler.remove_job(job_id) except JobLookupError: pass elif new_state == 'pause': print('pause game') game.update(now) game.state = 'pause' game.gameStarted = False try: app.apscheduler.remove_job(job_id) except JobLookupError: pass timer_value = game.update(now) if game.gameStarted and timer_value > 0: # schedule next update delay = timer_value if timer_value > 20: delay = 20 next_date = now + timedelta(seconds=delay) app.apscheduler.add_job(id=job_id, func=game_update, trigger='date', run_date=utc.fromutc(next_date), kwargs={'game_id': game_id}) text_data = json.dumps({ 'type': 'game_state', 'time': round(timer_value), 'timer_run': game.gameStarted, 'game_state': game.state }) app.redis.publish(game.name, text_data) db.session.commit()
def map_items(items): batch_list = [] for batch in items: normalized_cells = [] for c in batch['cellTowers']: cell_radio = c['radioType'] if not cell_radio: cell_radio = batch['radioType'] cell = {} cell['radio'] = cell_radio cell['mcc'] = c['mobileCountryCode'] cell['mnc'] = c['mobileNetworkCode'] cell['lac'] = c['locationAreaCode'] cell['cid'] = c['cellId'] cell['psc'] = c['psc'] cell['asu'] = c['asu'] cell['signal'] = c['signalStrength'] cell['ta'] = c['timingAdvance'] normalized_cells.append(cell) normalized_wifi = [] for w in batch['wifiAccessPoints']: wifi = {} wifi['key'] = w['macAddress'] wifi['frequency'] = w['frequency'] wifi['channel'] = w['channel'] wifi['signal'] = w['signalStrength'] wifi['snr'] = w['signalToNoiseRatio'] normalized_wifi.append(wifi) if batch['timestamp'] == 0: batch['timestamp'] = time.time() * 1000.0 dt = utc.fromutc( datetime.utcfromtimestamp(batch['timestamp'] / 1000.0).replace(tzinfo=utc)) normalized_batch = { 'lat': batch['latitude'], 'lon': batch['longitude'], 'time': dt, 'accuracy': batch['accuracy'], 'altitude': batch['altitude'], 'altitude_accuracy': batch['altitudeAccuracy'], 'radio': batch['radioType'], 'heading': batch['heading'], 'speed': batch['speed'], 'cell': normalized_cells, 'wifi': normalized_wifi, } batch_list.append(normalized_batch) return batch_list
def prepare_measure_data(self, request_data): batch_list = [] for batch in request_data['items']: normalized_cells = [] for c in batch['cellTowers']: cell_radio = c['radioType'] if not cell_radio: cell_radio = batch['radioType'] cell = {} cell['radio'] = cell_radio cell['mcc'] = c['mobileCountryCode'] cell['mnc'] = c['mobileNetworkCode'] cell['lac'] = c['locationAreaCode'] cell['cid'] = c['cellId'] cell['psc'] = c['psc'] cell['asu'] = c['asu'] cell['signal'] = c['signalStrength'] cell['ta'] = c['timingAdvance'] normalized_cells.append(cell) normalized_wifi = [] for w in batch['wifiAccessPoints']: wifi = {} wifi['key'] = w['macAddress'] wifi['frequency'] = w['frequency'] wifi['channel'] = w['channel'] wifi['signal'] = w['signalStrength'] wifi['snr'] = w['signalToNoiseRatio'] normalized_wifi.append(wifi) timestamp = batch['timestamp'] if timestamp == 0: timestamp = time.time() * 1000.0 dt = utc.fromutc(datetime.utcfromtimestamp( timestamp / 1000.0).replace(tzinfo=utc)) normalized_batch = {'lat': batch['latitude'], 'lon': batch['longitude'], 'time': dt, 'accuracy': batch['accuracy'], 'altitude': batch['altitude'], 'altitude_accuracy': batch['altitudeAccuracy'], 'heading': batch['heading'], 'speed': batch['speed'], 'cell': normalized_cells, 'wifi': normalized_wifi, } batch_list.append(normalized_batch) return batch_list
def get_default_date(date_type, user_profile, format): """ Returns default start and stop date in UTC and user's timezone depending on the stats type and duration requested. """ def get_today(_user_profile, _format): """ user_start is today's midnight but it needs to be in user's TZ. user_stop is current time simply, in user's timezone again. """ user_now = now(timezone(_user_profile.timezone)).replace(tzinfo=None) user_today_midnight = datetime(user_now.year, user_now.month, user_now.day) utc_start = from_local_to_utc(user_today_midnight, _user_profile.timezone) utc_stop = from_local_to_utc(user_now, _user_profile.timezone) user_start = from_utc_to_user(utc_start, _user_profile, _format) user_stop = None return utc_start, utc_stop, user_start, user_stop if date_type == 'last_hour': # stop is what current time is now so return it in UTC and user's TZ # along with start which will be equal to stop - 1 hour. utc_stop = utc.fromutc(utcnow()) utc_start = utc.fromutc(utc_stop + relativedelta(hours=-1)) user_start = from_utc_to_user(utc_start, user_profile) user_stop = from_utc_to_user(utc_stop, user_profile) label = 'Last hour' step = 'hour' elif date_type == 'today': utc_start, utc_stop, user_start, user_stop = get_today(user_profile, format) label = 'Today' step = 'day' elif date_type == 'yesterday': # Yesterday's start is today's start - 1 day today_utc_start, today_utc_stop, today_user_start, user_stop = get_today(user_profile, format) utc_start = today_utc_start + relativedelta(days=-1) utc_stop = utc_start + relativedelta(days=1) user_start = from_utc_to_user(utc_start, user_profile, format) label = 'Yesterday' step = 'day' elif date_type == 'this_week': # This week extends from Monday midnight to right now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_prev_monday = user_now + relativedelta(weekday=MO(-1)) user_prev_monday = datetime(year=user_prev_monday.year, month=user_prev_monday.month, day=user_prev_monday.day) utc_start = from_local_to_utc(user_prev_monday, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = from_utc_to_user(utc_stop, user_profile, format) label = 'This week' step = 'week' elif date_type == 'this_month': # From midnight the first day of month up until now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_1st_of_month = datetime(year=user_now.year, month=user_now.month, day=1) utc_start = from_local_to_utc(user_1st_of_month, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = None label = 'This month' step = 'month' elif date_type == 'this_year': # From midnight the first day of year up until now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_new_year = datetime(year=user_now.year, month=1, day=1) utc_start = from_local_to_utc(user_new_year, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = None label = 'This year' step = 'year' else: raise ValueError('Unrecognized date_type:[{}]'.format(date_type)) return DateInfo(utc_start.isoformat(), utc_stop.isoformat(), user_start, user_stop, label, step)
def get_default_date(date_type, user_profile, format): """ Returns default start and stop date in UTC and user's timezone depending on the stats type and duration requested. """ def get_today(_user_profile, _format): """ user_start is today's midnight but it needs to be in user's TZ. user_stop is current time simply, in user's timezone again. """ user_now = now(timezone(_user_profile.timezone)).replace(tzinfo=None) user_today_midnight = datetime(user_now.year, user_now.month, user_now.day) utc_start = from_local_to_utc(user_today_midnight, _user_profile.timezone) utc_stop = from_local_to_utc(user_now, _user_profile.timezone) user_start = from_utc_to_user(utc_start, _user_profile, _format) user_stop = None return utc_start, utc_stop, user_start, user_stop if date_type == 'last_hour': # stop is what current time is now so return it in UTC and user's TZ # along with start which will be equal to stop - 1 hour. utc_stop = utc.fromutc(utcnow()) utc_start = utc.fromutc(utc_stop + relativedelta(hours=-1)) user_start = from_utc_to_user(utc_start, user_profile) user_stop = from_utc_to_user(utc_stop, user_profile) label = 'Last hour' step = 'hour' elif date_type == 'today': utc_start, utc_stop, user_start, user_stop = get_today( user_profile, format) label = 'Today' step = 'day' elif date_type == 'yesterday': # Yesterday's start is today's start - 1 day today_utc_start, today_utc_stop, today_user_start, user_stop = get_today( user_profile, format) utc_start = today_utc_start + relativedelta(days=-1) utc_stop = utc_start + relativedelta(days=1) user_start = from_utc_to_user(utc_start, user_profile, format) label = 'Yesterday' step = 'day' elif date_type == 'this_week': # This week extends from Monday midnight to right now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_prev_monday = user_now + relativedelta(weekday=MO(-1)) user_prev_monday = datetime(year=user_prev_monday.year, month=user_prev_monday.month, day=user_prev_monday.day) utc_start = from_local_to_utc(user_prev_monday, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = from_utc_to_user(utc_stop, user_profile, format) label = 'This week' step = 'week' elif date_type == 'this_month': # From midnight the first day of month up until now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_1st_of_month = datetime(year=user_now.year, month=user_now.month, day=1) utc_start = from_local_to_utc(user_1st_of_month, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = None label = 'This month' step = 'month' elif date_type == 'this_year': # From midnight the first day of year up until now user_now = now(timezone(user_profile.timezone)).replace(tzinfo=None) user_new_year = datetime(year=user_now.year, month=1, day=1) utc_start = from_local_to_utc(user_new_year, user_profile.timezone) utc_stop = from_local_to_utc(user_now, user_profile.timezone) user_start = from_utc_to_user(utc_start, user_profile, format) user_stop = None label = 'This year' step = 'year' else: raise ValueError('Unrecognized date_type:[{}]'.format(date_type)) return DateInfo(utc_start.isoformat(), utc_stop.isoformat(), user_start, user_stop, label, step)
def process_upload(nickname, email, items): if isinstance(nickname, str): # pragma: no cover nickname = nickname.decode('utf-8', 'ignore') if isinstance(email, str): # pragma: no cover email = email.decode('utf-8', 'ignore') batch_list = [] for batch in items: normalized_cells = [] for c in batch['cellTowers']: cell = {} cell['radio'] = batch['radioType'] cell['mcc'] = c['mobileCountryCode'] cell['mnc'] = c['mobileNetworkCode'] cell['lac'] = c['locationAreaCode'] cell['cid'] = c['cellId'] cell['psc'] = c['psc'] cell['asu'] = c['asu'] cell['signal'] = c['signalStrength'] cell['ta'] = c['timingAdvance'] normalized_cells.append(cell) normalized_wifi = [] for w in batch['wifiAccessPoints']: wifi = {} wifi['key'] = w['macAddress'] wifi['frequency'] = w['frequency'] wifi['channel'] = w['channel'] wifi['signal'] = w['signalStrength'] wifi['signalToNoiseRatio'] = w['signalToNoiseRatio'] normalized_wifi.append(wifi) if batch['timestamp'] == 0: batch['timestamp'] = time.time() * 1000.0 dt = utc.fromutc(datetime.utcfromtimestamp( batch['timestamp'] / 1000.0).replace(tzinfo=utc)) ts = dt.isoformat() normalized_batch = {'lat': batch['latitude'], 'lon': batch['longitude'], 'time': ts, 'accuracy': batch['accuracy'], 'altitude': batch['altitude'], 'altitude_accuracy': batch['altitudeAccuracy'], 'radio': batch['radioType'], 'heading': batch['heading'], 'speed': batch['speed'], 'cell': normalized_cells, 'wifi': normalized_wifi, } batch_list.append(normalized_batch) # Run the SubmitSchema validator against the normalized submit # data. schema = SubmitSchema() body = {'items': batch_list} errors = [] validated = {} verify_schema(schema, body, errors, validated) if errors: # pragma: no cover # Short circuit on any error in schema validation return errors for i in range(0, len(batch_list), 100): batch_items = dumps(batch_list[i:i + 100]) # insert measures, expire the task if it wasn't processed # after six hours to avoid queue overload try: insert_measures.apply_async( kwargs={ 'email': email, 'items': batch_items, 'nickname': nickname, }, expires=21600) except ConnectionError: # pragma: no cover return SENTINEL return errors
def process_upload(nickname, email, items): if isinstance(nickname, str): # pragma: no cover nickname = nickname.decode('utf-8', 'ignore') if isinstance(email, str): # pragma: no cover email = email.decode('utf-8', 'ignore') batch_list = [] for batch in items: normalized_cells = [] for c in batch['cellTowers']: cell = {} cell['radio'] = batch['radioType'] cell['mcc'] = c['mobileCountryCode'] cell['mnc'] = c['mobileNetworkCode'] cell['lac'] = c['locationAreaCode'] cell['cid'] = c['cellId'] cell['psc'] = c['psc'] cell['asu'] = c['asu'] cell['signal'] = c['signalStrength'] cell['ta'] = c['timingAdvance'] normalized_cells.append(cell) normalized_wifi = [] for w in batch['wifiAccessPoints']: wifi = {} wifi['key'] = w['macAddress'] wifi['frequency'] = w['frequency'] wifi['channel'] = w['channel'] wifi['signal'] = w['signalStrength'] wifi['signalToNoiseRatio'] = w['signalToNoiseRatio'] normalized_wifi.append(wifi) if batch['timestamp'] == 0: batch['timestamp'] = time.time() * 1000.0 dt = utc.fromutc( datetime.utcfromtimestamp(batch['timestamp'] / 1000.0).replace(tzinfo=utc)) ts = dt.isoformat() normalized_batch = { 'lat': batch['latitude'], 'lon': batch['longitude'], 'time': ts, 'accuracy': batch['accuracy'], 'altitude': batch['altitude'], 'altitude_accuracy': batch['altitudeAccuracy'], 'radio': batch['radioType'], 'heading': batch['heading'], 'speed': batch['speed'], 'cell': normalized_cells, 'wifi': normalized_wifi, } batch_list.append(normalized_batch) # Run the SubmitSchema validator against the normalized submit # data. schema = SubmitSchema() body = {'items': batch_list} errors = [] validated = {} verify_schema(schema, body, errors, validated) if errors: # pragma: no cover # Short circuit on any error in schema validation return errors for i in range(0, len(batch_list), 100): batch_items = dumps(batch_list[i:i + 100]) # insert measures, expire the task if it wasn't processed # after six hours to avoid queue overload try: insert_measures.apply_async(kwargs={ 'email': email, 'items': batch_items, 'nickname': nickname, }, expires=21600) except ConnectionError: # pragma: no cover return SENTINEL return errors