def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
m = entry.metadata
fc.expand_metadata_times(m)
formatted_entry.metadata = m
data = ad.AttrDict()
data.curr_state = state_map[entry.data.currState].value
logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
# The iOS state diagram is significantly more complex than the android state diagram
# So there are a lot more transitions. But some of the intermediate states are
# not interesting, so it seems like it should be possible to collapse them to the
# simple 2-state android state machine. But that requires looking at a window of
# transitions, which we don't have here. Let's focus on simply mapping here and
# deal with collapsing later
# data.transition_raw = entry.data.transition
data.transition = transition_map[entry.data.transition].value
logging.debug("Mapped %s -> %s" % (entry.data.transition, data.transition))
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts) / 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(
entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
if 'agb' in entry.data:
data.type = ecwa.MotionTypes(entry.data.agb).value
else:
data.type = ecwa.MotionTypes(entry.data.zzaEg).value
if 'agc' in entry.data:
data.confidence = entry.data.agc
else:
data.confidence = entry.data.zzaEh
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts)/ 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
# There are lots of fields incoming on android, so instead of copying each
# one over, let's just copy the whole thing
data = copy.copy(entry.data)
data.battery_status = status_map[entry.data.battery_status].value
logging.debug("Mapped %s -> %s" % (entry.data.battery_status, data.battery_status))
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
# ios sets data in a ratio between 0 and 1, so let's convert to percent to be consistent
# with android
data.battery_level_pct = entry.data.battery_level_ratio * 100
data.battery_status = status_map[entry.data.battery_status].value
logging.debug("Mapped %s -> %s" %
(entry.data.battery_status, data.battery_status))
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
if 'agb' in entry.data:
data.type = ecwa.MotionTypes(entry.data.agb).value
elif 'zzaEg' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzaEg).value
else:
data.type = ecwa.MotionTypes(entry.data.zzaKM).value
if 'agc' in entry.data:
data.confidence = entry.data.agc
elif 'zzaEh' in entry.data:
data.confidence = entry.data.zzaEh
else:
data.confidence = entry.data.zzaKN
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
data.type = type_flags_to_enum(entry.data).value
# TODO: for ios, the confidence is currently a string "high/medium/low".
# should we convert it to a number to be consistent with the android version?
# or should we leave it unchanged?
# Let us convert it so a number so that we can try to reuse the same code
# as android
if 'confidence' in entry.data and 'confidence_level' not in entry.data:
data.confidence_level = entry.data.confidence
data.confidence = level_to_number(data.confidence_level)
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts)/ 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = pydt.datetime.utcfromtimestamp(data.ts).replace(tzinfo=pytz.utc) \
.astimezone(pytz.timezone(formatted_entry.metadata.time_zone))
data.fmt_time = data.local_dt.isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
m = entry.metadata
if "time_zone" not in m:
m.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" %
(entry.metadata.write_ts, m.write_ts))
fc.expand_metadata_times(m)
formatted_entry.metadata = m
data = ad.AttrDict()
data.curr_state = state_map[entry.data.currState].value
logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
data.transition = transition_map[entry.data.transition].value
if "ts" not in data:
data.ts = formatted_entry.metadata.write_ts
logging.debug("No existing timestamp, copyied from metadata%s" %
data.ts)
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
else:
logging.debug("Retaining existing timestamp %s" % data.ts)
fc.expand_data_times(data, metadata)
formatted_entry.data = data
return formatted_entry
def old2new(old_style_data):
import emission.core.wrapper.entry as ecwe
import emission.core.wrapper.statsevent as ecws
import emission.core.wrapper.battery as ecwb
none2None = lambda s: None if s == 'none' else s
float_with_none = lambda s: float(s) if s is not None else None
ms_to_sec_with_none = lambda s: (float(s))/1000 if type(s) == str or type(s) == unicode else float(s)
user_id = old_style_data["user"]
del old_style_data["user"]
if old_style_data["stat"] == "battery_level":
new_style_data = ecwb.Battery({
"battery_level_pct" : float_with_none(none2None(old_style_data["reading"])),
"ts": ms_to_sec_with_none(old_style_data["ts"])
})
new_key = "background/battery"
else:
new_style_data = ecws.Statsevent()
new_style_data.name = old_style_data["stat"]
new_style_data.ts = ms_to_sec_with_none(old_style_data["ts"])
new_style_data.reading = float_with_none(none2None(old_style_data["reading"]))
new_style_data.client_app_version = old_style_data["client_app_version"]
new_style_data.client_os_version = old_style_data["client_os_version"]
new_key = stat2key(old_style_data["stat"])
new_entry = ecwe.Entry.create_entry(user_id, new_key, new_style_data)
# For legacy entries, make sure that the write_ts doesn't become the conversion
# time or the server arrival time
new_entry["metadata"]["write_ts"] = float_with_none(old_style_data["reported_ts"])
del new_entry["metadata"]["write_local_dt"]
del new_entry["metadata"]["write_fmt_time"]
enufc.expand_metadata_times(new_entry["metadata"])
return new_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = old_div(float(entry.metadata.write_ts), 1000)
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = old_div(float(entry.data.mTime),
1000) # convert the ms from the phone to secs
fc.expand_data_times(data, metadata)
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
m = entry.metadata
if "time_zone" not in m:
m.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" % (entry.metadata.write_ts, m.write_ts))
fc.expand_metadata_times(m)
formatted_entry.metadata = m
data = ad.AttrDict()
data.curr_state = state_map[entry.data.currState].value
logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
data.transition = transition_map[entry.data.transition].value
if "ts" not in data:
data.ts = formatted_entry.metadata.write_ts
logging.debug("No existing timestamp, copyied from metadata%s" % data.ts)
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
else:
logging.debug("Retaining existing timestamp %s" % data.ts)
fc.expand_data_times(data, metadata)
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts) / 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = float(
entry.data.mTime) / 1000 # convert the ms from the phone to secs
data.local_dt = pydt.datetime.utcfromtimestamp(data.ts).replace(tzinfo=pytz.utc) \
.astimezone(pytz.timezone(formatted_entry.metadata.time_zone))
data.fmt_time = data.local_dt.isoformat()
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
# There are lots of fields incoming on android, so instead of copying each
# one over, let's just copy the whole thing
data = copy.copy(entry.data)
data.battery_status = status_map[entry.data.battery_status].value
logging.debug("Mapped %s -> %s" %
(entry.data.battery_status, data.battery_status))
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
data.type = type_flags_to_enum(entry.data).value
# TODO: for ios, the confidence is currently a string "high/medium/low".
# should we convert it to a number to be consistent with the android version?
# or should we leave it unchanged?
# Let us convert it so a number so that we can try to reuse the same code
# as android
if 'confidence' in entry.data and 'confidence_level' not in entry.data:
data.confidence_level = entry.data.confidence
data.confidence = level_to_number(data.confidence_level)
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
#logging.info('*** Motion Data write_ts: %d' % metadata.write_ts)
if 'ts' not in entry.data:
# old style entries
data = ad.AttrDict()
else:
data = entry.data
if 'agb' in entry.data:
data.type = ecwa.MotionTypes(entry.data.agb).value
elif 'zzaEg' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzaEg).value
elif 'zzbjA' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzbjA).value
elif 'ajO' in entry.data:
data.type = ecwa.MotionTypes(entry.data.ajO).value
elif 'zzaKM' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzaKM).value
elif 'zzbhB' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzbhB).value
if 'agc' in entry.data:
data.confidence = entry.data.agc
elif 'zzaEh' in entry.data:
data.confidence = entry.data.zzaEh
elif 'zzbjB' in entry.data:
data.confidence = entry.data.zzbjB
elif 'ajP' in entry.data:
data.confidence = entry.data.ajP
elif 'zzaKN' in entry.data:
data.confidence = entry.data.zzaKN
elif 'zzbhC' in entry.data:
data.confidence = entry.data.zzbhC
if 'ts' not in entry.data:
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = entry
metadata = entry.metadata
try:
valid_tz = pytz.timezone(entry.metadata.time_zone)
except pytz.UnknownTimeZoneError as e:
logging.warning("Got error %s while checking format validity" % e)
# Default timezone in for the Bay Area, which is probably a fairly safe
# assumption for now
metadata.time_zone = "America/Los_Angeles"
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
formatted_entry.data = entry.data
return formatted_entry
def format(entry):
formatted_entry = entry
metadata = entry.metadata
try:
valid_tz = pytz.timezone(entry.metadata.time_zone)
except pytz.UnknownTimeZoneError as e:
logging.warning("Got error %s while checking format validity" % e)
# Default timezone in for the Bay Area, which is probably a fairly safe
# assumption for now
metadata.time_zone = "America/Los_Angeles"
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
formatted_entry.data = entry.data
return formatted_entry;
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
#logging.info('*** Motion Data write_ts: %d' % metadata.write_ts)
data = ad.AttrDict()
if 'agb' in entry.data:
data.type = ecwa.MotionTypes(entry.data.agb).value
elif 'zzaEg' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzaEg).value
elif 'zzbjA' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzbjA).value
elif 'ajO' in entry.data:
data.type = ecwa.MotionTypes(entry.data.ajO).value
elif 'zzaKM' in entry.data:
data.type = ecwa.MotionTypes(entry.data.zzaKM).value
else:
data.type = ecwa.MotionTypes(entry.data.zzbhB).value
if 'agc' in entry.data:
data.confidence = entry.data.agc
elif 'zzaEh' in entry.data:
data.confidence = entry.data.zzaEh
elif 'zzbjB' in entry.data:
data.confidence = entry.data.zzbjB
elif 'ajP' in entry.data:
data.confidence = entry.data.ajP
elif 'zzaKN' in entry.data:
data.confidence = entry.data.zzaKN
else:
data.confidence = entry.data.zzbhC
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" % (entry.metadata.write_ts, metadata.write_ts))
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" % (entry.metadata.write_ts, metadata.write_ts))
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
local_aware_dt = pydt.datetime.utcfromtimestamp(data.ts).replace(tzinfo=pytz.utc) \
.astimezone(pytz.timezone(formatted_entry.metadata.time_zone))
data.local_dt = local_aware_dt.replace(tzinfo=None)
data.fmt_time = local_aware_dt.isoformat()
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
logging.debug("Timestamp conversion: %s -> %s done" %
(entry.metadata.write_ts, metadata.write_ts))
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
fc.expand_data_times(data, metadata)
data.local_dt = ecsdlq.get_local_date(data.ts, metadata.time_zone)
data.fmt_time = arrow.get(data.ts).to(metadata.time_zone).isoformat()
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
local_aware_dt = pydt.datetime.utcfromtimestamp(data.ts).replace(tzinfo=pytz.utc) \
.astimezone(pytz.timezone(formatted_entry.metadata.time_zone))
data.local_dt = local_aware_dt.replace(tzinfo=None)
data.fmt_time = local_aware_dt.isoformat()
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def old2new(old_style_data):
import emission.core.wrapper.entry as ecwe
import emission.core.wrapper.statsevent as ecws
import emission.core.wrapper.battery as ecwb
none2None = lambda s: None if s == 'none' else s
float_with_none = lambda s: float(s) if s is not None else None
ms_to_sec_with_none = lambda s: (float(s)) / 1000 if type(
s) == str or type(s) == unicode else float(s)
user_id = old_style_data["user"]
del old_style_data["user"]
if old_style_data["stat"] == "battery_level":
new_style_data = ecwb.Battery({
"battery_level_pct":
float_with_none(none2None(old_style_data["reading"])),
"ts":
ms_to_sec_with_none(old_style_data["ts"])
})
new_key = "background/battery"
else:
new_style_data = ecws.Statsevent()
new_style_data.name = old_style_data["stat"]
new_style_data.ts = ms_to_sec_with_none(old_style_data["ts"])
new_style_data.reading = float_with_none(
none2None(old_style_data["reading"]))
new_style_data.client_app_version = old_style_data[
"client_app_version"]
new_style_data.client_os_version = old_style_data["client_os_version"]
new_key = stat2key(old_style_data["stat"])
new_entry = ecwe.Entry.create_entry(user_id, new_key, new_style_data)
# For legacy entries, make sure that the write_ts doesn't become the conversion
# time or the server arrival time
new_entry["metadata"]["write_ts"] = float_with_none(
old_style_data["reported_ts"])
del new_entry["metadata"]["write_local_dt"]
del new_entry["metadata"]["write_fmt_time"]
enufc.expand_metadata_times(new_entry["metadata"])
return new_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
m = entry.metadata
if "time_zone" not in m:
m.time_zone = "America/Los_Angeles"
m.write_ts = float(entry.metadata.write_ts) / 1000
logging.debug("Timestamp conversion: %s -> %s done" % (entry.metadata.write_ts, m.write_ts))
fc.expand_metadata_times(m)
formatted_entry.metadata = m
data = ad.AttrDict()
data.curr_state = state_map[entry.data.currState].value
logging.debug("Mapped %s -> %s" % (entry.data.currState, data.curr_state))
data.transition = transition_map[entry.data.transition].value
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
# ios sets data in a ratio between 0 and 1, so let's convert to percent to be consistent
# with android
data.battery_level_pct = entry.data.battery_level_ratio * 100
data.battery_status = status_map[entry.data.battery_status].value
logging.debug("Mapped %s -> %s" % (entry.data.battery_status, data.battery_status))
data.ts = formatted_entry.metadata.write_ts
data.local_dt = formatted_entry.metadata.write_local_dt
data.fmt_time = formatted_entry.metadata.write_fmt_time
formatted_entry.data = data
return formatted_entry
def format_location_simple(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
if "time_zone" not in metadata:
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = float(entry.metadata.write_ts) / 1000
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = entry.data
data.ts = data.ts / 1000 # convert from ms to seconds
local_aware_dt = pydt.datetime.utcfromtimestamp(data.ts).replace(tzinfo=pytz.utc) \
.astimezone(pytz.timezone(formatted_entry.metadata.time_zone))
data.local_dt = local_aware_dt.replace(tzinfo=None)
data.fmt_time = local_aware_dt.isoformat()
data.loc = geojson.Point((data.longitude, data.latitude))
data.heading = entry.data.bearing
del data.bearing
formatted_entry.data = data
return formatted_entry
def format_location_raw(entry):
formatted_entry = ad.AttrDict()
formatted_entry["_id"] = entry["_id"]
formatted_entry.user_id = entry.user_id
metadata = entry.metadata
metadata.time_zone = "America/Los_Angeles"
metadata.write_ts = old_div(float(entry.metadata.write_ts), 1000)
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
data = ad.AttrDict()
data.latitude = entry.data.mLatitude
data.longitude = entry.data.mLongitude
data.loc = geojson.Point((data.longitude, data.latitude))
data.ts = old_div(float(entry.data.mTime), 1000) # convert the ms from the phone to secs
fc.expand_data_times(data, metadata)
data.altitude = entry.data.mAltitude
data.accuracy = entry.data.mAccuracy
data.sensed_speed = entry.data.mSpeed
data.heading = entry.data.mBearing
formatted_entry.data = data
return formatted_entry
import logging
import pytz
import attrdict as ad
import copy
import emission.core.wrapper.sensorconfig as ecws
import emission.net.usercache.formatters.common as fc
# Currently, we just reflect this back to the user, so not much editing to do
# here. Since we get the timezone from javascript guessing, though, let's just
# verify that it is correct.
def format(entry):
formatted_entry = entry
metadata = entry.metadata
try:
valid_tz = pytz.timezone(entry.metadata.time_zone)
except UnknownTimeZoneError, e:
logging.warn("Got error %s while checking format validity" % e)
# Default timezone in for the Bay Area, which is probably a fairly safe
# assumption for now
metadata.time_zone = "America/Los_Angeles"
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
formatted_entry.data = entry.data
return formatted_entry;
import logging
import pytz
import attrdict as ad
import copy
import emission.core.wrapper.sensorconfig as ecws
import emission.net.usercache.formatters.common as fc
# Currently, we just reflect this back to the user, so not much editing to do
# here. Since we get the timezone from javascript guessing, though, let's just
# verify that it is correct.
def format(entry):
formatted_entry = entry
metadata = entry.metadata
try:
valid_tz = pytz.timezone(entry.metadata.time_zone)
except pytz.UnknownTimeZoneError, e:
logging.warn("Got error %s while checking format validity" % e)
# Default timezone in for the Bay Area, which is probably a fairly safe
# assumption for now
metadata.time_zone = "America/Los_Angeles"
# adds the python datetime and fmt_time entries. important for future searches!
fc.expand_metadata_times(metadata)
formatted_entry.metadata = metadata
formatted_entry.data = entry.data
return formatted_entry