def process(self, db_params):
"""Process database data for an activity into a an XML tree in TCX format."""
garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
with garmin_act_db.managed_session() as garmin_act_db_session:
activity = GarminDB.Activities.s_get(garmin_act_db_session,
self.activity_id)
self.tcx = GarminDbTcx()
self.tcx.create(activity.sport, activity.start_time)
laps = GarminDB.ActivityLaps.s_get_activity(
garmin_act_db_session, self.activity_id)
records = GarminDB.ActivityRecords.s_get_activity(
garmin_act_db_session, self.activity_id)
for lap in laps:
distance = Distance.from_meters_or_feet(
lap.distance, self.measurement_system)
track = self.tcx.add_lap(lap.start_time, lap.stop_time,
distance, lap.calories)
for record in records:
if record.timestamp >= lap.start_time and record.timestamp <= lap.stop_time:
alititude = Distance.from_meters_or_feet(
record.altitude, self.measurement_system)
speed = Speed.from_kph_or_mph(record.speed,
self.measurement_system)
self.tcx.add_point(track, record.timestamp,
record.position, alititude,
record.hr, speed)
garmindb = GarminDB.GarminDB(db_params)
with garmindb.managed_session() as garmin_db_session:
file = GarminDB.File.s_get(garmin_db_session, self.activity_id)
device = GarminDB.Device.s_get(garmin_db_session,
file.serial_number)
self.tcx.add_creator(device.product, file.serial_number)
def __init__(self, db_params_dict, input_file, input_dir, latest,
english_units, debug):
GarminJsonData.__init__(self, input_file, input_dir,
'activity_details_\\d*\.json', latest,
english_units, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict,
self.debug - 1)
def setUpClass(cls):
cls.garmin_act_db = GarminDB.ActivitiesDB(GarminDBConfigManager.get_db_params())
table_dict = {
'activities_table' : GarminDB.Activities,
'activity_laps_table' : GarminDB.ActivityLaps,
'activity_records_table' : GarminDB.ActivityRecords,
'run_activities_table' : GarminDB.StepsActivities,
'paddle_activities_table' : GarminDB.PaddleActivities,
'cycle_activities_table' : GarminDB.CycleActivities,
'elliptical_activities_table' : GarminDB.EllipticalActivities
}
super().setUpClass(cls.garmin_act_db, table_dict, {GarminDB.Activities : [GarminDB.Activities.name]})
cls.test_db_params = GarminDBConfigManager.get_db_params(test_db=True)
cls.test_mon_db = GarminDB.GarminDB(cls.test_db_params)
cls.test_act_db = GarminDB.ActivitiesDB(cls.test_db_params)
cls.measurement_system = Fit.field_enums.DisplayMeasure.statute
def __init__(self, db_params_dict, debug):
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params_dict, debug)
self.garmin_sum_db = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sum_db = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.measurement_system = GarminDB.Attributes.measurements_type(
self.garmin_db)
def __init__(self, db_params_dict, debug):
self.garmindb = GarminDB.GarminDB(db_params_dict, debug)
self.mondb = GarminDB.MonitoringDB(db_params_dict, debug)
self.garminsumdb = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sumdb = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.english_units = (GarminDB.Attributes.get(
self.garmindb, 'dist_setting') == 'statute')
def test_summary_json_file_import(self):
db_params_dict = GarminDBConfigManager.get_db_params(test_db=True)
gjsd = GarminJsonSummaryData(db_params_dict, 'test_files/json/activity/summary', latest=False, measurement_system=Fit.field_enums.DisplayMeasure.statute, debug=2)
if gjsd.file_count() > 0:
gjsd.process()
self.check_not_none_cols(GarminDB.ActivitiesDB(db_params_dict),
{GarminDB.Activities : [GarminDB.Activities.name, GarminDB.Activities.type, GarminDB.Activities.sport, GarminDB.Activities.sub_sport]}
)
def __init__(self, db_params_dict, debug):
"""Return an instance of the Analyze class."""
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params_dict, debug)
self.garmin_sum_db = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sum_db = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.measurement_system = GarminDB.Attributes.measurements_type(self.garmin_db)
def __init__(self, db_params_dict, debug):
self.garmindb = GarminDB.GarminDB(db_params_dict, debug)
self.mondb = GarminDB.MonitoringDB(db_params_dict, debug)
self.garminsumdb = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sumdb = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.english_units = (GarminDB.Attributes.measurements_type_metric(
self.garmindb) == False)
def test_fit_file_import(self):
db_params_dict = GarminDBConfigManager.get_db_params(test_db=True)
self.profile_function('fit_activities_import', self.fit_file_import, db_params_dict)
test_mon_db = GarminDB.GarminDB(db_params_dict)
self.check_db_tables_exists(test_mon_db, {'device_table' : GarminDB.Device})
self.check_db_tables_exists(test_mon_db, {'file_table' : GarminDB.File, 'device_info_table' : GarminDB.DeviceInfo}, self.gfd_file_count)
self.check_not_none_cols(GarminDB.ActivitiesDB(db_params_dict),
{GarminDB.Activities : [GarminDB.Activities.start_time, GarminDB.Activities.stop_time, GarminDB.Activities.elapsed_time]}
)
def process_files(self, db_params_dict):
"""Import data from files into the databse."""
garmin_db = GarminDB.GarminDB(db_params_dict, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug)
with garmin_db.managed_session() as self.garmin_db_session:
with garmin_act_db.managed_session() as self.garmin_act_db_session:
for file_name in progressbar.progressbar(self.file_names):
self.__process_file(file_name)
self.garmin_db_session.commit()
self.garmin_act_db_session.commit()
def __init__(self, db_params_dict, debug):
logger.info("Debug: %s", str(debug))
self.db_params_dict = db_params_dict
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(self.db_params_dict,
self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict,
self.debug - 1)
def test_garmin_act_db_tables_exists(self):
garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict)
self.assertGreater(GarminDB.Activities.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.ActivityLaps.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.ActivityRecords.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.RunActivities.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.WalkActivities.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.PaddleActivities.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.CycleActivities.row_count(garmin_act_db), 0)
self.assertGreater(GarminDB.EllipticalActivities.row_count(garmin_act_db), 0)
def activity_course(self, course_id):
"""Run a checkup on all activities matching the course_id."""
activity_db = GarminDB.ActivitiesDB(self.db_params, self.debug)
activities = GarminDB.Activities.get_by_course_id(activity_db, course_id)
activities_count = len(activities)
fastest_activity = GarminDB.Activities.get_fastest_by_course_id(activity_db, course_id)
slowest_activity = GarminDB.Activities.get_slowest_by_course_id(activity_db, course_id)
logger.info('Matching Activities: %d', activities_count)
logger.info(' first: %s', self.__activity_string(activity_db, activities[0]))
logger.info(' lastest: %s', self.__activity_string(activity_db, activities[-1]))
logger.info(' fastest: %s', self.__activity_string(activity_db, fastest_activity))
logger.info(' slowest: %s', self.__activity_string(activity_db, slowest_activity))
def process_files(self, db_params):
"""Import data from TCX files into the database."""
garmin_db = GarminDB.GarminDB(db_params, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
with garmin_db.managed_session() as self.garmin_db_session, garmin_act_db.managed_session() as self.garmin_act_db_session:
for file_name in tqdm(self.file_names, unit='files'):
try:
self.__process_file(file_name)
except Exception as e:
logger.error('Failed to processes file %s: %s', file_name, e)
self.garmin_db_session.commit()
self.garmin_act_db_session.commit()
def write_file(self, fit_file):
"""Given a Fit File object, write all of its messages to the DB."""
self.activity_file_plugins = [plugin for plugin in self.plugin_manager.get_activity_file_processors(fit_file).values()]
if len(self.activity_file_plugins):
root_logger.info("Loaded %d activity plugins %r for file %s", len(self.activity_file_plugins), self.activity_file_plugins, fit_file)
# Create the db after setting up the plugins so that plugin tables are handled properly
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params, self.debug - 1)
with self.garmin_db.managed_session() as self.garmin_db_session, self.garmin_act_db.managed_session() as self.garmin_act_db_session:
self._write_message_types(fit_file, fit_file.message_types)
# Now write a file's worth of data to the DB
self.garmin_act_db_session.commit()
self.garmin_db_session.commit()
def setUpClass(cls):
db_params_dict = GarminDBConfigManager.get_db_params()
cls.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict)
super(TestActivitiesDb, cls).setUpClass(
cls.garmin_act_db, {
'activities_table': GarminDB.Activities,
'activity_laps_table': GarminDB.ActivityLaps,
'activity_records_table': GarminDB.ActivityRecords,
'run_activities_table': GarminDB.StepsActivities,
'paddle_activities_table': GarminDB.PaddleActivities,
'cycle_activities_table': GarminDB.CycleActivities,
'elliptical_activities_table': GarminDB.EllipticalActivities
}, {GarminDB.Activities: [GarminDB.Activities.name]})
def __init__(self, db_params, debug):
"""
Return a new FitFileProcessor instance.
Paramters:
db_params (dict): database access configuration
debug (Boolean): if True, debug logging is enabled
"""
root_logger.info("Debug: %s", debug)
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params, self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
def __init__(self, db_params_dict, english_units, debug):
self.db_params_dict = db_params_dict
self.english_units = english_units
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(self.db_params_dict, self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict, self.debug - 1)
if english_units:
GarminDB.Attributes.set_newer(self.garmin_db, 'dist_setting', 'statute')
else:
GarminDB.Attributes.set_newer(self.garmin_db, 'dist_setting', 'metric')
logger.info("Debug: %s English units: %s" % (str(debug), str(english_units)))
def __init__(self, db_params_dict, input_dir, latest, debug):
"""
Return an instance of GarminActivitiesExtraData.
Parameters:
db_params_dict (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for data files
latest (Boolean): check for latest files only
measurement_system (enum): which measurement system to use when importing the files
debug (Boolean): enable debug logging
"""
logger.info("Processing activities extra data")
super(GarminActivitiesExtraData, self).__init__(None, input_dir, r'extra_data_\d*\.json', latest, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug - 1)
self.conversions = {}
def __init__(self, db_params_dict, input_dir, latest, measurement_system, debug):
"""
Return an instance of GarminTcxData.
Parameters:
db_params_dict (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for data files
latest (Boolean): check for latest files only
measurement_system (enum): which measurement system to use when importing the files
debug (Boolean): enable debug logging
"""
logger.info("Processing %s activities summary data from %s", 'latest' if latest else 'all', input_dir)
super(GarminJsonSummaryData, self).__init__(None, input_dir, r'activity_\d*\.json', latest, debug)
self.input_dir = input_dir
self.measurement_system = measurement_system
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug - 1)
self.conversions = {}
def __init__(self, db_params, input_dir, latest, measurement_system, debug):
"""
Return an instance of GarminJsonDetailsData.
Parameters:
----------
db_params (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for data files
latest (Boolean): check for latest files only
measurement_system (enum): which measurement system to use when importing the files
debug (Boolean): enable debug logging
"""
logger.info("Processing activities detail data")
super().__init__(r'activity_details_\d*\.json', input_dir=input_dir, latest=latest, debug=debug)
self.measurement_system = measurement_system
self.garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
self.conversions = {}
def __init__(self, db_params, ignore_dev_fields, debug):
"""
Return a new FitFileProcessor instance.
Paramters:
db_params (dict): database access configuration
ignore_dev_fields (Boolean): If True, then ignore develoepr fields in Fit files
debug (Boolean): if True, debug logging is enabled
"""
root_logger.info("Ignore dev fields: %s Debug: %s", ignore_dev_fields,
debug)
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params, self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
self.ignore_dev_fields = ignore_dev_fields
if not self.ignore_dev_fields:
self.field_prefixes = ['dev_', '']
else:
self.field_prefixes = ['']
def __init__(self, db_params_dict, english_units, debug):
self.db_params_dict = db_params_dict
self.english_units = english_units
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(self.db_params_dict,
self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict,
self.debug - 1)
if english_units:
GarminDB.Attributes.set_newer(
self.garmin_db, 'dist_setting',
str(Fit.FieldEnums.DisplayMeasure.statute))
else:
GarminDB.Attributes.set_newer(
self.garmin_db, 'dist_setting',
str(Fit.FieldEnums.DisplayMeasure.metric))
logger.info("Debug: %s English units: %s", str(debug),
str(english_units))
def test_tcx_file_import(self):
db_params_dict = GarminDBConfigManager.get_db_params(test_db=True)
gtd = GarminTcxData('test_files/tcx', latest=False, measurement_system=Fit.field_enums.DisplayMeasure.statute, debug=2)
if gtd.file_count() > 0:
gtd.process_files(db_params_dict)
self.check_not_none_cols(GarminDB.ActivitiesDB(db_params_dict), {GarminDB.Activities : [GarminDB.Activities.sport, GarminDB.Activities.laps]})
def process_files(self, db_params_dict):
garmin_db = GarminDB.GarminDB(db_params_dict, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug)
for file_name in self.file_names:
logger.info("Processing file: " + file_name)
tcx = tcxparser.TCXParser(file_name)
end_time = dateutil.parser.parse(tcx.completed_at, ignoretz=True)
start_time = dateutil.parser.parse(tcx.started_at, ignoretz=True)
manufacturer = 'Unknown'
product = tcx.creator
if product is not None:
match = re.search('Microsoft', product)
if match:
manufacturer = 'Microsoft'
serial_number = tcx.creator_version
if serial_number is None or serial_number == 0:
serial_number = GarminDB.Device.unknown_device_serial_number
device = {
'serial_number': serial_number,
'timestamp': start_time,
'manufacturer': manufacturer,
'product': product,
'hardware_version': None,
}
GarminDB.Device.create_or_update_not_none(garmin_db, device)
file = {
'name': file_name,
'type': 'tcx',
'serial_number': serial_number,
}
GarminDB.File.find_or_create(garmin_db, file)
activity_id = GarminDB.File.get(garmin_db, file_name)
if self.english_units and tcx.distance_units == 'meters':
distance = Fit.Conversions.meters_to_miles(tcx.distance)
ascent = Fit.Conversions.meters_to_feet(tcx.ascent)
descent = Fit.Conversions.meters_to_feet(tcx.descent)
else:
distance = tcx.distance / 1000.0
ascent = tcx.ascent / 1000.0
descent = tcx.descent / 1000.0
activity = {
'activity_id': activity_id,
'start_time': start_time,
'stop_time': end_time,
'laps': len(tcx.activity.Lap),
# 'sport' : tcx.activity_type,
'start_lat': tcx.start_latitude,
'start_long': tcx.start_longitude,
'stop_lat': tcx.end_latitude,
'stop_long': tcx.end_longitude,
'distance': distance,
'avg_hr': tcx.hr_avg,
'max_hr': tcx.hr_max,
'calories': tcx.calories,
'max_cadence': tcx.cadence_max,
'avg_cadence': tcx.cadence_avg,
#'ascent' : ascent,
#'descent' : descent
}
activity_not_zero = {
key: value
for (key, value) in activity.iteritems() if value
}
print repr(activity_not_zero)
GarminDB.Activities.create_or_update_not_none(
garmin_act_db, activity_not_zero)
def process_files(self, db_params_dict):
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict,
self.debug - 1)
for file_name in self.file_names:
json_data = json.load(open(file_name))
activity_id = json_data['activityId']
sub_sport = json_data['activityType']['typeKey']
distance_in_meters = self.get_garmin_json_data(
json_data, 'distance', float)
ascent_in_meters = self.get_garmin_json_data(
json_data, 'elevationGain', float)
descent_in_meters = self.get_garmin_json_data(
json_data, 'elevationLoss', float)
avg_speed_mps = self.get_garmin_json_data(json_data,
'averageSpeed', float)
max_speed_mps = self.get_garmin_json_data(json_data, 'maxSpeed',
float)
max_temperature_c = self.get_garmin_json_data(
json_data, 'maxTemperature', float)
min_temperature_c = self.get_garmin_json_data(
json_data, 'minTemperature', float)
if self.english_units:
distance = Fit.Conversions.meters_to_miles(distance_in_meters)
ascent = Fit.Conversions.meters_to_feet(ascent_in_meters)
descent = Fit.Conversions.meters_to_feet(descent_in_meters)
avg_speed = Fit.Conversions.mps_to_mph(avg_speed_mps)
max_speed = Fit.Conversions.mps_to_mph(max_speed_mps)
max_temperature = Fit.Conversions.celsius_to_fahrenheit(
max_temperature_c)
min_temperature = Fit.Conversions.celsius_to_fahrenheit(
min_temperature_c)
else:
distance = distance_in_meters / 1000.0
ascent = ascent_in_meters
descent = descent_in_meters
avg_speed = avg_speed_mps
max_speed = max_speed_mps
max_temperature = max_temperature_c
min_temperature = min_temperature_c
sport = self.get_garmin_json_data(json_data['activityType'],
'parentTypeId')
sport_mappings = {
1: 'running',
2: 'cycling',
3: 'hiking',
4: 'other',
5: 'mountain_biking',
9: 'walking',
17: sub_sport,
29: 'fitness_equipment',
30: 'elliptical'
}
sport_name = sport_mappings.get(sport, None)
activity = {
'activity_id':
activity_id,
'name':
json_data['activityName'],
'description':
self.get_garmin_json_data(json_data, 'description'),
'type':
self.get_garmin_json_data(json_data['eventType'], 'typeKey'),
'sport':
sport_name,
'sub_sport':
sub_sport,
'start_time':
dateutil.parser.parse(self.get_garmin_json_data(
json_data, 'startTimeLocal'),
ignoretz=True),
#'stop_time' : dateutil.parser.parse(self.get_garmin_json_data(json_data, 'EndTimestamp', ignoretz=True)),
'elapsed_time':
Fit.Conversions.secs_to_dt_time(
self.get_garmin_json_data(json_data, 'elapsedDuration',
int)),
'moving_time':
Fit.Conversions.secs_to_dt_time(
self.get_garmin_json_data(json_data, 'movingDuration',
int)),
'start_lat':
self.get_garmin_json_data(json_data, 'startLatitude', float),
'start_long':
self.get_garmin_json_data(json_data, 'startLongitude', float),
'stop_lat':
self.get_garmin_json_data(json_data, 'endLatitude', float),
'stop_long':
self.get_garmin_json_data(json_data, 'endLongitude', float),
'distance':
distance,
'laps':
self.get_garmin_json_data(json_data, 'lapCount'),
'avg_hr':
self.get_garmin_json_data(json_data, 'averageHR', float),
'max_hr':
self.get_garmin_json_data(json_data, 'maxHR', float),
'calories':
self.get_garmin_json_data(json_data, 'calories', float),
'avg_speed':
avg_speed,
#'avg_moving_speed' : self.get_garmin_json_data(json_data, 'WeightedMeanMovingSpeed', float),
'max_speed':
max_speed,
'ascent':
ascent,
'descent':
descent,
'max_temperature':
max_temperature,
'min_temperature':
min_temperature,
#'avg_temperature' : self.get_garmin_json_data(json_data, 'WeightedMeanAirTemperature', float),
'training_effect':
self.get_garmin_json_data(json_data, 'aerobicTrainingEffect',
float),
'anaerobic_training_effect':
self.get_garmin_json_data(json_data, 'anaerobicTrainingEffect',
float),
}
GarminDB.Activities.create_or_update_not_none(
self.garmin_act_db, activity)
try:
function = getattr(self, 'process_' + sub_sport)
function(activity_id, json_data)
except AttributeError:
logger.info("No sport handler for type %s from %s" %
(sub_sport, activity_id))
def test_garmin_act_db_exists(self):
garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict)
self.assertIsNotNone(garmin_act_db)