def download_data(overwite, latest, stats):
"""Download selected activity types from Garmin Connect and save the data in files. Overwrite previously downloaded data if indicated."""
logger.info("___Downloading %s Data___", 'Latest' if latest else 'All')
db_params_dict = GarminDBConfigManager.get_db_params()
download = Download()
if not download.login():
logger.error("Failed to login!")
sys.exit()
if Statistics.activities in stats:
if latest:
activity_count = gc_config.latest_activity_count()
else:
activity_count = gc_config.all_activity_count()
activities_dir = GarminDBConfigManager.get_or_create_activities_dir()
root_logger.info("Fetching %d activities to %s", activity_count, activities_dir)
download.get_activity_types(activities_dir, overwite)
download.get_activities(activities_dir, activity_count, overwite)
if Statistics.monitoring in stats:
date, days = __get_date_and_days(GarminDB.MonitoringDB(db_params_dict), latest, GarminDB.MonitoringHeartRate, GarminDB.MonitoringHeartRate.heart_rate, 'monitoring')
if days > 0:
root_logger.info("Date range to update: %s (%d) to %s", date, days, GarminDBConfigManager.get_monitoring_base_dir())
download.get_daily_summaries(GarminDBConfigManager.get_or_create_monitoring_dir, date, days, overwite)
download.get_hydration(GarminDBConfigManager.get_or_create_monitoring_dir, date, days, overwite)
download.get_monitoring(GarminDBConfigManager.get_or_create_monitoring_dir, date, days)
root_logger.info("Saved monitoring files for %s (%d) to %s for processing", date, days, GarminDBConfigManager.get_monitoring_base_dir())
if Statistics.sleep in stats:
date, days = __get_date_and_days(GarminDB.GarminDB(db_params_dict), latest, GarminDB.Sleep, GarminDB.Sleep.total_sleep, 'sleep')
if days > 0:
sleep_dir = GarminDBConfigManager.get_or_create_sleep_dir()
root_logger.info("Date range to update: %s (%d) to %s", date, days, sleep_dir)
download.get_sleep(sleep_dir, date, days, overwite)
root_logger.info("Saved sleep files for %s (%d) to %s for processing", date, days, sleep_dir)
if Statistics.weight in stats:
date, days = __get_date_and_days(GarminDB.GarminDB(db_params_dict), latest, GarminDB.Weight, GarminDB.Weight.weight, 'weight')
if days > 0:
weight_dir = GarminDBConfigManager.get_or_create_weight_dir()
root_logger.info("Date range to update: %s (%d) to %s", date, days, weight_dir)
download.get_weight(weight_dir, date, days, overwite)
root_logger.info("Saved weight files for %s (%d) to %s for processing", date, days, weight_dir)
if Statistics.rhr in stats:
date, days = __get_date_and_days(GarminDB.GarminDB(db_params_dict), latest, GarminDB.RestingHeartRate, GarminDB.RestingHeartRate.resting_heart_rate, 'rhr')
if days > 0:
rhr_dir = GarminDBConfigManager.get_or_create_rhr_dir()
root_logger.info("Date range to update: %s (%d) to %s", date, days, rhr_dir)
download.get_rhr(rhr_dir, date, days, overwite)
root_logger.info("Saved rhr files for %s (%d) to %s for processing", date, days, rhr_dir)
def process(self, db_params):
"""Process database data for an activity into a an XML tree in TCX format."""
garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
with garmin_act_db.managed_session() as garmin_act_db_session:
activity = GarminDB.Activities.s_get(garmin_act_db_session,
self.activity_id)
self.tcx = GarminDbTcx()
self.tcx.create(activity.sport, activity.start_time)
laps = GarminDB.ActivityLaps.s_get_activity(
garmin_act_db_session, self.activity_id)
records = GarminDB.ActivityRecords.s_get_activity(
garmin_act_db_session, self.activity_id)
for lap in laps:
distance = Distance.from_meters_or_feet(
lap.distance, self.measurement_system)
track = self.tcx.add_lap(lap.start_time, lap.stop_time,
distance, lap.calories)
for record in records:
if record.timestamp >= lap.start_time and record.timestamp <= lap.stop_time:
alititude = Distance.from_meters_or_feet(
record.altitude, self.measurement_system)
speed = Speed.from_kph_or_mph(record.speed,
self.measurement_system)
self.tcx.add_point(track, record.timestamp,
record.position, alititude,
record.hr, speed)
garmindb = GarminDB.GarminDB(db_params)
with garmindb.managed_session() as garmin_db_session:
file = GarminDB.File.s_get(garmin_db_session, self.activity_id)
device = GarminDB.Device.s_get(garmin_db_session,
file.serial_number)
self.tcx.add_creator(device.product, file.serial_number)
def goals(self):
"""Do a checkup of th euser's goals."""
garmin_db = GarminDB.GarminDB(self.db_params_dict, self.debug)
look_back_days = GarminDBConfigManager.checkup('look_back_days')
end_ts = datetime.now()
start_ts = end_ts - timedelta(days=look_back_days)
results = GarminDB.DailySummary.get_for_period(garmin_db, start_ts,
end_ts)
step_goal_days = 0
floors_goal_days = 0
intensity_days = 0
intensity_weeks = 0
intensity_time_goal_percent = 0
intensity_goal_weeks = 0
for result in results:
if result.steps_goal_percent >= 100:
step_goal_days += 1
if result.floors_goal_percent >= 100:
floors_goal_days += 1
if result.day.weekday() == 0:
intensity_days = 0
intensity_time_goal_percent += result.intensity_time_goal_percent
intensity_days += 1
if result.day.weekday() == 6:
if intensity_days == 7:
intensity_weeks += 1
if intensity_time_goal_percent >= 100:
intensity_goal_weeks += 1
logger.info('Steps: on goal %d of %d days', step_goal_days,
look_back_days)
logger.info('Floors: on goal %d of %d days', floors_goal_days,
look_back_days)
logger.info('Intensity mins: on goal %d of %d weeks',
intensity_goal_weeks, intensity_weeks)
def __init__(self, db_params, input_dir, latest, debug):
"""
Return an instance of GarminSleepData.
Parameters:
----------
db_params (object): configuration data for accessing the database
input_dir (string): directory (full path) to check for sleep data files
latest (Boolean): check for latest files only
debug (Boolean): enable debug logging
"""
logger.info("Processing sleep data")
super().__init__(r'sleep_\d{4}-\d{2}-\d{2}\.json',
input_dir=input_dir,
latest=latest,
debug=debug)
self.garmin_db = GarminDB.GarminDB(db_params)
self.conversions = {
'calendarDate': dateutil.parser.parse,
'sleepTimeSeconds': Fit.conversions.secs_to_dt_time,
'sleepStartTimestampGMT': Fit.conversions.epoch_ms_to_dt,
'sleepEndTimestampGMT': Fit.conversions.epoch_ms_to_dt,
'deepSleepSeconds': Fit.conversions.secs_to_dt_time,
'lightSleepSeconds': Fit.conversions.secs_to_dt_time,
'remSleepSeconds': Fit.conversions.secs_to_dt_time,
'awakeSleepSeconds': Fit.conversions.secs_to_dt_time,
'startGMT': dateutil.parser.parse,
'endGMT': dateutil.parser.parse
}
def __init__(self, debug):
"""Return an instance of the CheckUp class."""
self.db_params = GarminDBConfigManager.get_db_params()
self.debug = debug
self.garmin_db = GarminDB.GarminDB(self.db_params)
self.measurement_system = GarminDB.Attributes.measurements_type(self.garmin_db)
self.unit_strings = Fit.units.unit_strings[self.measurement_system]
def __init__(self, db_params, input_dir, latest, measurement_system,
debug):
"""
Return an instance of GarminSummaryData.
Parameters:
----------
db_params (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for data files
latest (Boolean): check for latest files only
measurement_system (enum): which measurement system to use when importing the files
debug (Boolean): enable debug logging
"""
logger.info("Processing daily summary data")
super().__init__(r'daily_summary_\d{4}-\d{2}-\d{2}\.json',
input_dir=input_dir,
latest=latest,
debug=debug,
recursive=True)
self.input_dir = input_dir
self.measurement_system = measurement_system
self.garmin_db = GarminDB.GarminDB(db_params)
self.conversions = {
'calendarDate': dateutil.parser.parse,
'moderateIntensityMinutes': Fit.conversions.min_to_dt_time,
'vigorousIntensityMinutes': Fit.conversions.min_to_dt_time,
'intensityMinutesGoal': Fit.conversions.min_to_dt_time,
}
def __init__(self, db_params_dict, debug):
self.garmindb = GarminDB.GarminDB(db_params_dict, debug)
self.mondb = GarminDB.MonitoringDB(db_params_dict, debug)
self.garminsumdb = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sumdb = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.english_units = (GarminDB.Attributes.measurements_type_metric(
self.garmindb) == False)
def export_activity(debug, directory, export_activity_id):
"""Export an activity given its database id."""
db_params_dict = GarminDBConfigManager.get_db_params()
garmindb = GarminDB.GarminDB(db_params_dict)
measurement_system = GarminDB.Attributes.measurements_type(garmindb)
ae = ActivityExporter(directory, export_activity_id, measurement_system, debug)
ae.process(db_params_dict)
return ae.write('activity_%s.tcx' % export_activity_id)
def __init__(self, db_params_dict, debug):
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params_dict, debug)
self.garmin_sum_db = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sum_db = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.measurement_system = GarminDB.Attributes.measurements_type(
self.garmin_db)
def __init__(self, db_params_dict, debug):
"""Return an instance of the Analyze class."""
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params_dict, debug)
self.garmin_sum_db = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sum_db = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.measurement_system = GarminDB.Attributes.measurements_type(self.garmin_db)
def __init__(self, db_params_dict, debug):
self.garmindb = GarminDB.GarminDB(db_params_dict, debug)
self.mondb = GarminDB.MonitoringDB(db_params_dict, debug)
self.garminsumdb = GarminDB.GarminSummaryDB(db_params_dict, debug)
self.sumdb = HealthDB.SummaryDB(db_params_dict, debug)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, debug)
self.english_units = (GarminDB.Attributes.get(
self.garmindb, 'dist_setting') == 'statute')
def test_fit_file_import(self):
db_params = GarminDBConfigManager.get_db_params(test_db=True)
self.profile_function('fit_mon_import', self.fit_file_import, db_params)
test_mon_db = GarminDB.GarminDB(db_params)
self.check_db_tables_exists(test_mon_db, {'device_table' : GarminDB.Device})
self.check_db_tables_exists(test_mon_db, {'file_table' : GarminDB.File, 'device_info_table' : GarminDB.DeviceInfo}, self.gfd_file_count)
table_not_none_cols_dict = {GarminDB.Monitoring : [GarminDB.Monitoring.timestamp, GarminDB.Monitoring.activity_type, GarminDB.Monitoring.duration]}
self.check_not_none_cols(GarminDB.MonitoringDB(db_params), table_not_none_cols_dict)
def test_summary_json_file_import(self):
db_params = GarminDBConfigManager.get_db_params(test_db=True)
gjsd = GarminSummaryData(db_params, 'test_files/json/monitoring/summary', latest=False, measurement_system=Fit.field_enums.DisplayMeasure.statute, debug=2)
if gjsd.file_count() > 0:
gjsd.process()
table_not_none_cols_dict = {
GarminDB.DailySummary : [GarminDB.DailySummary.rhr, GarminDB.DailySummary.distance, GarminDB.DailySummary.steps, GarminDB.DailySummary.floors_goal]
}
self.check_not_none_cols(GarminDB.GarminDB(db_params), table_not_none_cols_dict)
def test_fit_file_import(self):
db_params_dict = GarminDBConfigManager.get_db_params(test_db=True)
self.profile_function('fit_activities_import', self.fit_file_import, db_params_dict)
test_mon_db = GarminDB.GarminDB(db_params_dict)
self.check_db_tables_exists(test_mon_db, {'device_table' : GarminDB.Device})
self.check_db_tables_exists(test_mon_db, {'file_table' : GarminDB.File, 'device_info_table' : GarminDB.DeviceInfo}, self.gfd_file_count)
self.check_not_none_cols(GarminDB.ActivitiesDB(db_params_dict),
{GarminDB.Activities : [GarminDB.Activities.start_time, GarminDB.Activities.stop_time, GarminDB.Activities.elapsed_time]}
)
def process_files(self, db_params_dict):
"""Import data from files into the databse."""
garmin_db = GarminDB.GarminDB(db_params_dict, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug)
with garmin_db.managed_session() as self.garmin_db_session:
with garmin_act_db.managed_session() as self.garmin_act_db_session:
for file_name in progressbar.progressbar(self.file_names):
self.__process_file(file_name)
self.garmin_db_session.commit()
self.garmin_act_db_session.commit()
def __init__(self, db_params_dict, debug):
logger.info("Debug: %s", str(debug))
self.db_params_dict = db_params_dict
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(self.db_params_dict,
self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict,
self.debug - 1)
def test_garmindb_tables_exists(self):
garmindb = GarminDB.GarminDB(self.db_params_dict)
self.assertGreater(GarminDB.Attributes.row_count(garmindb), 0)
self.assertGreater(GarminDB.Device.row_count(garmindb), 0)
self.assertGreater(GarminDB.DeviceInfo.row_count(garmindb), 0)
self.assertGreater(GarminDB.File.row_count(garmindb), 0)
self.assertGreater(GarminDB.Weight.row_count(garmindb), 0)
self.assertGreater(GarminDB.Stress.row_count(garmindb), 0)
self.assertGreater(GarminDB.Sleep.row_count(garmindb), 0)
self.assertGreater(GarminDB.SleepEvents.row_count(garmindb), 0)
self.assertGreater(GarminDB.RestingHeartRate.row_count(garmindb), 0)
def test_parse_uprofile(self):
db_params = GarminDBConfigManager.get_db_params(test_db=True)
gp = GarminProfile(db_params, self.file_path, debug=2)
if gp.file_count() > 0:
gp.process()
garmindb = GarminDB.GarminDB(db_params)
measurement_system = GarminDB.Attributes.measurements_type(garmindb)
self.assertEqual(
measurement_system, Fit.field_enums.DisplayMeasure.statute,
'DisplayMeasure expected %r found %r' %
(Fit.field_enums.DisplayMeasure.statute, measurement_system))
def process_files(self, db_params):
"""Import data from TCX files into the database."""
garmin_db = GarminDB.GarminDB(db_params, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
with garmin_db.managed_session() as self.garmin_db_session, garmin_act_db.managed_session() as self.garmin_act_db_session:
for file_name in tqdm(self.file_names, unit='files'):
try:
self.__process_file(file_name)
except Exception as e:
logger.error('Failed to processes file %s: %s', file_name, e)
self.garmin_db_session.commit()
self.garmin_act_db_session.commit()
def process_files(self, db_params_dict):
garmindb = GarminDB.GarminDB(db_params_dict)
for file_name in self.file_names:
json_data = parse_json_file(
file_name, {'calendarDate': dateutil.parser.parse})
for sample in json_data:
data = {
'day': sample['calendarDate'].date(),
'resting_heart_rate': sample['value']
}
GarminDB.RestingHeartRate.create_or_update_not_none(
garmindb, data)
def __init__(self, db_params, debug):
"""
Return a new FitFileProcessor instance.
Paramters:
db_params (dict): database access configuration
debug (Boolean): if True, debug logging is enabled
"""
root_logger.info("Debug: %s", debug)
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(db_params, self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(db_params, self.debug - 1)
def __init__(self, db_params_dict, english_units, debug):
self.db_params_dict = db_params_dict
self.english_units = english_units
self.debug = debug
self.garmin_db = GarminDB.GarminDB(db_params_dict, debug - 1)
self.garmin_mon_db = GarminDB.MonitoringDB(self.db_params_dict, self.debug - 1)
self.garmin_act_db = GarminDB.ActivitiesDB(self.db_params_dict, self.debug - 1)
if english_units:
GarminDB.Attributes.set_newer(self.garmin_db, 'dist_setting', 'statute')
else:
GarminDB.Attributes.set_newer(self.garmin_db, 'dist_setting', 'metric')
logger.info("Debug: %s English units: %s" % (str(debug), str(english_units)))
def test_garmindb_tables_bounds(self):
garmindb = GarminDB.GarminDB(self.db_params_dict)
self.check_col_stats(garmindb, GarminDB.Weight, GarminDB.Weight.weight, 'Weight', False, False,
(0, 10*365), (25, 300), (25, 300), (25, 300), (25, 300))
self.check_col_stats(garmindb, GarminDB.Stress, GarminDB.Stress.stress, 'Stress', True, False,
(1, 10000000), (25, 100), (0, 2), (0, 100), (0, 100))
self.check_col_stats(garmindb, GarminDB.RestingHeartRate, GarminDB.RestingHeartRate.resting_heart_rate, 'RHR', True, False,
(1, 10000000), (30, 100), (30, 100), (30, 100), (30, 100))
self.check_col_stats(garmindb, GarminDB.Sleep, GarminDB.Sleep.total_sleep, 'Sleep', True, True,
(1, 10000000), (datetime.time(8), datetime.time(12)), (datetime.time(0), datetime.time(4)),
(datetime.time(4), datetime.time(10)), (datetime.time(2), datetime.time(12)))
self.check_col_stats(garmindb, GarminDB.Sleep, GarminDB.Sleep.rem_sleep, 'REM Sleep', True, True,
(1, 10000000), (datetime.time(2), datetime.time(4)), (datetime.time(0), datetime.time(2)),
(datetime.time(1), datetime.time(6)), (datetime.time(2), datetime.time(6)))
def __init__(self, db_params, input_dir, debug):
"""
Return an instance of GarminProfile.
Parameters:
----------
db_params (object): configuration data for accessing the database
input_dir (string): directory (full path) to check for profile data files
debug (Boolean): enable debug logging
"""
logger.info("Processing profile data")
super().__init__(r'profile\.json', input_dir=input_dir, latest=False, debug=debug)
self.garmin_db = GarminDB.GarminDB(db_params)
self.conversions = {'calendarDate': self._parse_date}
def setUpClass(cls):
db_params = GarminDBConfigManager.get_db_params()
cls.garmin_db = GarminDB.GarminDB(db_params)
table_dict = {
'attributes_table' : GarminDB.Attributes,
'device_table' : GarminDB.Device,
'device_info_table' : GarminDB.DeviceInfo,
'file_table' : GarminDB.File,
'weight_table' : GarminDB.Weight,
'stress_table' : GarminDB.Stress,
'sleep_table' : GarminDB.Sleep,
'sleep_events_table' : GarminDB.SleepEvents,
'resting_heart_rate_table' : GarminDB.RestingHeartRate
}
super().setUpClass(cls.garmin_db, table_dict)
def __init__(self, db_params_dict, input_dir, debug):
"""
Return an instance of GarminProfile.
Parameters:
db_params_dict (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for profile data files
debug (Boolean): enable debug logging
"""
logger.info("Processing profile data")
super(GarminProfile, self).__init__(None, input_dir, r'profile\.json',
False, debug)
self.garmin_db = GarminDB.GarminDB(db_params_dict)
self.conversions = {'calendarDate': dateutil.parser.parse}
def process_files(self, db_params_dict):
garmindb = GarminDB.GarminDB(db_params_dict)
def json_parser(entry):
if 'calendarDate' in entry:
entry['calendarDate'] = dateutil.parser.parse(entry['calendarDate'])
return entry
for file_name in self.file_names:
json_data = json.load(open(file_name), object_hook=json_parser)
for sample in json_data:
data = {
'day' : sample['calendarDate'].date(),
'resting_heart_rate' : sample['value']
}
GarminDB.RestingHeartRate.create_or_update_not_none(garmindb, data)
logger.info("DB updated with %d rhr entries" % len(json_data))
def setUpClass(cls):
cls.garmin_act_db = GarminDB.ActivitiesDB(GarminDBConfigManager.get_db_params())
table_dict = {
'activities_table' : GarminDB.Activities,
'activity_laps_table' : GarminDB.ActivityLaps,
'activity_records_table' : GarminDB.ActivityRecords,
'run_activities_table' : GarminDB.StepsActivities,
'paddle_activities_table' : GarminDB.PaddleActivities,
'cycle_activities_table' : GarminDB.CycleActivities,
'elliptical_activities_table' : GarminDB.EllipticalActivities
}
super().setUpClass(cls.garmin_act_db, table_dict, {GarminDB.Activities : [GarminDB.Activities.name]})
cls.test_db_params = GarminDBConfigManager.get_db_params(test_db=True)
cls.test_mon_db = GarminDB.GarminDB(cls.test_db_params)
cls.test_act_db = GarminDB.ActivitiesDB(cls.test_db_params)
cls.measurement_system = Fit.field_enums.DisplayMeasure.statute
def __init__(self, db_params, input_dir, latest, debug):
"""
Return an instance of GarminRhrData.
Parameters:
----------
db_params (object): configuration data for accessing the database
input_dir (string): directory (full path) to check for resting heart rate data files
latest (Boolean): check for latest files only
debug (Boolean): enable debug logging
"""
logger.info("Processing rhr data")
super().__init__(r'rhr_\d{4}-\d{2}-\d{2}\.json', input_dir=input_dir, latest=latest, debug=debug)
self.garmin_db = GarminDB.GarminDB(db_params)
self.conversions = {'statisticsStartDate': self._parse_date}
def __init__(self, db_params_dict, input_dir, latest, debug):
"""
Return an instance of GarminActivitiesExtraData.
Parameters:
db_params_dict (dict): configuration data for accessing the database
input_dir (string): directory (full path) to check for data files
latest (Boolean): check for latest files only
measurement_system (enum): which measurement system to use when importing the files
debug (Boolean): enable debug logging
"""
logger.info("Processing activities extra data")
super(GarminActivitiesExtraData,
self).__init__(None, input_dir, r'extra_data_\d*\.json', latest,
debug)
self.garmin_db = GarminDB.GarminDB(db_params_dict)