def import_fit_data(self):
"""
Will look at the file location for our google fit files and loop though imported them by category. It will
ignore any walking activity to remove clutter. Then call a function to move any read files to the ./imported
folder
"""
categories = ['Running', 'Yoga', 'Weights', 'Wal']
for category in categories:
file_names = self.get_tcx_files(category)
for file in file_names:
if category is not 'Wal':
tcx = tcxparser.TCXParser(self._activitiesLocation + '\\' +
file)
date_str, duration, pace = self.convert_data(tcx)
name = category + " - " + date_str
notes = tcx.activity_notes
calories = tcx.calories
heart_rate = self.get_heart_rate(tcx)
distance = tcx.distance
fit_activity = Activity.Activity(category, name, date_str,
notes)
fit_activity.add_fit_entry(calories, heart_rate, pace,
duration, distance)
self.move_read_files(file)
def _parse(f: Path) -> Workout:
tcx = tcxparser.TCXParser(str(f))
sport = f.stem.split('_')[-1] # todo not sure how reliable...
hr_avg = tcx.hr_avg
distance_m = tcx.distance
duration_s = tcx.duration
# kmh to match endomondo.. should probably be CI
speed_avg_kmh = (distance_m / 1000) / (duration_s / 3600)
# eh. not sure if there is a better way
# for now use this to be compatible with Endomondo
# https://beepb00p.xyz/heartbeats_vs_kcals.html
# filtered for Endomondo running:
reg_coeff = 0.0993
intercept = -11.0739
total_beats = hr_avg * (duration_s / 60)
kcal_estimate = total_beats * reg_coeff + intercept
return {
'id': f.name, # not sure?
'start_time': isoparse(tcx.started_at),
'duration': timedelta(seconds=tcx.duration),
'sport': sport,
'heart_rate_avg': tcx.hr_avg,
'speed_avg': speed_avg_kmh,
'kcal': kcal_estimate,
}
def __process_file(self, file_name):
root_logger.info("Processing file: " + file_name)
tcx = tcxparser.TCXParser(file_name)
end_time = dateutil.parser.parse(tcx.completed_at, ignoretz=True)
start_time = dateutil.parser.parse(tcx.started_at, ignoretz=True)
manufacturer = GarminDB.Device.Manufacturer.Unknown
product = tcx.creator
if product is not None:
match = re.search('Microsoft', product)
if match:
manufacturer = GarminDB.Device.Manufacturer.Microsoft
serial_number = tcx.creator_version
if serial_number is None or serial_number == 0:
serial_number = GarminDB.Device.unknown_device_serial_number
device = {
'serial_number': serial_number,
'timestamp': start_time,
'manufacturer': manufacturer,
'product': product,
'hardware_version': None,
}
GarminDB.Device._create_or_update_not_none(self.garmin_db_session,
device)
(file_id, file_name) = GarminDB.File.name_and_id_from_path(file_name)
file = {
'id': file_id,
'name': file_name,
'type': GarminDB.File.FileType.tcx,
'serial_number': serial_number,
}
GarminDB.File._find_or_create(self.garmin_db_session, file)
distance = Fit.measurement.Distance.from_meters(tcx.distance)
# ascent = Fit.Distance.from_meters(tcx.ascent)
# descent = Fit.Distance.from_meters(tcx.descent)
activity = {
'activity_id': file_id,
'start_time': start_time,
'stop_time': end_time,
'laps': len(tcx.activity.Lap),
# 'sport' : tcx.activity_type,
'calories': tcx.calories,
'start_lat': tcx.start_latitude,
'start_long': tcx.start_longitude,
'stop_lat': tcx.end_latitude,
'stop_long': tcx.end_longitude,
'distance': distance.kms_or_miles(self.measurement_system),
'avg_hr': tcx.hr_avg,
'max_hr': tcx.hr_max,
'max_cadence': tcx.cadence_max,
'avg_cadence': tcx.cadence_avg,
# 'ascent' : ascent.meters_or_feet(self.measurement_system),
# 'descent' : descent.meters_or_feet(self.measurement_system)
}
activity_not_zero = {
key: value
for (key, value) in activity.iteritems() if value
}
GarminDB.Activities._create_or_update_not_none(
self.garmin_act_db_session, activity_not_zero)
def __init__(self, cpath=''):
FILEDIRECTORY = './data'
FILENAME = 'act1.tcx'
FILEPATH = os.path.join(FILEDIRECTORY, FILENAME)
if cpath:
FILEPATH = cpath
self.TCX_object = tp.TCXParser(FILEPATH)
self.dict_data = {}
self.getdata()
def __init__(self, path_to_tcx: Union[Path, str]):
self.path = Path(path_to_tcx)
if not self.path.exists():
raise FileNotFoundError(
f'Provided path {self.path} does not exist.')
elif self.path.suffix != '.tcx':
raise ValueError('Provided path should lead to a .tcx file.')
# needs string path for internal reasons
self.tcx = tcxparser.TCXParser(str(self.path))
def _get_ifit_workouts(workout_dir):
workouts = []
for workout_file in glob.glob(os.path.join(workout_dir, '*')):
tcx = tcxparser.TCXParser(workout_file)
workout = Workout(workout_id=os.path.basename(workout_file),
started_at=tcx.started_at,
duration=tcx.duration,
notes=tcx.activity_notes,
tcx_file=workout_file)
logging.debug(f"Workout: {workout}")
workouts.append(workout)
workouts.sort(key=lambda x: x.started_at)
return workouts
def parseActivity(act):
fn_key = 'filename'
date_key = 'date'
type_key = 'type'
distance_key = 'distance'
ascent_key = 'elevation'
if 'Filename' in act:
fn_key = 'Filename'
date_key = 'Activity Date'
type_key = 'Activity Type'
distance_key = 'Distance'
ascent_key = 'Elevation Gain'
if not act[fn_key]:
return None
if act[fn_key].endswith('gpx.gz'):
with skipWhitespace(gzip.open(act[fn_key], 'r')) as f:
gpx = gpxpy.parse(f)
ascent = calcGPXAscent(gpx)
elif act[fn_key].endswith('tcx.gz'):
with skipWhitespace(gzip.open(act[fn_key], 'r')) as f:
tcx = tcxparser.TCXParser(f)
ascent = tcx.ascent
elif act[fn_key].endswith('fit.gz'):
# TODO: parse fit.gz
if ascent_key in act:
ascent = float(act[ascent_key])
else:
return None
else:
raise Exception("Unknown activity file type")
distance = 0
distance_str = act[distance_key].replace(',', '')
if distance_str: distance = float(distance_str)
return dict(date=act[date_key],
ascent=ascent,
distance=distance,
type=act[type_key])
def parse_tcx(filename):
global last_tcx
if last_tcx is not None and last_tcx["filename"] == filename:
return last_tcx["tr"]
tcx = tcxparser.TCXParser(filename)
train = {}
train['activity_type'] = tcx.activity_type
train['calories'] = tcx.calories
train['distance_units'] = tcx.distance_units
train['date'] = tcx.time_values()[0]
tr = pd.DataFrame({'time': [], 'hr': []})
t1 = datetime.strptime(tcx.time_values()[0][:-1], "%Y-%m-%dT%H:%M:%S.%f")
for i in range(min(len(tcx.time_values()), len(tcx.hr_values()))):
t = (datetime.strptime(tcx.time_values()[i][:-1],
"%Y-%m-%dT%H:%M:%S.%f") - t1).total_seconds()
tr = tr.append({
"time": t,
"hr": tcx.hr_values()[i]
},
ignore_index=True)
tr["duration"] = -tr["time"].diff(periods=-1)
last_tcx = {"filename": filename, "tr": tr}
def tcx_to_dataframe(filename):
with open(filename, "r") as f:
lines = f.readlines()
lines[0] = lines[0].lstrip()
with open(filename, "w") as f:
f.writelines(lines)
tcx = tcxparser.TCXParser(filename)
time = tcx.time_values()
positions = tcx.position_values()
latitude, longitude = zip(*positions)
altitude = tcx.altitude_points()
speed = np.nan * np.ones([len(time), 1])
df = pd.DataFrame(
zip(*[time, latitude, longitude]),
columns=["time", "latitude", "longitude"],
)
df = clean(df)
return df
for child in root.getiterator():
if not hasattr(child.tag, 'find'): continue # (1)
i = child.tag.find('}')
if i >= 0:
child.tag = child.tag[i + 1:]
objectify.deannotate(root, cleanup_namespaces=True)
# Get activity id from file name
activity_id = filepath.split("_")
try:
activity_id = int(activity_id[1])
except:
activity_id = int(activity_id[1].split(".")[0])
# Get activity summary data
tcx = tcxparser.TCXParser(filepath)
# Finding summary heart rate values, if hrt rate = 0 it will return ZeroDivisionError
try:
avg_hrt_rate = int(tcx.hr_avg)
max_hrt_rate = int(tcx.hr_max)
except ZeroDivisionError:
avg_hrt_rate = 0
max_hrt_rate = 0
# Adding activities row
session.add(Activities(act_id=activity_id, \
act_type=tcx.activity_type, \
act_duration=int(tcx.duration), \
act_tot_distance=int(tcx.distance), \
act_max_speed=1, \
def process_files(self, db_params_dict):
garmin_db = GarminDB.GarminDB(db_params_dict, self.debug - 1)
garmin_act_db = GarminDB.ActivitiesDB(db_params_dict, self.debug)
for file_name in self.file_names:
logger.info("Processing file: " + file_name)
tcx = tcxparser.TCXParser(file_name)
end_time = dateutil.parser.parse(tcx.completed_at, ignoretz=True)
start_time = dateutil.parser.parse(tcx.started_at, ignoretz=True)
manufacturer = 'Unknown'
product = tcx.creator
if product is not None:
match = re.search('Microsoft', product)
if match:
manufacturer = 'Microsoft'
serial_number = tcx.creator_version
if serial_number is None or serial_number == 0:
serial_number = GarminDB.Device.unknown_device_serial_number
device = {
'serial_number': serial_number,
'timestamp': start_time,
'manufacturer': manufacturer,
'product': product,
'hardware_version': None,
}
GarminDB.Device.create_or_update_not_none(garmin_db, device)
file = {
'name': file_name,
'type': 'tcx',
'serial_number': serial_number,
}
GarminDB.File.find_or_create(garmin_db, file)
activity_id = GarminDB.File.get(garmin_db, file_name)
if self.english_units and tcx.distance_units == 'meters':
distance = Fit.Conversions.meters_to_miles(tcx.distance)
ascent = Fit.Conversions.meters_to_feet(tcx.ascent)
descent = Fit.Conversions.meters_to_feet(tcx.descent)
else:
distance = tcx.distance / 1000.0
ascent = tcx.ascent / 1000.0
descent = tcx.descent / 1000.0
activity = {
'activity_id': activity_id,
'start_time': start_time,
'stop_time': end_time,
'laps': len(tcx.activity.Lap),
# 'sport' : tcx.activity_type,
'start_lat': tcx.start_latitude,
'start_long': tcx.start_longitude,
'stop_lat': tcx.end_latitude,
'stop_long': tcx.end_longitude,
'distance': distance,
'avg_hr': tcx.hr_avg,
'max_hr': tcx.hr_max,
'calories': tcx.calories,
'max_cadence': tcx.cadence_max,
'avg_cadence': tcx.cadence_avg,
#'ascent' : ascent,
#'descent' : descent
}
activity_not_zero = {
key: value
for (key, value) in activity.iteritems() if value
}
print repr(activity_not_zero)
GarminDB.Activities.create_or_update_not_none(
garmin_act_db, activity_not_zero)
def __init__(self, filename):
"""Return an instance of TcxFile containing a parsed TCX file."""
self.tcx = tcxparser.TCXParser(filename)
# -*- coding: utf-8 -*-
import os,tcxparser
cd = os.path.dirname(os.path.abspath(__file__)) #use to browse tcx files at the correct location
file_list=os.listdir(os.getcwd())
f=open("result.txt","a")
f.write("date (ISO UTC Format)"+'\t'+"Distance [m]"+'\t'+"Duration [s]"+'\n')
for a in file_list:
if a == "result.txt":
print() #Quick and dirty...
elif a== "readTCXFiles.py":
print() #Quick and dirty...
else:
tcx=tcxparser.TCXParser(cd+'/'+a) #see https://github.com/vkurup/python-tcxparser/ for details
if tcx.activity_type == 'biking' : #To select only my biking session (could be change to 'hiking', 'running' etc.)
f.write(str(tcx.completed_at)+'\t'+str(tcx.distance)+'\t'+str(tcx.duration)+'\n')
f.close
def preproc_nike():
# Grab all zipfiles
nike_runs = sorted(
[os.path.join(nike, i) for i in os.listdir(nike) if i.endswith('zip')])
# Create Pandas Dataframe
df_nike = pd.DataFrame(index=range(len(nike_runs)))
# Loop through zip files, read json data and populate df
for idx, nike_run in enumerate(nike_runs):
# load zipfile
nike_zip = zipfile.ZipFile(nike_run, 'r')
# load json only
nike_json = json.load([
nike_zip.open(i) for i in nike_zip.namelist() if i.endswith('json')
][0])
nike_tcx = tcxparser.TCXParser([
nike_zip.open(i) for i in nike_zip.namelist() if i.endswith('tcx')
][0])
try:
if nike_tcx.latitude:
lat = nike_tcx.latitude
lon = nike_tcx.longitude
city, country = getplace(lat, lon)
location = '%s, %s' % (city, country)
print idx, lat, lon, city, country
else:
location = 'indoor'
except:
location = 'indoor'
df_nike.loc[idx, 'Source'] = '%s %s ' % (nike_json['source'],
nike_json['appversion'][:-5])
df_nike.loc[idx, 'Location'] = location # nike_json['title']
df_nike.loc[idx, 'Latitude'] = lat
df_nike.loc[idx, 'Longitude'] = lon
df_nike.loc[idx, 'Source'] = '%s %s ' % (nike_json['source'],
nike_json['appversion'][:-5])
df_nike.loc[idx, 'Speed_Max'] = nike_json['maxSpeed']
df_nike.loc[idx, 'Date'] = nike_json['startTime']['time'][0:10]
df_nike.loc[idx, 'Description'] = nike_json['description']
df_nike.loc[idx, 'Speed'] = nike_json['avgSpeed']
df_nike.loc[idx, 'Speed_Max'] = nike_json['maxSpeed']
try:
if nike_tcx.pace:
df_nike.loc[idx, 'Tempo'] = nike_tcx.pace
else:
df_nike.loc[idx, 'Tempo'] = 'xxxxxxxxxxxx'
except:
pass
df_nike.loc[idx, 'Distance'] = nike_tcx.distance / 1000
df_nike.loc[idx, 'Duration'] = nike_tcx.duration / 60
df_nike.loc[idx, 'Start'] = nike_json['startTime']['time'][11:-1]
df_nike.loc[idx, 'Ascent'] = nike_tcx.ascent
df_nike.loc[idx, 'Descent'] = nike_tcx.descent
df_nike.loc[idx, 'Elevation_Gain'] = nike_json['elevationGain']
df_nike.loc[idx, 'Elevation_Max'] = nike_json['maxElevation']
df_nike.loc[idx, 'Calories'] = nike_json['calories']
for lap_idx, lap in enumerate(nike_json['laps']):
df_nike.loc[idx, 'Km_%s' %
(lap_idx + 1)] = 'Duration=%s, Speed=%s' % (np.round(
lap['duration'] / 60.,
3), np.round(lap['avgSpeed'], 2))
df_nike.to_csv('./Data/NikeRunClub.csv', encoding='utf-8')
def output():
#
# man, this sucks
#
tracker = request.form.get('Tracker')
username = request.form.get('Username')
password = request.form.get('Password')
location = request.form.get('Location')
# dateStart=request.form.get('StartDate')
# print("Date start is: " + str(dateStart))
# dateEnd=request.form.get('EndDate')
dataModified = [] # for Jawbone file check
try:
#
# THIS GETS THE FILE. FLASK.
#
files = request.files.getlist('file[]')
print files
filenames = []
for ifile in files:
if ifile and allowed_file(ifile.filename):
filenames.append(ifile.filename)
ifile.save(
os.path.join(app.config['UPLOAD_RAW_DEST'],
ifile.filename))
print ifile.filename
#get location of saved file
#print( 'filename: ' + filename)
filepath = os.path.join(app.config['UPLOAD_RAW_DEST'], filenames[0])
print filepath
print app.config['UPLOAD_RAW_DEST']
if tracker == 'Jawbone':
import glob
# then this is a csv file
os.chdir('/root/tmp/raw')
# get the file
for ifile in range(len(filenames)):
print("FILE NAME IS: " + str(filenames[ifile]))
ff = glob.glob(filenames[ifile])[0]
dataAll = np.swapaxes(np.genfromtxt(ff, delimiter=','), 1, 0)
temp=[ datetime.datetime.strptime(str(int(dataAll[0][j])), \
'%Y%m%d') for j in range(1,len(dataAll[0]))]
dataYears = list(dataAll[0])
time = [np.nan] + temp
# now get the data that matters
#
# 41 is steps
# 37 is distance in meters
# 43 is calories
# 35 is active time in seconds
# 48 is sleep time
#
params = [
'Dates', 'Steps', 'Distance', 'Calories', 'ActiveTime',
'SleepTime'
]
indicies = [0, 41, 37, 43, 35, 48]
dataModified = {}
# now have to adjust for the right dates
for iP in range(len(params)):
dataModified[params[iP]] = []
goodvals = np.isnan(dataYears[41]) # get rid of the nans
dataModified['Steps'] += list(dataAll[41])
dataModified['Distance'] += list(dataAll[37])
dataModified['Calories'] += list(dataAll[43])
dataModified['ActiveTime'] += list(dataAll[35])
dataModified['SleepTime'] += list(dataAll[48])
dataModified['Dates'] += list(np.array(time))
#
if tracker == 'Endomondo':
import glob
from geopy.geocoders import Nominatim
import tcxparser
# then this is a tcx file
os.chdir('/root/tmp/raw')
# get the file
params = ['Date', 'ActiveTime', 'Distance', 'Calories', 'Location']
dataModified = {}
for iparam in params:
dataModified[iparam] = []
for ifile in range(len(filenames)):
print("FILE NAME IS: " + str(filenames[ifile]))
ff = glob.glob(filenames[ifile])[0]
startDate = ff[0:10]
tcx = tcxparser.TCXParser(ff)
# get data
workoutTime = tcx.duration
latitude = str(tcx.latitude)
longitude = str(tcx.longitude)
timeStamp = tcx.completed_at
try:
distance = tcx.distance
except:
distance = 1e-31
calories = tcx.calories
# turn lat and longitude into a city
geolocator = Nominatim()
location = (geolocator.reverse(latitude + ',' +
longitude)).address
# now have to adjust for the right dates
print startDate
dataModified['Date'].append(startDate)
dataModified['Distance'].append(distance)
dataModified['Calories'].append(calories)
dataModified['ActiveTime'].append(workoutTime)
dataModified['Location'].append(location)
except:
print("no file uploaded")
#
# now load the relevant figures
#
dateStart = 0
dateEnd = 0
if tracker == 'Jawbone':
import readJawBone as PL
data=PL.loadData(username, password, location,\
data=dataModified)
# elif tracker=='Polar Loop':
# import dataStats as PL
# data=PL.loadData(username, password, location, instartDate=dateStart, \
# inendDate=dateEnd)
#
# Jimmy's data
elif tracker == 'Endomondo':
import readTapiriik as PL
data = PL.loadData(username,
password,
location,
dataModified=dataModified)
dd = data[0]
temp = data[0] * 1.1
goal = ("%.0f" % temp)
cals = ("%.0f" % dd)
dayofweek = data[2]
weather = data[1]
boxwhisker = "/static/" + username + "/boxwhisker_Calories_daysofweek.png"
scatter1 = "/static/" + username + "/seaborn_Calories_Wind_scatter.png"
scatter2 = "/static/" + username + "/seaborn_Calories_MeanTemperature_scatter.png"
scatter3 = "/static/" + username + "/seaborn_Calories_Precip_scatter.png"
os.chdir('/root/Hosting/flaskexample')
import stat
os.chmod('/root/Hosting/flaskexample' + boxwhisker, 0o777)
os.chmod('/root/Hosting/flaskexample' + scatter1, 0o777)
os.chmod('/root/Hosting/flaskexample' + scatter2, 0o777)
os.chmod('/root/Hosting/flaskexample' + scatter3, 0o777)
print(scatter1)
return render_template("output.html", calories=cals, tracker=tracker, \
goal=goal, dayofweek=str(dayofweek),\
weather=str(weather), boxwhisker=boxwhisker, \
scatter1=scatter1, scatter2=scatter2, \
scatter3=scatter3)