def DownloadActivityList(self, svcRec, exhaustive=False):
dbcl = self._getClient(svcRec)
if not svcRec.Authorization["Full"]:
syncRoot = "/"
else:
syncRoot = svcRec.Config["SyncRoot"]
cache = cachedb.dropbox_cache.find_one({"ExternalID": svcRec.ExternalID})
if cache is None:
cache = {"ExternalID": svcRec.ExternalID, "Structure": [], "Activities": {}}
if "Structure" not in cache:
cache["Structure"] = []
self._folderRecurse(cache["Structure"], dbcl, syncRoot)
activities = []
exclusions = []
for dir in cache["Structure"]:
for file in dir["Files"]:
path = file["Path"]
if svcRec.Authorization["Full"]:
relPath = path.replace(syncRoot, "", 1)
else:
relPath = path.replace("/Apps/tapiriik/", "", 1) # dropbox api is meh api
hashedRelPath = self._hash_path(relPath)
if hashedRelPath in cache["Activities"]:
existing = cache["Activities"][hashedRelPath]
else:
existing = None
if not existing:
# Continue to use the old records keyed by UID where possible
existing = [
(k, x) for k, x in cache["Activities"].items() if "Path" in x and x["Path"] == relPath
] # path is relative to syncroot to reduce churn if they relocate it
existing = existing[0] if existing else None
if existing is not None:
existUID, existing = existing
existing["UID"] = existUID
if existing and existing["Rev"] == file["Rev"]:
# don't need entire activity loaded here, just UID
act = UploadedActivity()
act.UID = existing["UID"]
try:
act.StartTime = datetime.strptime(existing["StartTime"], "%H:%M:%S %d %m %Y %z")
except:
act.StartTime = datetime.strptime(
existing["StartTime"], "%H:%M:%S %d %m %Y"
) # Exactly one user has managed to break %z :S
if "EndTime" in existing: # some cached activities may not have this, it is not essential
act.EndTime = datetime.strptime(existing["EndTime"], "%H:%M:%S %d %m %Y %z")
else:
logger.debug(
"Retrieving %s (%s)" % (path, "outdated meta cache" if existing else "not in meta cache")
)
# get the full activity
try:
act, rev = self._getActivity(svcRec, dbcl, path)
except APIExcludeActivity as e:
logger.info("Encountered APIExcludeActivity %s" % str(e))
exclusions.append(strip_context(e))
continue
try:
act.EnsureTZ()
except:
pass # We tried.
if hasattr(act, "OriginatedFromTapiriik") and not act.CountTotalWaypoints():
# This is one of the files created when TCX export was hopelessly broken for non-GPS activities.
# Right now, no activities in dropbox from tapiriik should be devoid of waypoints - since dropbox doesn't receive stationary activities
# In the future when this changes, will obviously have to modify this code to also look at modification dates or similar.
if ".tcx.summary-data" in path:
logger.info("...summary file already moved")
else:
logger.info("...moving summary-only file")
dbcl.file_move(path, path.replace(".tcx", ".tcx.summary-data"))
continue # DON'T include in listing - it'll be regenerated
del act.Laps
act.Laps = (
[]
) # Yeah, I'll process the activity twice, but at this point CPU time is more plentiful than RAM.
cache["Activities"][hashedRelPath] = {
"Rev": rev,
"UID": act.UID,
"StartTime": act.StartTime.strftime("%H:%M:%S %d %m %Y %z"),
"EndTime": act.EndTime.strftime("%H:%M:%S %d %m %Y %z"),
}
tagRes = self._tagActivity(relPath)
act.ServiceData = {"Path": path, "Tagged": tagRes is not None}
act.Type = tagRes if tagRes is not None else ActivityType.Other
logger.debug("Activity s/t %s" % act.StartTime)
activities.append(act)
if "_id" in cache:
cachedb.dropbox_cache.save(cache)
else:
cachedb.dropbox_cache.insert(cache)
return activities, exclusions
def DownloadActivityList(self, svcRec, exhaustive=False):
dbcl = self._getClient(svcRec)
if not svcRec.Authorization["Full"]:
syncRoot = "/"
else:
syncRoot = svcRec.Config["SyncRoot"]
# Dropbox API v2 doesn't like / as root.
if syncRoot == "/":
syncRoot = ""
# New Dropbox API prefers path_lower, it would seem.
syncRoot = syncRoot.lower()
# There used to be a massive affair going on here to cache the folder structure locally.
# Dropbox API 2.0 doesn't support the hashes I need for that.
# Oh well. Throw that data out now. Well, don't load it at all.
cache = cachedb.dropbox_cache.find_one({"ExternalID": svcRec.ExternalID}, {"ExternalID": True, "Activities": True})
if cache is None:
cache = {"ExternalID": svcRec.ExternalID, "Activities": {}}
try:
list_result = dbcl.files_list_folder(syncRoot, recursive=True)
except dropbox.exceptions.DropboxException as e:
self._raiseDbException(e)
def cache_writeback():
if "_id" in cache:
cachedb.dropbox_cache.save(cache)
else:
insert_result = cachedb.dropbox_cache.insert(cache)
cache["_id"] = insert_result.inserted_id
activities = []
exclusions = []
discovered_activity_cache_keys = set()
while True:
for entry in list_result.entries:
if not hasattr(entry, "rev"):
# Not a file -> we don't care.
continue
path = entry.path_lower
if not path.endswith(".gpx") and not path.endswith(".tcx"):
# Not an activity file -> we don't care.
continue
if svcRec.Authorization["Full"]:
relPath = path.replace(syncRoot, "", 1)
else:
relPath = path.replace("/Apps/tapiriik/", "", 1) # dropbox api is meh api
hashedRelPath = self._hash_path(relPath)
discovered_activity_cache_keys.add(hashedRelPath)
if hashedRelPath in cache["Activities"]:
existing = cache["Activities"][hashedRelPath]
else:
existing = None
if existing and existing["Rev"] == entry.rev:
# don't need entire activity loaded here, just UID
act = UploadedActivity()
act.UID = existing["UID"]
try:
act.StartTime = datetime.strptime(existing["StartTime"], "%H:%M:%S %d %m %Y %z")
except:
act.StartTime = datetime.strptime(existing["StartTime"], "%H:%M:%S %d %m %Y") # Exactly one user has managed to break %z :S
if "EndTime" in existing: # some cached activities may not have this, it is not essential
act.EndTime = datetime.strptime(existing["EndTime"], "%H:%M:%S %d %m %Y %z")
else:
logger.debug("Retrieving %s (%s)" % (path, "outdated meta cache" if existing else "not in meta cache"))
# get the full activity
try:
act, rev = self._getActivity(svcRec, dbcl, path)
except APIExcludeActivity as e:
logger.info("Encountered APIExcludeActivity %s" % str(e))
exclusions.append(strip_context(e))
continue
try:
act.EnsureTZ()
except:
pass # We tried.
act.Laps = [] # Yeah, I'll process the activity twice, but at this point CPU time is more plentiful than RAM.
cache["Activities"][hashedRelPath] = {"Rev": rev, "UID": act.UID, "StartTime": act.StartTime.strftime("%H:%M:%S %d %m %Y %z"), "EndTime": act.EndTime.strftime("%H:%M:%S %d %m %Y %z")}
# Incrementally update the cache db.
# Otherwise, if we crash later on in listing
# (due to OOM or similar), we'll never make progress on this account.
cache_writeback()
tagRes = self._tagActivity(relPath)
act.ServiceData = {"Path": path, "Tagged": tagRes is not None}
act.Type = tagRes if tagRes is not None else ActivityType.Other
logger.debug("Activity s/t %s" % act.StartTime)
activities.append(act)
# Perform pagination.
if list_result.has_more:
list_result = dbcl.files_list_folder_continue(list_result.cursor)
else:
break
# Drop deleted activities' records from cache.
all_activity_cache_keys = set(cache["Activities"].keys())
for deleted_key in all_activity_cache_keys - discovered_activity_cache_keys:
del cache["Activities"][deleted_key]
cache_writeback()
return activities, exclusions
def DownloadActivityList(self, svcRec, exhaustive=False):
dbcl = self._getClient(svcRec)
if not svcRec.Authorization["Full"]:
syncRoot = "/"
else:
syncRoot = svcRec.Config["SyncRoot"]
# Dropbox API v2 doesn't like / as root.
if syncRoot == "/":
syncRoot = ""
# New Dropbox API prefers path_lower, it would seem.
syncRoot = syncRoot.lower()
# There used to be a massive affair going on here to cache the folder structure locally.
# Dropbox API 2.0 doesn't support the hashes I need for that.
# Oh well. Throw that data out now. Well, don't load it at all.
cache = cachedb.dropbox_cache.find_one(
{"ExternalID": svcRec.ExternalID}, {
"ExternalID": True,
"Activities": True
})
if cache is None:
cache = {"ExternalID": svcRec.ExternalID, "Activities": {}}
try:
list_result = dbcl.files_list_folder(syncRoot, recursive=True)
except dropbox.exceptions.DropboxException as e:
self._raiseDbException(e)
def cache_writeback():
if "_id" in cache:
cachedb.dropbox_cache.save(cache)
else:
insert_result = cachedb.dropbox_cache.insert(cache)
cache["_id"] = insert_result.inserted_id
activities = []
exclusions = []
discovered_activity_cache_keys = set()
while True:
for entry in list_result.entries:
if not hasattr(entry, "rev"):
# Not a file -> we don't care.
continue
path = entry.path_lower
if not path.endswith(".gpx") and not path.endswith(".tcx"):
# Not an activity file -> we don't care.
continue
if svcRec.Authorization["Full"]:
relPath = path.replace(syncRoot, "", 1)
else:
relPath = path.replace("/Apps/tapiriik/", "",
1) # dropbox api is meh api
hashedRelPath = self._hash_path(relPath)
discovered_activity_cache_keys.add(hashedRelPath)
if hashedRelPath in cache["Activities"]:
existing = cache["Activities"][hashedRelPath]
else:
existing = None
if existing and existing["Rev"] == entry.rev:
# don't need entire activity loaded here, just UID
act = UploadedActivity()
act.UID = existing["UID"]
try:
act.StartTime = datetime.strptime(
existing["StartTime"], "%H:%M:%S %d %m %Y %z")
except:
act.StartTime = datetime.strptime(
existing["StartTime"], "%H:%M:%S %d %m %Y"
) # Exactly one user has managed to break %z :S
if "EndTime" in existing: # some cached activities may not have this, it is not essential
act.EndTime = datetime.strptime(
existing["EndTime"], "%H:%M:%S %d %m %Y %z")
else:
logger.debug("Retrieving %s (%s)" %
(path, "outdated meta cache"
if existing else "not in meta cache"))
# get the full activity
try:
act, rev = self._getActivity(svcRec, dbcl, path)
except APIExcludeActivity as e:
logger.info("Encountered APIExcludeActivity %s" %
str(e))
exclusions.append(strip_context(e))
continue
try:
act.EnsureTZ()
except:
pass # We tried.
act.Laps = [
] # Yeah, I'll process the activity twice, but at this point CPU time is more plentiful than RAM.
cache["Activities"][hashedRelPath] = {
"Rev": rev,
"UID": act.UID,
"StartTime":
act.StartTime.strftime("%H:%M:%S %d %m %Y %z"),
"EndTime": act.EndTime.strftime("%H:%M:%S %d %m %Y %z")
}
# Incrementally update the cache db.
# Otherwise, if we crash later on in listing
# (due to OOM or similar), we'll never make progress on this account.
cache_writeback()
tagRes = self._tagActivity(relPath)
act.ServiceData = {"Path": path, "Tagged": tagRes is not None}
act.Type = tagRes if tagRes is not None else ActivityType.Other
logger.debug("Activity s/t %s" % act.StartTime)
activities.append(act)
# Perform pagination.
if list_result.has_more:
list_result = dbcl.files_list_folder_continue(
list_result.cursor)
else:
break
# Drop deleted activities' records from cache.
all_activity_cache_keys = set(cache["Activities"].keys())
for deleted_key in all_activity_cache_keys - discovered_activity_cache_keys:
del cache["Activities"][deleted_key]
cache_writeback()
return activities, exclusions
def DownloadActivityList(self, svcRec, exhaustive=False):
dbcl = self._getClient(svcRec)
if not svcRec.Authorization["Full"]:
syncRoot = "/"
else:
syncRoot = svcRec.Config["SyncRoot"]
cache = cachedb.dropbox_cache.find_one(
{"ExternalID": svcRec.ExternalID})
if cache is None:
cache = {
"ExternalID": svcRec.ExternalID,
"Structure": [],
"Activities": {}
}
if "Structure" not in cache:
cache["Structure"] = []
self._folderRecurse(cache["Structure"], dbcl, syncRoot)
activities = []
exclusions = []
for dir in cache["Structure"]:
for file in dir["Files"]:
path = file["Path"]
if svcRec.Authorization["Full"]:
relPath = path.replace(syncRoot, "", 1)
else:
relPath = path.replace("/Apps/tapiriik/", "",
1) # dropbox api is meh api
hashedRelPath = self._hash_path(relPath)
if hashedRelPath in cache["Activities"]:
existing = cache["Activities"][hashedRelPath]
else:
existing = None
if not existing:
# Continue to use the old records keyed by UID where possible
existing = [
(k, x) for k, x in cache["Activities"].items()
if "Path" in x and x["Path"] == relPath
] # path is relative to syncroot to reduce churn if they relocate it
existing = existing[0] if existing else None
if existing is not None:
existUID, existing = existing
existing["UID"] = existUID
if existing and existing["Rev"] == file["Rev"]:
# don't need entire activity loaded here, just UID
act = UploadedActivity()
act.UID = existing["UID"]
try:
act.StartTime = datetime.strptime(
existing["StartTime"], "%H:%M:%S %d %m %Y %z")
except:
act.StartTime = datetime.strptime(
existing["StartTime"], "%H:%M:%S %d %m %Y"
) # Exactly one user has managed to break %z :S
if "EndTime" in existing: # some cached activities may not have this, it is not essential
act.EndTime = datetime.strptime(
existing["EndTime"], "%H:%M:%S %d %m %Y %z")
else:
logger.debug("Retrieving %s (%s)" %
(path, "outdated meta cache"
if existing else "not in meta cache"))
# get the full activity
try:
act, rev = self._getActivity(svcRec, dbcl, path)
except APIExcludeActivity as e:
logger.info("Encountered APIExcludeActivity %s" %
str(e))
exclusions.append(strip_context(e))
continue
try:
act.EnsureTZ()
except:
pass # We tried.
if hasattr(act, "OriginatedFromTapiriik"
) and not act.CountTotalWaypoints():
# This is one of the files created when TCX export was hopelessly broken for non-GPS activities.
# Right now, no activities in dropbox from tapiriik should be devoid of waypoints - since dropbox doesn't receive stationary activities
# In the future when this changes, will obviously have to modify this code to also look at modification dates or similar.
if ".tcx.summary-data" in path:
logger.info("...summary file already moved")
else:
logger.info("...moving summary-only file")
dbcl.file_move(
path, path.replace(".tcx",
".tcx.summary-data"))
continue # DON'T include in listing - it'll be regenerated
del act.Laps
act.Laps = [
] # Yeah, I'll process the activity twice, but at this point CPU time is more plentiful than RAM.
cache["Activities"][hashedRelPath] = {
"Rev": rev,
"UID": act.UID,
"StartTime":
act.StartTime.strftime("%H:%M:%S %d %m %Y %z"),
"EndTime": act.EndTime.strftime("%H:%M:%S %d %m %Y %z")
}
tagRes = self._tagActivity(relPath)
act.ServiceData = {"Path": path, "Tagged": tagRes is not None}
act.Type = tagRes if tagRes is not None else ActivityType.Other
logger.debug("Activity s/t %s" % act.StartTime)
activities.append(act)
if "_id" in cache:
cachedb.dropbox_cache.save(cache)
else:
cachedb.dropbox_cache.insert(cache)
return activities, exclusions