def _download(self, name, source, destination_dir, overwrite=False):
"""use rclone to download a file, untar if it is a .tar file, and update self.local_files with the file path
Args:
name: brief descriptor of the file, used as the filepath key in self.local_files
source: full path to a dropbox file, including the remote
destination_dir: full path to the local destination directory
overwrite: if True, run rclone copy even if a local file with the intended name already exists
Returns:
the full path to the newly downloaded file (or directory, if the file was a tarfile)
"""
local_path = join(destination_dir, os.path.basename(source))
if not os.path.exists(local_path) or overwrite:
run(['rclone', 'copy', source, destination_dir])
assert os.path.exists(
local_path
), "download failed\nsource: {}\ndestination_dir: {}".format(
source, destination_dir)
if os.path.splitext(local_path)[1] == '.tar':
if not os.path.exists(os.path.splitext(local_path)[0]):
run([
'tar', '-xvf', local_path, '-C',
os.path.dirname(local_path)
])
local_path = os.path.splitext(local_path)[0]
self.local_files.update({name: local_path})
return local_path
def _locate_cloud_files(self):
"""locate the required files in Dropbox.
Returns:
string: cloud_master_dir, the outermost Dropbox directory that will be used henceforth
dict: cloud_files, a dict of paths to remote files, keyed by brief descriptors
"""
# establish the correct remote
possible_remotes = run(['rclone', 'listremotes']).split()
if len(possible_remotes) == 1:
remote = possible_remotes[0]
elif 'cichlidVideo:' in possible_remotes:
remote = 'cichlidVideo:'
elif 'd:' in possible_remotes:
remote = 'd:'
else:
raise Exception('unable to establish rclone remote')
# establish the correct path to the CichlidPiData directory
root_dir = [
r for r in run(['rclone', 'lsf', remote]).split() if 'McGrath' in r
][0]
cloud_master_dir = join(remote + root_dir, 'Apps', 'CichlidPiData')
# locate essential, project non-specific files
cloud_files = {
'boxed_fish_csv':
join(cloud_master_dir, '__AnnotatedData/BoxedFish/BoxedFish.csv')
}
return cloud_master_dir, cloud_files
def _locate_cloud_files(self):
"""locate the required files in Dropbox.
Returns:
string: cloud_master_dir, the outermost Dropbox directory that will be used henceforth
dict: cloud_files, a dict of paths to remote files, keyed by brief descriptors
"""
# establish the correct remote
possible_remotes = run(['rclone', 'listremotes']).split()
if len(possible_remotes) == 1:
remote = possible_remotes[0]
elif 'cichlidVideo:' in possible_remotes:
remote = 'cichlidVideo:'
elif 'd:' in possible_remotes:
remote = 'd:'
else:
raise Exception('unable to establish rclone remote')
# establish the correct path to the CichlidPiData directory
root_dir = [
r for r in run(['rclone', 'lsf', remote]).split() if 'McGrath' in r
][0]
cloud_master_dir = join(remote + root_dir, 'Apps', 'CichlidPiData')
# locate essential, project non-specific files
remoteDir = cloud_master_dir + '/{}/Frames/'.format(self.pid)
remote_files = subprocess.run(['rclone', 'lsf', remoteDir],
capture_output=True,
encoding='utf-8')
return cloud_master_dir, remoteDir, remote_files
def _locate_cloud_files(self):
"""track down project-specific files in Dropbox.
Overwrites FileManager._locate_cloud_files() method
Returns:
dict: cloud file paths keyed by brief file descriptors.
"""
cloud_image_dir = join(
self.cloud_master_dir,
'__AnnotatedData/BoxedFish/BoxedImages/{}.tar'.format(self.pid))
cloud_files = {
'project_image_dir': cloud_image_dir
} if self.download_images else {}
remote_files = run(
['rclone', 'lsf',
join(self.cloud_master_dir, self.pid)])
if 'videoCropPoints.npy' and 'videoCrop.npy' in remote_files.split():
cloud_files.update({
'video_points_numpy':
join(self.cloud_master_dir, self.pid, 'videoCropPoints.npy')
})
cloud_files.update({
'video_crop_numpy':
join(self.cloud_master_dir, self.pid, 'videoCrop.npy')
})
else:
cloud_files.update({
'video_points_numpy':
join(self.cloud_master_dir, self.pid, 'MasterAnalysisFiles',
'VideoPoints.npy')
})
cloud_files.update({
'video_crop_numpy':
join(self.cloud_master_dir, self.pid, 'MasterAnalysisFiles',
'VideoCrop.npy')
})
if self.download_videos:
cloud_video_dir = join(self.cloud_master_dir, self.pid, 'Videos')
videos_remote = run(
['rclone', 'lsf', cloud_video_dir, '--include',
'*.mp4']).split()
for v in videos_remote:
cloud_files.update({
'{}'.format(v):
join(self.cloud_master_dir, self.pid, 'Videos', v)
})
return cloud_files
def sync_detection_dir(exclude=None, quiet=False):
""" Sync the detection directory bidirectionally, keeping the newer version of each file
Args:
exclude (list of str): files/directories to exclude. Accepts both explicit file/directory names and
regular expressions. Expects a list, even if it's a list of length one. Default None.
quiet: if True, suppress the output of rclone copy. Default False
"""
print('syncing training directory')
cloud_detection_dir = join(fm.cloud_master_dir, '___Rhiya',
'CichlidDetection', 'detection')
down = [
'rclone', 'copy', '-u', '-c', cloud_detection_dir,
fm.local_files['detection_dir']
]
up = [
'rclone', 'copy', '-u', '-c', fm.local_files['detection_dir'],
cloud_detection_dir, '--exclude', '.*{/**,}'
]
if not quiet:
[com.insert(3, '-P') for com in [down, up]]
if exclude is not None:
[
com.extend(
list(
chain.from_iterable(
zip(['--exclude'] * len(exclude), exclude))))
for com in [down, up]
]
[run(com) for com in [down, up]]