def test_last_photos_picker_dropbox_uploader(self, filters,
expected_files):
"""
Test with LastPhotosPicker and DropboxUploader
:param array filters: filters to use
:param dict expected_files: expected files with hash of their content
"""
if 'DROPBOX_TOKEN' not in os.environ.keys():
raise SkipTest("DROPBOX_TOKEN environment variable is not defined")
api_token = os.environ['DROPBOX_TOKEN']
picker = LastPhotosPicker(self.sample_dir, 5, -1)
uploader = DropboxUploader(api_token, self.remote_test_dir)
photo_picker = PhotosPicker(picker, filters, uploader)
photo_picker.run()
dbx = Dropbox(api_token)
test_dir = '/' + self.remote_test_dir
files = dbx.files_list_folder(test_dir)
actual_files = {}
for file_metadata in files.entries:
fullpath = self.target_dir + '/' + file_metadata.name
dbx.files_download_to_file(fullpath,
test_dir + '/' + file_metadata.name)
md5 = self._compute_file_md5(fullpath)
actual_files[file_metadata.name] = md5
os.remove(fullpath)
self.assertEqual(expected_files, actual_files)
class DropboxHelper(object):
def __init__(self, access_token):
self.dropbox = Dropbox(oauth2_access_token=access_token)
def upload(self, filename, file_path):
with open(file_path, 'rb') as f:
try:
self.dropbox.files_upload(f.read(), '/' + filename)
except Exception:
os.remove(file_path)
raise CommandError(
'Unable to upload file to Dropbox. Maybe access token is invalid.'
)
def delete_all_files(self):
for i in self.dropbox.files_list_folder('').entries:
self.dropbox.files_delete(i.path_lower)
def download_last_backup(self, dir_path):
entries = self.dropbox.files_list_folder('').entries
if len(entries) == 0:
raise CommandError('We could not find any backup.')
entry = entries[-1]
full_path = dir_path + entry.path_lower
self.dropbox.files_download_to_file(full_path, entry.path_lower)
return full_path, entry.content_hash
class DropboxHelper(object):
def __init__(self, access_token):
self.dropbox = Dropbox(oauth2_access_token=access_token)
def upload(self, filename, file_path):
with open(file_path, 'rb') as f:
try:
self.dropbox.files_upload(f.read(), '/' + filename)
except Exception:
os.remove(file_path)
raise CommandError('Unable to upload file to Dropbox. Maybe access token is invalid.')
def delete_all_files(self):
for i in self.dropbox.files_list_folder('').entries:
self.dropbox.files_delete(i.path_lower)
def download_last_backup(self, dir_path):
entries = self.dropbox.files_list_folder('').entries
if len(entries) == 0:
raise CommandError('We could not find any backup.')
entry = entries[-1]
full_path = dir_path + entry.path_lower
self.dropbox.files_download_to_file(full_path, entry.path_lower)
return full_path, entry.content_hash
def download_file(hashed_user_name, mapped_image_name, original_image_name):
dbx = Dropbox(token)
download_path = path.join(getcwd().strip('utilities'), "temp",
hashed_user_name)
if not path.exists(download_path):
mkdir(download_path)
dbx.files_download_to_file(path.join(download_path, mapped_image_name),
'/' + mapped_image_name)
def download_db():
dbx = Dropbox(token)
if cfg['environment']['host'] == 'linux':
print("Linux Environment")
dbx.files_download_to_file(
path.join(getcwd().strip('utilities'), 'costrajectory.db'),
'/costrajectory.db')
else:
print("Windows Environment")
dbx.files_download_to_file(getcwd() + '\\costrajectory.db',
'/costrajectory.db')
print('DB Successfully updated to remote')
class DropBoxDataProvider(DataProviderBase):
smoke_url = DROPBOX_SMOKE_URL
def __init__(self, acs_token):
self.dbx = Dropbox(acs_token)
def api_smoke(self) -> int:
return len(self.dbx.files_list_folder('').entries)
def get_list_of_objects(self, dbx_folder='') -> list:
result = namedtuple('Result', ['filename', 'filepatch'])
return [
result(el.name, el.path_lower)
for el in self.dbx.files_list_folder(dbx_folder).entries
]
def file_delete(self, dbx_file) -> str:
return self.dbx.files_delete_v2(dbx_file).metadata.path_lower
def file_download(self, local_file, dbx_file) -> str:
return self.dbx.files_download_to_file(local_file, dbx_file).path_lower
def file_upload(self, local_file, dbx_file) -> str:
if isinstance(local_file, str):
if local_file.startswith("https://"):
waiting_time = 0.1
waiting_attempt = 100
url_result = self.dbx.files_save_url(dbx_file, local_file)
job_id = url_result.get_async_job_id()
while waiting_attempt > 0:
st = self.dbx.files_save_url_check_job_status(job_id)
if st.is_complete():
return st.get_complete().path_lower
sleep(waiting_time)
waiting_attempt -= 1
else:
with open(local_file, 'rb') as f:
return self.dbx.files_upload(
f.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
else:
return self.dbx.files_upload(local_file.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
def file_move(self, dbx_file_from, dbx_file_to) -> str:
return self.dbx.files_move_v2(dbx_file_from,
dbx_file_to).metadata.path_lower
def create_folder(self, dbx_folder) -> str:
return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower
def get_file_tmp_link(self, dbx_path) -> str:
return self.dbx.files_get_temporary_link(dbx_path).link
def download_dropboxfiles(payload):
# Get the Project
project = Project.objects.get(pk=payload['project_id'])
project.set_status('downloading')
# Check to see what files to download from Dropbox
client = Dropbox(project.user.dropboxinfo.access_token)
num_files = 0
for x in client.files_list_folder(project.path).entries:
if x.path_lower.endswith('.jpg') and x.size > 0:
# Download the file from Dropbox to local disk
local_filename = os.path.split(x.path_lower)[-1]
local_filepath = os.path.join(project.originals_path,
local_filename)
num_files += 1
if os.path.exists(local_filepath
): # and not payload.get('redownload') == True
continue
client.files_download_to_file(local_filepath, x.path_lower)
# Get the metadata as a separate task
new_task(project.user, {
'action': 'extract_metadata',
'project_id': project.pk
})
# schedule a thumbnail task
new_task(project.user, {
'action': 'makethumbnails',
'project_id': project.pk
})
# Downloading files can take a long time
# In the meantime this Project could have been changed by other tasks
# Reload it before setting the status
project = Project.objects.get(pk=payload['project_id'])
project.num_files_on_dropbox = num_files
project.status = 'layout'
project.save()
return {'downloaded_files_count': num_files}
def download_dropboxfiles(payload):
# Get the Project
project = Project.objects.get(pk=payload['project_id'])
project.set_status('downloading')
# Check to see what files to download from Dropbox
client = Dropbox(project.user.dropboxinfo.access_token)
num_files = 0
for x in client.files_list_folder(project.path).entries:
if x.path_lower.endswith('.jpg') and x.size > 0:
# Download the file from Dropbox to local disk
local_filename = os.path.split(x.path_lower)[-1]
local_filepath = os.path.join(project.originals_path, local_filename)
num_files += 1
if os.path.exists(local_filepath): # and not payload.get('redownload') == True
continue
client.files_download_to_file(local_filepath, x.path_lower)
# Get the metadata as a separate task
new_task(project.user, {
'action': 'extract_metadata',
'project_id': project.pk
})
# schedule a thumbnail task
new_task(project.user, {
'action': 'makethumbnails',
'project_id': project.pk
})
# Downloading files can take a long time
# In the meantime this Project could have been changed by other tasks
# Reload it before setting the status
project = Project.objects.get(pk=payload['project_id'])
project.num_files_on_dropbox = num_files
project.status = 'layout'
project.save()
return {'downloaded_files_count':num_files}
import os
from dropbox import Dropbox
dbx = Dropbox(os.environ.get('DROPBOX_TOKEN'))
os.mkdir('samples/test_data/')
dbx.files_download_to_file('samples/test_data/test_image.jpeg', '/annotation_service_test_data/test_image.jpeg')
class DropboxUploader(BaseUploader):
def __init__(self):
super(BaseUploader, self).__init__()
self.oauth2_access_token = os.getenv(
"DROPBOX_OAUTH2_TOKEN") or get_app_config("DROPBOX_OAUTH2_TOKEN")
self.root_path = (os.getenv("DROPBOX_ROOT_PATH")
or get_app_config("DROPBOX_ROOT_PATH") or "/CTFd")
self.client = Dropbox(self.oauth2_access_token, timeout=100)
self.write_mode = "add" # can be set to overwrite
def _clean_filename(self, c):
if c in string.ascii_letters + string.digits + "-" + "_" + ".":
return True
def _full_path(self, name):
return safe_join(self.root_path, name).replace("\\", "/")
def store(self, fileobj, filename):
self.client.files_upload(fileobj.read(),
self._full_path(filename),
mode=WriteMode(self.write_mode))
return filename
def upload(self, file_obj, filename):
filename = filter(self._clean_filename,
secure_filename(filename).replace(" ", "_"))
filename = "".join(filename)
if len(filename) <= 0:
return False
md5hash = hexencode(os.urandom(16))
dst = md5hash + "/" + filename
self.store(file_obj, dst)
return dst
def download(self, filename):
media = self.client.files_get_temporary_link(self._full_path(filename))
print(media.link)
return redirect(media.link)
def delete(self, filename):
directory = os.path.dirname(self._full_path(filename))
self.client.files_delete(directory)
return True
def sync(self):
local_folder = current_app.config.get("UPLOAD_FOLDER")
root_metadata = self.client.files_list_folder(self.root_path)
for folder_entry in root_metadata.entries:
if isinstance(folder_entry, FolderMetadata):
filemetadata = self.client.files_list_folder(
folder_entry.path_lower)
for file_entry in filemetadata.entries:
if not isinstance(file_entry, FolderMetadata):
dropbox_path = file_entry.path_lower.replace(
self.root_path.lower() + "/", "")
local_path = os.path.join(local_folder, dropbox_path)
directory = os.path.dirname(local_path)
if not os.path.exists(directory):
os.makedirs(directory)
self.client.files_download_to_file(
local_path, file_entry.path_lower)
