def test_last_photos_picker_dropbox_uploader(self, filters, expected_files): """ Test with LastPhotosPicker and DropboxUploader :param array filters: filters to use :param dict expected_files: expected files with hash of their content """ if 'DROPBOX_TOKEN' not in os.environ.keys(): raise SkipTest("DROPBOX_TOKEN environment variable is not defined") api_token = os.environ['DROPBOX_TOKEN'] picker = LastPhotosPicker(self.sample_dir, 5, -1) uploader = DropboxUploader(api_token, self.remote_test_dir) photo_picker = PhotosPicker(picker, filters, uploader) photo_picker.run() dbx = Dropbox(api_token) test_dir = '/' + self.remote_test_dir files = dbx.files_list_folder(test_dir) actual_files = {} for file_metadata in files.entries: fullpath = self.target_dir + '/' + file_metadata.name dbx.files_download_to_file(fullpath, test_dir + '/' + file_metadata.name) md5 = self._compute_file_md5(fullpath) actual_files[file_metadata.name] = md5 os.remove(fullpath) self.assertEqual(expected_files, actual_files)
class DropboxHelper(object): def __init__(self, access_token): self.dropbox = Dropbox(oauth2_access_token=access_token) def upload(self, filename, file_path): with open(file_path, 'rb') as f: try: self.dropbox.files_upload(f.read(), '/' + filename) except Exception: os.remove(file_path) raise CommandError( 'Unable to upload file to Dropbox. Maybe access token is invalid.' ) def delete_all_files(self): for i in self.dropbox.files_list_folder('').entries: self.dropbox.files_delete(i.path_lower) def download_last_backup(self, dir_path): entries = self.dropbox.files_list_folder('').entries if len(entries) == 0: raise CommandError('We could not find any backup.') entry = entries[-1] full_path = dir_path + entry.path_lower self.dropbox.files_download_to_file(full_path, entry.path_lower) return full_path, entry.content_hash
class DropboxHelper(object): def __init__(self, access_token): self.dropbox = Dropbox(oauth2_access_token=access_token) def upload(self, filename, file_path): with open(file_path, 'rb') as f: try: self.dropbox.files_upload(f.read(), '/' + filename) except Exception: os.remove(file_path) raise CommandError('Unable to upload file to Dropbox. Maybe access token is invalid.') def delete_all_files(self): for i in self.dropbox.files_list_folder('').entries: self.dropbox.files_delete(i.path_lower) def download_last_backup(self, dir_path): entries = self.dropbox.files_list_folder('').entries if len(entries) == 0: raise CommandError('We could not find any backup.') entry = entries[-1] full_path = dir_path + entry.path_lower self.dropbox.files_download_to_file(full_path, entry.path_lower) return full_path, entry.content_hash
def download_file(hashed_user_name, mapped_image_name, original_image_name): dbx = Dropbox(token) download_path = path.join(getcwd().strip('utilities'), "temp", hashed_user_name) if not path.exists(download_path): mkdir(download_path) dbx.files_download_to_file(path.join(download_path, mapped_image_name), '/' + mapped_image_name)
def download_db(): dbx = Dropbox(token) if cfg['environment']['host'] == 'linux': print("Linux Environment") dbx.files_download_to_file( path.join(getcwd().strip('utilities'), 'costrajectory.db'), '/costrajectory.db') else: print("Windows Environment") dbx.files_download_to_file(getcwd() + '\\costrajectory.db', '/costrajectory.db') print('DB Successfully updated to remote')
class DropBoxDataProvider(DataProviderBase): smoke_url = DROPBOX_SMOKE_URL def __init__(self, acs_token): self.dbx = Dropbox(acs_token) def api_smoke(self) -> int: return len(self.dbx.files_list_folder('').entries) def get_list_of_objects(self, dbx_folder='') -> list: result = namedtuple('Result', ['filename', 'filepatch']) return [ result(el.name, el.path_lower) for el in self.dbx.files_list_folder(dbx_folder).entries ] def file_delete(self, dbx_file) -> str: return self.dbx.files_delete_v2(dbx_file).metadata.path_lower def file_download(self, local_file, dbx_file) -> str: return self.dbx.files_download_to_file(local_file, dbx_file).path_lower def file_upload(self, local_file, dbx_file) -> str: if isinstance(local_file, str): if local_file.startswith("https://"): waiting_time = 0.1 waiting_attempt = 100 url_result = self.dbx.files_save_url(dbx_file, local_file) job_id = url_result.get_async_job_id() while waiting_attempt > 0: st = self.dbx.files_save_url_check_job_status(job_id) if st.is_complete(): return st.get_complete().path_lower sleep(waiting_time) waiting_attempt -= 1 else: with open(local_file, 'rb') as f: return self.dbx.files_upload( f.read(), dbx_file, autorename=True, strict_conflict=True).path_lower else: return self.dbx.files_upload(local_file.read(), dbx_file, autorename=True, strict_conflict=True).path_lower def file_move(self, dbx_file_from, dbx_file_to) -> str: return self.dbx.files_move_v2(dbx_file_from, dbx_file_to).metadata.path_lower def create_folder(self, dbx_folder) -> str: return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower def get_file_tmp_link(self, dbx_path) -> str: return self.dbx.files_get_temporary_link(dbx_path).link
def download_dropboxfiles(payload): # Get the Project project = Project.objects.get(pk=payload['project_id']) project.set_status('downloading') # Check to see what files to download from Dropbox client = Dropbox(project.user.dropboxinfo.access_token) num_files = 0 for x in client.files_list_folder(project.path).entries: if x.path_lower.endswith('.jpg') and x.size > 0: # Download the file from Dropbox to local disk local_filename = os.path.split(x.path_lower)[-1] local_filepath = os.path.join(project.originals_path, local_filename) num_files += 1 if os.path.exists(local_filepath ): # and not payload.get('redownload') == True continue client.files_download_to_file(local_filepath, x.path_lower) # Get the metadata as a separate task new_task(project.user, { 'action': 'extract_metadata', 'project_id': project.pk }) # schedule a thumbnail task new_task(project.user, { 'action': 'makethumbnails', 'project_id': project.pk }) # Downloading files can take a long time # In the meantime this Project could have been changed by other tasks # Reload it before setting the status project = Project.objects.get(pk=payload['project_id']) project.num_files_on_dropbox = num_files project.status = 'layout' project.save() return {'downloaded_files_count': num_files}
def download_dropboxfiles(payload): # Get the Project project = Project.objects.get(pk=payload['project_id']) project.set_status('downloading') # Check to see what files to download from Dropbox client = Dropbox(project.user.dropboxinfo.access_token) num_files = 0 for x in client.files_list_folder(project.path).entries: if x.path_lower.endswith('.jpg') and x.size > 0: # Download the file from Dropbox to local disk local_filename = os.path.split(x.path_lower)[-1] local_filepath = os.path.join(project.originals_path, local_filename) num_files += 1 if os.path.exists(local_filepath): # and not payload.get('redownload') == True continue client.files_download_to_file(local_filepath, x.path_lower) # Get the metadata as a separate task new_task(project.user, { 'action': 'extract_metadata', 'project_id': project.pk }) # schedule a thumbnail task new_task(project.user, { 'action': 'makethumbnails', 'project_id': project.pk }) # Downloading files can take a long time # In the meantime this Project could have been changed by other tasks # Reload it before setting the status project = Project.objects.get(pk=payload['project_id']) project.num_files_on_dropbox = num_files project.status = 'layout' project.save() return {'downloaded_files_count':num_files}
import os from dropbox import Dropbox dbx = Dropbox(os.environ.get('DROPBOX_TOKEN')) os.mkdir('samples/test_data/') dbx.files_download_to_file('samples/test_data/test_image.jpeg', '/annotation_service_test_data/test_image.jpeg')
class DropboxUploader(BaseUploader): def __init__(self): super(BaseUploader, self).__init__() self.oauth2_access_token = os.getenv( "DROPBOX_OAUTH2_TOKEN") or get_app_config("DROPBOX_OAUTH2_TOKEN") self.root_path = (os.getenv("DROPBOX_ROOT_PATH") or get_app_config("DROPBOX_ROOT_PATH") or "/CTFd") self.client = Dropbox(self.oauth2_access_token, timeout=100) self.write_mode = "add" # can be set to overwrite def _clean_filename(self, c): if c in string.ascii_letters + string.digits + "-" + "_" + ".": return True def _full_path(self, name): return safe_join(self.root_path, name).replace("\\", "/") def store(self, fileobj, filename): self.client.files_upload(fileobj.read(), self._full_path(filename), mode=WriteMode(self.write_mode)) return filename def upload(self, file_obj, filename): filename = filter(self._clean_filename, secure_filename(filename).replace(" ", "_")) filename = "".join(filename) if len(filename) <= 0: return False md5hash = hexencode(os.urandom(16)) dst = md5hash + "/" + filename self.store(file_obj, dst) return dst def download(self, filename): media = self.client.files_get_temporary_link(self._full_path(filename)) print(media.link) return redirect(media.link) def delete(self, filename): directory = os.path.dirname(self._full_path(filename)) self.client.files_delete(directory) return True def sync(self): local_folder = current_app.config.get("UPLOAD_FOLDER") root_metadata = self.client.files_list_folder(self.root_path) for folder_entry in root_metadata.entries: if isinstance(folder_entry, FolderMetadata): filemetadata = self.client.files_list_folder( folder_entry.path_lower) for file_entry in filemetadata.entries: if not isinstance(file_entry, FolderMetadata): dropbox_path = file_entry.path_lower.replace( self.root_path.lower() + "/", "") local_path = os.path.join(local_folder, dropbox_path) directory = os.path.dirname(local_path) if not os.path.exists(directory): os.makedirs(directory) self.client.files_download_to_file( local_path, file_entry.path_lower)