def upload(request, path): if request.method == "GET": if 'access_token' in request.session and request.session[ 'access_token']: dbx = Dropbox(request.session['access_token']) if path == '': dirs = dbx.files_list_folder('') else: dirs = dbx.files_list_folder('/' + path) paths = [] for dir in dirs.entries: paths.append(dir.path_display[1:]) request.session['current_path'] = path current_path = request.session['current_path'] return render(request, "myzapier/upload.html", { 'paths': paths, 'current_path': current_path }) if request.method == "POST": dbx = Dropbox(request.session['access_token']) files = request.FILES.getlist('file1') for f in files: if request.session['current_path'] == '': dbx.files_upload(f.read(), path='/' + f.name, autorename=True) else: dbx.files_upload(f.read(), path='/' + request.session['current_path'] + '/' + f.name, autorename=True) messages.success(request, 'Upload Done!.') return HttpResponse("Upload Done!")
def index(request): context = {'data': [], 'files': [], 'errors': []} try: integration = Integration.objects.get(user=request.user) db = Dropbox(integration.access_token) except Integration.DoesNotExist as e: db = None context['errors'].append(e.__str__()) if db: try: db.files_create_folder(PATH) except ApiError as e: context['errors'].append(e.error) except BadInputError as e: context['errors'].append(e.body) try: entries = db.files_list_folder(PATH).entries for each in entries: context['files'].append(each.name) except ApiError as e: context['errors'].append(e.error) except BadInputError as e: context['errors'].append(e.body) return render(request, 'index.html', {'context': context})
def disconnect(request): if request.method == 'POST': logger.info('Disconnect Dropbox.com requested by user...') try: dropbox_token = DropboxUserToken.objects.get(user=request.user) dropbox = Dropbox(dropbox_token.access_token) dropbox.auth_token_revoke() dropbox_user_token = request.user.dropbox_user_token dropbox_user_token.delete() except AuthError: dropbox_user_token = request.user.dropbox_user_token dropbox_user_token.delete() except DropboxUserToken.DoesNotExist: logger.warn('Disconnect Dropbox; DropboxUserToken does not exist.', extra={'user': request.user}) except: logger.error('Disconnect Dropbox; DropboxUserToken delete error.', extra={'user': request.user}) messages.success( request, 'Your Dropbox.com account has been disconnected from DesignSafe.') return HttpResponseRedirect(reverse('dropbox_integration:index')) return render(request, 'designsafe/apps/dropbox_integration/disconnect.html')
def imp(dapi, gapi, path, title): client = Dropbox(dapi) resp = client.file_requests_create(title, path) req = requests.post("{}?key={}".format(GOOGLE_URL, gapi), json={"longUrl": resp.url}) print("File request URL is:", req.json()["id"])
def main(): dbx = Dropbox( 'mNkdY-ZvlnAAAAAAAAAAJdViEYG10v_XnP1Eil4E5z4kF9pEL1cW2TyKFuMKYxxi') photo_directory = os.getcwd() + os.sep + 'drop_box_files' result = dbx.files_list_folder('', True, False, True) for entry in result.entries: process_entry(dbx, entry, photo_directory)
def auth_callback(self, user): # TODO: consider not using client library during auth flow try: access_token, dropbox_user_id, url_state = self.oauth_flow.finish(request.values) except (DropboxOAuth2Flow.NotApprovedException, DropboxOAuth2Flow.BadStateException): # 1) user cancelled and client library raised exc., or # 2) the state was manipulated, possibly due to time. # Either way, return and display info about how to properly connect. return except (DropboxOAuth2Flow.ProviderException, DropboxOAuth2Flow.CsrfException): raise HTTPError(http.FORBIDDEN) except DropboxOAuth2Flow.BadRequestException: raise HTTPError(http.BAD_REQUEST) self.client = Dropbox(access_token) info = self.client.users_get_current_account() return self._set_external_account( user, { 'key': access_token, 'provider_id': info.account_id, 'display_name': info.name.display_name, } )
def revoke(request): try: integration = Integration.objects.get(user=request.user) db = Dropbox(integration.access_token) db.auth_token_revoke() integration.delete() except Integration.DoesNotExist: pass return HttpResponseRedirect(reverse('index'))
def revoke_remote_oauth_access(self, external_account): """Overrides default behavior during external_account deactivation. Tells Dropbox to remove the grant for the OSF associated with this account. """ client = Dropbox(external_account.oauth_key) try: client.auth_token_revoke() except DropboxException: pass
def get_folders(self, **kwargs): folder_id = kwargs.get('folder_id') if folder_id is None: return [{ 'addon': 'dropbox', 'id': '/', 'path': '/', 'kind': 'folder', 'name': '/ (Full Dropbox)', 'urls': { 'folders': api_v2_url('nodes/{}/addons/dropbox/folders/'.format(self.owner._id), params={'id': '/'} ) } }] client = Dropbox(self.external_account.oauth_key) try: folder_id = '' if folder_id == '/' else folder_id list_folder = client.files_list_folder(folder_id) contents = [x for x in list_folder.entries] while list_folder.has_more: list_folder = client.files_list_folder_continue(list_folder.cursor) contents += [x for x in list_folder.entries] except ApiError as error: raise HTTPError(http.BAD_REQUEST, data={ 'message_short': error.user_message_text, 'message_long': error.user_message_text, }) except DropboxException: raise HTTPError(http.BAD_REQUEST) return [ { 'addon': 'dropbox', 'kind': 'folder', 'id': item.path_display, 'name': item.path_display.split('/')[-1], 'path': item.path_display, 'urls': { 'folders': api_v2_url('nodes/{}/addons/dropbox/folders/'.format(self.owner._id), params={'id': item.path_display} ) } } for item in contents if isinstance(item, FolderMetadata) ]
def __init__(self, api_token, folder_name='photos-picker'): """ Constructor :param str api_token: Dropbox api token :param str folder_name: Dropbox folder name """ super(DropboxUploader, self).__init__('/' + folder_name) if not re.match('^/[A-Za-z0-9._-]+$', self._path): raise UploaderException( UploaderException.INVALID_DIR_NAME, "Invalid dir name {dirname}".format(dirname=self._path) ) self._dbx = Dropbox(api_token)
def credentials_are_valid(self, user_settings, client): if user_settings: client = client or Dropbox(user_settings.external_accounts[0].oauth_key) try: client.users_get_current_account() except (AssertionError, DropboxException): return False return True
class DropboxUploader(AbstractUploader): """Upload picked photo to Dropbox""" def __init__(self, api_token, folder_name='photos-picker'): """ Constructor :param str api_token: Dropbox api token :param str folder_name: Dropbox folder name """ super(DropboxUploader, self).__init__('/' + folder_name) if not re.match('^/[A-Za-z0-9._-]+$', self._path): raise UploaderException( UploaderException.INVALID_DIR_NAME, "Invalid dir name {dirname}".format(dirname=self._path) ) self._dbx = Dropbox(api_token) def initialize(self): """Check name and clear remote directory""" # Clear application directory try: self._dbx.files_delete_v2(self._path) except ApiError as e: if not isinstance(e.error, DeleteError) \ or not e.error.is_path_lookup(): raise e def upload(self, binary, original_filename): """ Upload or copy files to destination :param str binary : binary data to upload :param str original_filename: original file name """ # Upload file path = "{root_dir}/{photo_name}".format( root_dir=self._path, photo_name=self._build_filename(original_filename) ) self._dbx.files_upload(binary, path)
def __init__( self, documents: List[Text], access_token: Text, upload_folder: Text, ) -> None: self._paths = [ os.path.realpath(os.path.expanduser(document)) for document in documents ] self._dropbox = Dropbox(access_token) self._upload_folder = upload_folder
def index(request): context = {} try: dropbox_token = DropboxUserToken.objects.get(user=request.user) context['dropbox_enabled'] = True try: dropbox = Dropbox(dropbox_token.access_token) dropbox_user = dropbox.users_get_account(dropbox_token.account_id) context['dropbox_connection'] = dropbox_user except (AuthError, BadRequestException): # authentication failed logger.warning( 'Dropbox oauth token for user=%s failed to authenticate' % request.user.username) context['dropbox_connection'] = False except DropboxUserToken.DoesNotExist: logger.debug('DropboxUserToken does not exist for user=%s' % request.user.username) return render(request, 'designsafe/apps/dropbox_integration/index.html', context)
def __init__(self, user_obj, **kwargs): self._user = user_obj if user_obj.is_anonymous(): raise ApiException( status=403, message='Log in required to access Dropbox files.') try: # self.dropbox_api = user_obj.dropbox_user_token.client dropbox_token = DropboxUserToken.objects.get(user=user_obj) self.dropbox_api = Dropbox(dropbox_token.access_token) except DropboxUserToken.DoesNotExist: message = 'Connect your Dropbox account <a href="' + reverse( 'dropbox_integration:index') + '">here</a>' raise ApiException(status=400, message=message, extra={ 'action_url': reverse('dropbox_integration:index'), 'action_label': 'Connect Dropbox.com Account' })
def dropbox_sync(request): try: integration = Integration.objects.get(user=request.user) db = Dropbox(integration.access_token) except Integration.DoesNotExist as e: db = None if db: file_name = 'feedback.csv' satisfaction_ratings = SatisfactionRating.objects.all() rows = [(each.customer_name, each.score) for each in satisfaction_ratings] with open(file_name, 'w') as csvfile: writer = csv.writer(csvfile, delimiter=',') writer.writerow(['Customer Name', 'Score']) writer.writerows(rows) with open(file_name, 'rb') as f: db.files_upload(f.read(), PATH + '/' + file_name, mode=WriteMode('overwrite')) return HttpResponseRedirect(reverse('index'))
def dropbox_upload_export(access_token, local_path, remote_fname): client = Dropbox(access_token) with open(local_path, 'rb') as inf: chunk_size = int(1.5e8 - 1) next_bytes = inf.read(chunk_size) upload_session_result = client.files_upload_session_start(next_bytes) uploaded_so_far = len(next_bytes) next_bytes = inf.read(chunk_size) cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) while next_bytes: client.files_upload_session_append_v2(next_bytes, cursor) next_bytes = inf.read(chunk_size) uploaded_so_far += chunk_size cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def dropbox_upload_export(access_token, local_path, remote_fname): client = Dropbox(access_token) with open(local_path, 'rb') as inf: chunk_size = int(1.5e8 - 1) next_bytes = inf.read(chunk_size) upload_session_result = client.files_upload_session_start(next_bytes) uploaded_so_far = len(next_bytes) next_bytes = inf.read(chunk_size) cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) while next_bytes: client.files_upload_session_append_v2(next_bytes, cursor) next_bytes = inf.read(chunk_size) uploaded_so_far += chunk_size cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far) client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def action(request, action_name, path): if 'access_token' in request.session and request.session['access_token']: dbx = Dropbox(request.session['access_token']) if action_name == 'get_dirs': '''This part gets the valid directories in form of choices for the move and copy operations.''' dirs = dbx.files_list_folder('', recursive=True) paths = [] for dir in dirs.entries: paths.append(dir.path_display[1:]) paths = filter(lambda s: not '.' in s, paths) dirs = [] if not '.' in path: subpaths = [] subdirs = dbx.files_list_folder('/' + path, recursive=True) for subdir in subdirs.entries: subpaths.append(subdir.path_display[1:]) paths = list(set(paths) - set(subpaths)) if request.session['current_path'] != '': paths.remove(request.session['current_path']) try: path.split('/')[1] except IndexError: pass else: dirs.append({'value': '', 'text': 'Home'}) for path_name in paths: dir = {} dir['value'] = path_name dir['text'] = path_name dirs.append(dir) return HttpResponse(json.dumps(dirs)) path = '/' + path f_name = path.split('/').pop() if action_name == 'search': results = dbx.files_search(path='', query=request.GET['term']) matches = [] for count, res in enumerate(results.matches): match = {} match['id'] = count match['label'] = res.metadata.path_display match['value'] = reverse( 'upload', kwargs={'path': res.metadata.path_display[1:]}) matches.append(match) return HttpResponse(json.dumps(matches)) if action_name == "create_dir": if request.session['current_path'] == '': dbx.files_create_folder(path=path + request.GET['dir_name'], autorename=True) else: dbx.files_create_folder(path=path + '/' + request.GET['dir_name'], autorename=True) msg = 'New Directory %s is created!.' % request.GET['dir_name'] messages.success(request, msg) data = { 'success': True, 'msg': msg, 'redirect_url': reverse('upload', kwargs={'path': request.session['current_path']}) } return HttpResponse(json.dumps(data)) elif action_name == "download": link = dbx.files_get_temporary_link(path).link return HttpResponseRedirect(link) elif action_name == "delete": dbx.files_delete(path) messages.success(request, '%s is deleted!.' % path) return redirect('upload', path=request.session['current_path']) elif action_name == "move": if request.GET['to_path'] != '': to = '/' + request.GET['to_path'] + '/' + f_name else: to = '/' + f_name dbx.files_move(path, to, autorename=True) msg = '%s is moved to %s!.' % (path, to) messages.success(request, msg) data = { 'success': True, 'msg': msg, 'redirect_url': reverse('upload', kwargs={'path': request.GET['to_path']}) } return HttpResponse(json.dumps(data)) elif action_name == "copy": if request.GET['to_path'] != '': to = '/' + request.GET['to_path'] + '/' + f_name else: to = '/' + f_name dbx.files_copy(path, to, autorename=True) msg = '%s is copied to %s!.' % (path, to) messages.success(request, msg) data = { 'success': True, 'msg': msg, 'redirect_url': reverse('upload', kwargs={'path': request.GET['to_path']}) } return HttpResponse(json.dumps(data))
def initialize(self, bot_handler: Any) -> None: self.config_info = bot_handler.get_config_info('dropbox_share') self.ACCESS_TOKEN = self.config_info.get('access_token') self.client = Dropbox(self.ACCESS_TOKEN)
class DropBoxSearchableStorage(Storage): def __init__(self, *args, **kwargs): self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN) self.location = kwargs.get('location', settings.MEDIA_ROOT) def path(self, name): return safe_join(self.location, name) def exists(self, name): try: return isinstance( self.client.files_get_metadata(self.path(name)), (FileMetadata, FolderMetadata) ) except Exception: return False def listdir(self, name): result = self.client.files_list_folder(self.path(name)) return self._list_from_contents(self.path(name), result.entries) def _list_from_contents(self, path, contents): directories, files = [], [] for entry in contents: if isinstance(entry, FileMetadata): files.append(entry.name) if isinstance(entry, FolderMetadata): directories.append(entry.name) return (directories, files) def open(self, name, mode='rb'): meta, resp = self.client.files_download(self.path(name)) return resp.raw def size(self, name): return self.client.files_get_metadata(self.path(name)).size def url(self, name): url = self.client.files_get_temporary_link(self.path(name)).link return url def search(self, query, name='', start=0): result = self.client.files_search(self.path(name), query, start) directories, files = [], [] for entry in result.matches: if isinstance(entry.metadata, FileMetadata): p = entry.metadata.path_display[len(self.location):] files.append(p) # Ignore directories for now return (directories, files) def preview(self, name, size='w128h128'): file_ = StringIO(self.client.files_get_thumbnail( self.path(name), format=ThumbnailFormat('jpeg', None), size=ThumbnailSize(size, None) )[1].content) return resize_thumb(file_, size) def can_preview(self, name): return ( '.' in name and name.rsplit('.', 1)[1].lower() in ( 'jpeg', 'jpg', 'gif', 'png', 'pdf', 'tif', 'tiff', 'mov', 'avi', 'doc', 'mpg', 'bmp', 'psd' ) )
def get_dropbox_object(provider): db = Dropbox(provider.access_token) return db
def __init__(self, provider_token): self.dropbox = Dropbox(provider_token)
def __init__(self, *args, **kwargs): self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN) self.location = kwargs.get('location', settings.MEDIA_ROOT)
class Provider(ExternalProvider): name = 'Dropbox' short_name = 'dropbox' client_id = settings.DROPBOX_KEY client_secret = settings.DROPBOX_SECRET # Explicitly override auth_url_base as None -- DropboxOAuth2Flow handles this for us auth_url_base = None callback_url = None handle_callback = None @property def oauth_flow(self): if 'oauth_states' not in session.data: session.data['oauth_states'] = {} if self.short_name not in session.data['oauth_states']: session.data['oauth_states'][self.short_name] = { 'state': None } return DropboxOAuth2Flow( self.client_id, self.client_secret, redirect_uri=web_url_for( 'oauth_callback', service_name=self.short_name, _absolute=True ), session=session.data['oauth_states'][self.short_name], csrf_token_session_key='state' ) @property def auth_url(self): return self.oauth_flow.start('force_reapprove=true') # Overrides ExternalProvider def auth_callback(self, user): # TODO: consider not using client library during auth flow try: access_token, dropbox_user_id, url_state = self.oauth_flow.finish(request.values) except (DropboxOAuth2Flow.NotApprovedException, DropboxOAuth2Flow.BadStateException): # 1) user cancelled and client library raised exc., or # 2) the state was manipulated, possibly due to time. # Either way, return and display info about how to properly connect. return except (DropboxOAuth2Flow.ProviderException, DropboxOAuth2Flow.CsrfException): raise HTTPError(http.FORBIDDEN) except DropboxOAuth2Flow.BadRequestException: raise HTTPError(http.BAD_REQUEST) self.client = Dropbox(access_token) info = self.client.users_get_current_account() return self._set_external_account( user, { 'key': access_token, 'provider_id': info.account_id, 'display_name': info.name.display_name, } )
def __init__(self): """Create connection and local,remote folders.""" self.svc = DropboxService(settings.FARMBOX_DROPBOX_ACCESS_TOKEN) create_media_dir() self.create_remote_folders()
class DropBoxSearchableStorage(Storage): def __init__(self, *args, **kwargs): self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN) self.location = kwargs.get('location', settings.MEDIA_ROOT) def path(self, name): return safe_join(self.location, name) def exists(self, name): try: return isinstance(self.client.files_get_metadata(self.path(name)), (FileMetadata, FolderMetadata)) except Exception: return False def listdir(self, name): result = self.client.files_list_folder(self.path(name)) return self._list_from_contents(self.path(name), result.entries) def _list_from_contents(self, path, contents): directories, files = [], [] for entry in contents: if isinstance(entry, FileMetadata): files.append(entry.name) if isinstance(entry, FolderMetadata): directories.append(entry.name) return (directories, files) def open(self, name, mode='rb'): meta, resp = self.client.files_download(self.path(name)) return resp.raw def size(self, name): return self.client.files_get_metadata(self.path(name)).size def url(self, name): url = self.client.files_get_temporary_link(self.path(name)).link return url def search(self, query, name='', start=0): result = self.client.files_search(self.path(name), query, start) directories, files = [], [] for entry in result.matches: if isinstance(entry.metadata, FileMetadata): p = entry.metadata.path_display[len(self.location):] files.append(p) # Ignore directories for now return (directories, files) def preview(self, name, size='w128h128'): file_ = StringIO( self.client.files_get_thumbnail( self.path(name), format=ThumbnailFormat('jpeg', None), size=ThumbnailSize(size, None))[1].content) return resize_thumb(file_, size) def can_preview(self, name): return ('.' in name and name.rsplit('.', 1)[1].lower() in ('jpeg', 'jpg', 'gif', 'png', 'pdf', 'tif', 'tiff', 'mov', 'avi', 'doc', 'mpg', 'bmp', 'psd'))
def __init__(self, *args, **kwargs): self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN) self.location = kwargs.get('location', settings.MEDIA_ROOT)
class FileManager(object): NAME = 'dropbox' def __init__(self, user_obj, **kwargs): self._user = user_obj if user_obj.is_anonymous(): raise ApiException( status=403, message='Log in required to access Dropbox files.') try: # self.dropbox_api = user_obj.dropbox_user_token.client dropbox_token = DropboxUserToken.objects.get(user=user_obj) self.dropbox_api = Dropbox(dropbox_token.access_token) except DropboxUserToken.DoesNotExist: message = 'Connect your Dropbox account <a href="' + reverse( 'dropbox_integration:index') + '">here</a>' raise ApiException(status=400, message=message, extra={ 'action_url': reverse('dropbox_integration:index'), 'action_label': 'Connect Dropbox.com Account' }) def parse_file_id(self, path): if path == '/' or path == '': file_type, path = 'folder', '' else: try: file_type, path = DropboxFile.parse_file_id(path) except AssertionError: # file path is hierarchical; need to find the DropboxObject here if path[:1] != '/': path = '/' + path try: dropbox_item = DropboxFile( self.dropbox_api.files_list_folder(path)) return dropbox_item.type, path except ApiError as e: if e.error.get_path().is_not_found(): raise ApiException( 'The Dropbox path "{0}" does not exist.'.format( path), status=404) return file_type, path def listing(self, file_id='', **kwargs): """ Lists contents of a folder or details of a file. Args: file_id: The type/path of the Dropbox Object. This should be formatted {type}/{path} where {type} is one of ['folder', 'file'] and {path} is the numeric Dropbox path for the object. Returns: Dictionary with two keys; - resource: File System resource that we're listing - files: Array of Api file-like objects """ default_pems = 'ALL' try: file_type, path = self.parse_file_id(file_id) dropbox_item = self.dropbox_api.files_list_folder(path) children = [] if file_type == 'folder': has_more = dropbox_item.has_more cursor = dropbox_item.cursor entries = dropbox_item.entries while True: children.extend([ DropboxFile(item, item.path_display, parent=dropbox_item).to_dict( default_pems=default_pems) for item in entries ]) if has_more: folder = self.dropbox_api.files_list_folder_continue( cursor) entries = folder.entries has_more = folder.has_more cursor = folder.cursor else: break else: children = None list_data = DropboxFile(dropbox_item, path).to_dict(default_pems=default_pems) if children: list_data['children'] = children return list_data except AssertionError: raise ApiException( status=404, message='The file/folder you requested does not exist.') except AuthError: # user needs to reconnect with Dropbox message = 'While you previously granted this application access to Dropbox, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \ 'to continue using Dropbox data.' % reverse('dropbox_integration:index') raise ApiException(status=403, message=message) except ApiError as e: if e.error.get_path().is_not_folder(): dropbox_item = self.dropbox_api.files_get_metadata(path) list_data = DropboxFile(dropbox_item, path) return list_data message = 'Unable to communicate with Dropbox: %s' % e.message raise ApiException(status=500, message=message) def file(self, file_id, action, path=None, **kwargs): pass def is_shared(self, *args, **kwargs): return False def is_search(self, *args, **kwargs): return False def copy(self, username, src_file_id, dest_file_id, **kwargs): try: n = Notification( event_type='data', status=Notification.INFO, operation='dropbox_download_start', message='Starting download file %s from dropbox.' % (src_file_id, ), user=username, extra={}) n.save() logger.debug( 'username: {}, src_file_id: {}, dest_file_id: {}'.format( username, src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) file_type, path = self.parse_file_id(src_file_id) levels = 0 downloaded_file_path = None if file_type == 'file': downloaded_file_path = self.download_file(path, dest_real_path) levels = 1 elif file_type == 'folder': downloaded_file_path = self.download_folder( path, dest_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='dropbox_download_end', message='File %s has been copied from dropbox successfully!' % (src_file_id, ), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace( project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace( base_mounted_path, '', 1).strip('/') if not agave_file_path.startswith('/'): agave_file_path = '/' + agave_file_path agave_indexer.apply_async(kwargs={ 'username': user.username, 'systemId': agave_system_id, 'filePath': os.path.dirname(agave_file_path), 'recurse': False }, queue='indexing') agave_indexer.apply_async(kwargs={ 'systemId': agave_system_id, 'filePath': agave_file_path, 'recurse': True }, routing_key='indexing') except: logger.exception('Unexpected task failure: dropbox_download', extra={ 'username': username, 'box_file_id': src_file_id, 'dest_file_id': dest_file_id }) n = Notification( event_type='data', status=Notification.ERROR, operation='dropbox_download_error', message='We were unable to get the specified file from dropbox. ' 'Please try again...', user=username, extra={}) n.save() raise def move(self, file_id, **kwargs): raise ApiException('Moving Dropbox files is not supported.', status=400, extra={ 'file_id': file_id, 'kwargs': kwargs }) def get_preview_url(self, file_id, **kwargs): file_type, path = self.parse_file_id(file_id) if file_type == 'file': shared_link = self.dropbox_api.sharing_create_shared_link(path).url embed_url = re.sub('dl=0$', 'raw=1', shared_link) return {'href': embed_url} return None def preview(self, file_id, file_mgr_name, **kwargs): try: file_type, path = self.parse_file_id(file_id) dropbox_item = self.dropbox_api.files_get_metadata(path) dropbox_file = DropboxFile(dropbox_item, path) if dropbox_file.previewable: preview_url = reverse('designsafe_api:box_files_media', args=[file_mgr_name, file_id.strip('/')]) return JsonResponse( {'href': '{}?preview=true'.format(preview_url)}) else: return HttpResponseBadRequest( 'Preview not available for this item.') except HTTPError as e: logger.exception('Unable to preview file') return HttpResponseBadRequest(e.response.text) def get_download_url(self, file_id, **kwargs): file_type, path = self.parse_file_id(file_id) if file_type == 'file': download_url = self.dropbox_api.files_get_temporary_link(path) return {'href': download_url.link} return None #def import_file(self, file_id, from_resource, import_file_id, **kwargs): # dropbox_upload.apply_async(args=(self._user.username, # file_id, # from_resource, # import_file_id), # countdown=10) # return {'message': 'Your file(s) have been scheduled for upload to box.'} def download_file(self, dropbox_file_path, download_directory_path): """ Downloads the file for dropbox_file_path to the given download_path. :param dropbox_file_path: :param download_directory_path: :return: the full path to the downloaded file """ dropbox_file = self.dropbox_api.files_get_metadata(dropbox_file_path) # convert utf-8 chars safe_filename = dropbox_file.name.encode(sys.getfilesystemencoding(), 'ignore') file_download_path = os.path.join(download_directory_path, safe_filename) logger.debug('Download file %s <= dropbox://file/%s', file_download_path, dropbox_file_path) self.dropbox_api.files_download_to_file(file_download_path, dropbox_file_path) return file_download_path def download_folder(self, path, download_path): """ Recursively download the folder for path, and all of its contents, to the given download_path. :param path: :param download_path: :return: """ dropbox_folder = self.dropbox_api.files_list_folder(path) has_more = dropbox_folder.has_more cursor = dropbox_folder.cursor dropbox_folder_metadata = self.dropbox_api.files_alpha_get_metadata( path) # convert utf-8 chars safe_dirname = dropbox_folder_metadata.name.encode( sys.getfilesystemencoding(), 'ignore') directory_path = os.path.join(download_path, safe_dirname) logger.debug('Creating directory %s <= dropbox://folder/%s', directory_path, path) try: os.mkdir(directory_path, 0o0755) except OSError as e: if e.errno == 17: # directory already exists? pass else: logger.exception('Error creating directory: %s', directory_path) raise items = dropbox_folder.entries while True: for item in items: if type(item) == FileMetadata: self.download_file(item.path_lower, directory_path) elif type(item) == FolderMetadata: self.download_folder(item.path_lower, directory_path) if has_more: folder = self.dropbox_api.files_list_folder_continue(cursor) items = folder.entries has_more = items.has_more cursor = items.cursor else: break return directory_path def upload(self, username, src_file_id, dest_file_id): try: n = Notification(event_type='data', status=Notification.INFO, operation='dropbox_upload_start', message='Starting uploading file %s to dropbox.' % (src_file_id, ), user=username, extra={}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split src ination file path src_file_path_comps = src_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = src_file_path_comps[0] # Start construction the actual real path into the NSF mount if src_file_path_comps[1:]: src_real_path = os.path.join(*src_file_path_comps[1:]) else: src_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) src_real_path = os.path.join(base_mounted_path, project_dir, src_real_path.strip('/')) else: src_real_path = os.path.join(base_mounted_path, src_real_path.strip('/')) logger.debug('src_real_path: {}'.format(src_real_path)) file_type, path = self.parse_file_id(dest_file_id) if os.path.isfile(src_real_path): self.upload_file(path, src_real_path) elif os.path.isdir(src_real_path): self.upload_directory(path, src_real_path) else: logger.error('Unable to upload %s: file does not exist!', src_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='dropbox_upload_end', message='File %s has been copied to dropbox successfully!' % (src_file_id, ), user=username, extra={}) n.save() except Exception as err: logger.exception('Unexpected task failure: dropbox_upload', extra={ 'username': username, 'src_file_id': src_file_id, 'dst_file_id': dest_file_id }) n = Notification( event_type='data', status=Notification.ERROR, operation='dropbox_upload_error', message='We were unable to get the specified file from dropbox. ' 'Please try again...', user=username, extra={}) n.save() raise def upload_file(self, dropbox_path, file_real_path): file_path, file_name = os.path.split(file_real_path) with open(file_real_path, 'rb') as file_handle: f = file_handle.read() file_size = os.path.getsize(file_path) CHUNK_SIZE = 4 * 1024 * 1024 # 4MB if file_size <= CHUNK_SIZE: logger.debug('dropbox_path: %s, file_name: %s', dropbox_path, file_name), self.dropbox_api.files_upload( f, '%s/%s' % (dropbox_path, file_name)) else: upload_session_start_result = self.dropbox_api.files_upload_session_start( f.read(CHUNK_SIZE)) cursor = dropbox.files.UploadSessionCursor( session_id=upload_session_start_result.session_id, offset=f.tell()) commit = dropbox.files.CommitInfo(path=file_path) while f.tell() < file_size: if ((file_size - f.tell()) <= CHUNK_SIZE): print( self.dropbox_api.files_upload_session_finish( f.read(CHUNK_SIZE), cursor, commit)) else: self.dropbox_api.files_upload_session_append( f.read(CHUNK_SIZE), cursor.session_id, cursor.offset) cursor.offset = f.tell() logger.info('Successfully uploaded %s to dropbox:folder/%s', file_real_path, file_path) def upload_directory(self, dropbox_parent_folder, dir_real_path): """ Recursively uploads the directory and all of its contents (subdirectories and files) to the dropbox folder specified by dropbox_parent_folder. :param dropbox_parent_folder: The dropbox folder to upload the directory to. :param dir_real_path: The real path on the filesystem of the directory to upload. :return: """ dirparentpath, dirname = os.path.split(dir_real_path) # dropbox_parent_folder = self.dropbox_api.folder(dropbox_parent_folder) logger.info('Create directory %s in dropbox folder/%s', dirname, dropbox_parent_folder) # box_folder = box_parent_folder.create_subfolder(dirname) for dirpath, subdirnames, filenames in os.walk(dir_real_path): # upload all the files for filename in filenames: filepath = os.path.join(dirpath, filename) self.upload_file('%s/%s' % (dropbox_parent_folder, dirname), filepath) # upload all the subdirectories for subdirname in subdirnames: subdirpath = os.path.join(dirpath, subdirname) self.upload_directory( '%s/%s' % (dropbox_parent_folder, dirname), subdirpath) # prevent further walk, because recursion subdirnames[:] = []
class DropboxApp: """ File operations for dropbox. See: https://dropbox-sdk-python.readthedocs.io/en/latest/index.html """ svc: DropboxService def __init__(self): """Create connection and local,remote folders.""" self.svc = DropboxService(settings.FARMBOX_DROPBOX_ACCESS_TOKEN) create_media_dir() self.create_remote_folders() def create_remote_folders(self): """Create root subfolders if they don't exist.""" subfolder_exists = {} for sub in settings.CLOUD_SUBFOLDERS: subfolder_exists[sub] = False for c in self.list_contents(): if isinstance(c, FolderMetadata) and c.name == sub: subfolder_exists[sub] = True for k, v in subfolder_exists.items(): if not v: self.svc.files_create_folder_v2(f"/{k}") def remove_remote_db(self) -> bool: path = settings.DATABASES["sqlite"]["NAME"] dest = f"/{os.path.join(settings.BACKUP_DB_FOLDER,os.path.basename(path))}" self.remove(dest) return True def backup_db(self): """Backup local db dump.""" src = settings.DATABASES["sqlite"]["NAME"] dest = f"/{settings.BACKUP_DB_FOLDER}" self.upload(src, dest) def restore_db(self): """Restore remote db dump""" dest = settings.DATABASES["sqlite"]["NAME"] src = f"/{os.path.join(settings.BACKUP_DB_FOLDER,os.path.basename(dest))}" self.svc.files_download_to_file(dest, src, rev=None) def upload(self, src, dest=""): """Upload bytes to dropbox.""" filename = os.path.basename(src) dest = f"{dest}/{filename}" with open(src, "rb") as f: self.svc.files_upload(f.read(), dest, mute=True) def list_contents(self, path=""): """Lists folder contents. :param str path: If empty, it's the default dropbox app folder. `/processed` is the default app's `processed` subfolder """ return self.svc.files_list_folder(path).entries def download_all_as_zip(self, local_path, remote_path) -> List[FileMetadata]: """Downloads all files to instance storage""" self.svc.files_download_zip_to_file(local_path, remote_path) return self.list_contents(remote_path) def remove(self, path: str): self.svc.files_delete_v2(path) def move_to_processed_dir(self, ext_ref, dest_path): pass