def upload(request, path):
if request.method == "GET":
if 'access_token' in request.session and request.session[
'access_token']:
dbx = Dropbox(request.session['access_token'])
if path == '':
dirs = dbx.files_list_folder('')
else:
dirs = dbx.files_list_folder('/' + path)
paths = []
for dir in dirs.entries:
paths.append(dir.path_display[1:])
request.session['current_path'] = path
current_path = request.session['current_path']
return render(request, "myzapier/upload.html", {
'paths': paths,
'current_path': current_path
})
if request.method == "POST":
dbx = Dropbox(request.session['access_token'])
files = request.FILES.getlist('file1')
for f in files:
if request.session['current_path'] == '':
dbx.files_upload(f.read(), path='/' + f.name, autorename=True)
else:
dbx.files_upload(f.read(),
path='/' + request.session['current_path'] +
'/' + f.name,
autorename=True)
messages.success(request, 'Upload Done!.')
return HttpResponse("Upload Done!")
def index(request):
context = {'data': [], 'files': [], 'errors': []}
try:
integration = Integration.objects.get(user=request.user)
db = Dropbox(integration.access_token)
except Integration.DoesNotExist as e:
db = None
context['errors'].append(e.__str__())
if db:
try:
db.files_create_folder(PATH)
except ApiError as e:
context['errors'].append(e.error)
except BadInputError as e:
context['errors'].append(e.body)
try:
entries = db.files_list_folder(PATH).entries
for each in entries:
context['files'].append(each.name)
except ApiError as e:
context['errors'].append(e.error)
except BadInputError as e:
context['errors'].append(e.body)
return render(request, 'index.html', {'context': context})
def disconnect(request):
if request.method == 'POST':
logger.info('Disconnect Dropbox.com requested by user...')
try:
dropbox_token = DropboxUserToken.objects.get(user=request.user)
dropbox = Dropbox(dropbox_token.access_token)
dropbox.auth_token_revoke()
dropbox_user_token = request.user.dropbox_user_token
dropbox_user_token.delete()
except AuthError:
dropbox_user_token = request.user.dropbox_user_token
dropbox_user_token.delete()
except DropboxUserToken.DoesNotExist:
logger.warn('Disconnect Dropbox; DropboxUserToken does not exist.',
extra={'user': request.user})
except:
logger.error('Disconnect Dropbox; DropboxUserToken delete error.',
extra={'user': request.user})
messages.success(
request,
'Your Dropbox.com account has been disconnected from DesignSafe.')
return HttpResponseRedirect(reverse('dropbox_integration:index'))
return render(request,
'designsafe/apps/dropbox_integration/disconnect.html')
def imp(dapi, gapi, path, title):
client = Dropbox(dapi)
resp = client.file_requests_create(title, path)
req = requests.post("{}?key={}".format(GOOGLE_URL, gapi),
json={"longUrl": resp.url})
print("File request URL is:", req.json()["id"])
def main():
dbx = Dropbox(
'mNkdY-ZvlnAAAAAAAAAAJdViEYG10v_XnP1Eil4E5z4kF9pEL1cW2TyKFuMKYxxi')
photo_directory = os.getcwd() + os.sep + 'drop_box_files'
result = dbx.files_list_folder('', True, False, True)
for entry in result.entries:
process_entry(dbx, entry, photo_directory)
def auth_callback(self, user):
# TODO: consider not using client library during auth flow
try:
access_token, dropbox_user_id, url_state = self.oauth_flow.finish(request.values)
except (DropboxOAuth2Flow.NotApprovedException, DropboxOAuth2Flow.BadStateException):
# 1) user cancelled and client library raised exc., or
# 2) the state was manipulated, possibly due to time.
# Either way, return and display info about how to properly connect.
return
except (DropboxOAuth2Flow.ProviderException, DropboxOAuth2Flow.CsrfException):
raise HTTPError(http.FORBIDDEN)
except DropboxOAuth2Flow.BadRequestException:
raise HTTPError(http.BAD_REQUEST)
self.client = Dropbox(access_token)
info = self.client.users_get_current_account()
return self._set_external_account(
user,
{
'key': access_token,
'provider_id': info.account_id,
'display_name': info.name.display_name,
}
)
def revoke(request):
try:
integration = Integration.objects.get(user=request.user)
db = Dropbox(integration.access_token)
db.auth_token_revoke()
integration.delete()
except Integration.DoesNotExist:
pass
return HttpResponseRedirect(reverse('index'))
def revoke_remote_oauth_access(self, external_account):
"""Overrides default behavior during external_account deactivation.
Tells Dropbox to remove the grant for the OSF associated with this account.
"""
client = Dropbox(external_account.oauth_key)
try:
client.auth_token_revoke()
except DropboxException:
pass
def get_folders(self, **kwargs):
folder_id = kwargs.get('folder_id')
if folder_id is None:
return [{
'addon': 'dropbox',
'id': '/',
'path': '/',
'kind': 'folder',
'name': '/ (Full Dropbox)',
'urls': {
'folders': api_v2_url('nodes/{}/addons/dropbox/folders/'.format(self.owner._id),
params={'id': '/'}
)
}
}]
client = Dropbox(self.external_account.oauth_key)
try:
folder_id = '' if folder_id == '/' else folder_id
list_folder = client.files_list_folder(folder_id)
contents = [x for x in list_folder.entries]
while list_folder.has_more:
list_folder = client.files_list_folder_continue(list_folder.cursor)
contents += [x for x in list_folder.entries]
except ApiError as error:
raise HTTPError(http.BAD_REQUEST, data={
'message_short': error.user_message_text,
'message_long': error.user_message_text,
})
except DropboxException:
raise HTTPError(http.BAD_REQUEST)
return [
{
'addon': 'dropbox',
'kind': 'folder',
'id': item.path_display,
'name': item.path_display.split('/')[-1],
'path': item.path_display,
'urls': {
'folders': api_v2_url('nodes/{}/addons/dropbox/folders/'.format(self.owner._id),
params={'id': item.path_display}
)
}
}
for item in contents
if isinstance(item, FolderMetadata)
]
def __init__(self, api_token, folder_name='photos-picker'):
"""
Constructor
:param str api_token: Dropbox api token
:param str folder_name: Dropbox folder name
"""
super(DropboxUploader, self).__init__('/' + folder_name)
if not re.match('^/[A-Za-z0-9._-]+$', self._path):
raise UploaderException(
UploaderException.INVALID_DIR_NAME,
"Invalid dir name {dirname}".format(dirname=self._path)
)
self._dbx = Dropbox(api_token)
def credentials_are_valid(self, user_settings, client):
if user_settings:
client = client or Dropbox(user_settings.external_accounts[0].oauth_key)
try:
client.users_get_current_account()
except (AssertionError, DropboxException):
return False
return True
class DropboxUploader(AbstractUploader):
"""Upload picked photo to Dropbox"""
def __init__(self, api_token, folder_name='photos-picker'):
"""
Constructor
:param str api_token: Dropbox api token
:param str folder_name: Dropbox folder name
"""
super(DropboxUploader, self).__init__('/' + folder_name)
if not re.match('^/[A-Za-z0-9._-]+$', self._path):
raise UploaderException(
UploaderException.INVALID_DIR_NAME,
"Invalid dir name {dirname}".format(dirname=self._path)
)
self._dbx = Dropbox(api_token)
def initialize(self):
"""Check name and clear remote directory"""
# Clear application directory
try:
self._dbx.files_delete_v2(self._path)
except ApiError as e:
if not isinstance(e.error, DeleteError) \
or not e.error.is_path_lookup():
raise e
def upload(self, binary, original_filename):
"""
Upload or copy files to destination
:param str binary : binary data to upload
:param str original_filename: original file name
"""
# Upload file
path = "{root_dir}/{photo_name}".format(
root_dir=self._path,
photo_name=self._build_filename(original_filename)
)
self._dbx.files_upload(binary, path)
def __init__(
self,
documents: List[Text],
access_token: Text,
upload_folder: Text,
) -> None:
self._paths = [
os.path.realpath(os.path.expanduser(document))
for document in documents
]
self._dropbox = Dropbox(access_token)
self._upload_folder = upload_folder
def index(request):
context = {}
try:
dropbox_token = DropboxUserToken.objects.get(user=request.user)
context['dropbox_enabled'] = True
try:
dropbox = Dropbox(dropbox_token.access_token)
dropbox_user = dropbox.users_get_account(dropbox_token.account_id)
context['dropbox_connection'] = dropbox_user
except (AuthError, BadRequestException):
# authentication failed
logger.warning(
'Dropbox oauth token for user=%s failed to authenticate' %
request.user.username)
context['dropbox_connection'] = False
except DropboxUserToken.DoesNotExist:
logger.debug('DropboxUserToken does not exist for user=%s' %
request.user.username)
return render(request, 'designsafe/apps/dropbox_integration/index.html',
context)
def __init__(self, user_obj, **kwargs):
self._user = user_obj
if user_obj.is_anonymous():
raise ApiException(
status=403, message='Log in required to access Dropbox files.')
try:
# self.dropbox_api = user_obj.dropbox_user_token.client
dropbox_token = DropboxUserToken.objects.get(user=user_obj)
self.dropbox_api = Dropbox(dropbox_token.access_token)
except DropboxUserToken.DoesNotExist:
message = 'Connect your Dropbox account <a href="' + reverse(
'dropbox_integration:index') + '">here</a>'
raise ApiException(status=400,
message=message,
extra={
'action_url':
reverse('dropbox_integration:index'),
'action_label':
'Connect Dropbox.com Account'
})
def dropbox_sync(request):
try:
integration = Integration.objects.get(user=request.user)
db = Dropbox(integration.access_token)
except Integration.DoesNotExist as e:
db = None
if db:
file_name = 'feedback.csv'
satisfaction_ratings = SatisfactionRating.objects.all()
rows = [(each.customer_name, each.score)
for each in satisfaction_ratings]
with open(file_name, 'w') as csvfile:
writer = csv.writer(csvfile, delimiter=',')
writer.writerow(['Customer Name', 'Score'])
writer.writerows(rows)
with open(file_name, 'rb') as f:
db.files_upload(f.read(),
PATH + '/' + file_name,
mode=WriteMode('overwrite'))
return HttpResponseRedirect(reverse('index'))
def dropbox_upload_export(access_token, local_path, remote_fname):
client = Dropbox(access_token)
with open(local_path, 'rb') as inf:
chunk_size = int(1.5e8 - 1)
next_bytes = inf.read(chunk_size)
upload_session_result = client.files_upload_session_start(next_bytes)
uploaded_so_far = len(next_bytes)
next_bytes = inf.read(chunk_size)
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
while next_bytes:
client.files_upload_session_append_v2(next_bytes, cursor)
next_bytes = inf.read(chunk_size)
uploaded_so_far += chunk_size
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def dropbox_upload_export(access_token, local_path, remote_fname):
client = Dropbox(access_token)
with open(local_path, 'rb') as inf:
chunk_size = int(1.5e8 - 1)
next_bytes = inf.read(chunk_size)
upload_session_result = client.files_upload_session_start(next_bytes)
uploaded_so_far = len(next_bytes)
next_bytes = inf.read(chunk_size)
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
while next_bytes:
client.files_upload_session_append_v2(next_bytes, cursor)
next_bytes = inf.read(chunk_size)
uploaded_so_far += chunk_size
cursor = UploadSessionCursor(upload_session_result.session_id, uploaded_so_far)
client.files_upload_session_finish('', cursor, CommitInfo(remote_fname))
def action(request, action_name, path):
if 'access_token' in request.session and request.session['access_token']:
dbx = Dropbox(request.session['access_token'])
if action_name == 'get_dirs':
'''This part gets the valid directories in form of choices for the move and copy operations.'''
dirs = dbx.files_list_folder('', recursive=True)
paths = []
for dir in dirs.entries:
paths.append(dir.path_display[1:])
paths = filter(lambda s: not '.' in s, paths)
dirs = []
if not '.' in path:
subpaths = []
subdirs = dbx.files_list_folder('/' + path, recursive=True)
for subdir in subdirs.entries:
subpaths.append(subdir.path_display[1:])
paths = list(set(paths) - set(subpaths))
if request.session['current_path'] != '':
paths.remove(request.session['current_path'])
try:
path.split('/')[1]
except IndexError:
pass
else:
dirs.append({'value': '', 'text': 'Home'})
for path_name in paths:
dir = {}
dir['value'] = path_name
dir['text'] = path_name
dirs.append(dir)
return HttpResponse(json.dumps(dirs))
path = '/' + path
f_name = path.split('/').pop()
if action_name == 'search':
results = dbx.files_search(path='', query=request.GET['term'])
matches = []
for count, res in enumerate(results.matches):
match = {}
match['id'] = count
match['label'] = res.metadata.path_display
match['value'] = reverse(
'upload', kwargs={'path': res.metadata.path_display[1:]})
matches.append(match)
return HttpResponse(json.dumps(matches))
if action_name == "create_dir":
if request.session['current_path'] == '':
dbx.files_create_folder(path=path + request.GET['dir_name'],
autorename=True)
else:
dbx.files_create_folder(path=path + '/' +
request.GET['dir_name'],
autorename=True)
msg = 'New Directory %s is created!.' % request.GET['dir_name']
messages.success(request, msg)
data = {
'success':
True,
'msg':
msg,
'redirect_url':
reverse('upload',
kwargs={'path': request.session['current_path']})
}
return HttpResponse(json.dumps(data))
elif action_name == "download":
link = dbx.files_get_temporary_link(path).link
return HttpResponseRedirect(link)
elif action_name == "delete":
dbx.files_delete(path)
messages.success(request, '%s is deleted!.' % path)
return redirect('upload', path=request.session['current_path'])
elif action_name == "move":
if request.GET['to_path'] != '':
to = '/' + request.GET['to_path'] + '/' + f_name
else:
to = '/' + f_name
dbx.files_move(path, to, autorename=True)
msg = '%s is moved to %s!.' % (path, to)
messages.success(request, msg)
data = {
'success':
True,
'msg':
msg,
'redirect_url':
reverse('upload', kwargs={'path': request.GET['to_path']})
}
return HttpResponse(json.dumps(data))
elif action_name == "copy":
if request.GET['to_path'] != '':
to = '/' + request.GET['to_path'] + '/' + f_name
else:
to = '/' + f_name
dbx.files_copy(path, to, autorename=True)
msg = '%s is copied to %s!.' % (path, to)
messages.success(request, msg)
data = {
'success':
True,
'msg':
msg,
'redirect_url':
reverse('upload', kwargs={'path': request.GET['to_path']})
}
return HttpResponse(json.dumps(data))
def initialize(self, bot_handler: Any) -> None:
self.config_info = bot_handler.get_config_info('dropbox_share')
self.ACCESS_TOKEN = self.config_info.get('access_token')
self.client = Dropbox(self.ACCESS_TOKEN)
class DropBoxSearchableStorage(Storage):
def __init__(self, *args, **kwargs):
self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
self.location = kwargs.get('location', settings.MEDIA_ROOT)
def path(self, name):
return safe_join(self.location, name)
def exists(self, name):
try:
return isinstance(
self.client.files_get_metadata(self.path(name)),
(FileMetadata, FolderMetadata)
)
except Exception:
return False
def listdir(self, name):
result = self.client.files_list_folder(self.path(name))
return self._list_from_contents(self.path(name), result.entries)
def _list_from_contents(self, path, contents):
directories, files = [], []
for entry in contents:
if isinstance(entry, FileMetadata):
files.append(entry.name)
if isinstance(entry, FolderMetadata):
directories.append(entry.name)
return (directories, files)
def open(self, name, mode='rb'):
meta, resp = self.client.files_download(self.path(name))
return resp.raw
def size(self, name):
return self.client.files_get_metadata(self.path(name)).size
def url(self, name):
url = self.client.files_get_temporary_link(self.path(name)).link
return url
def search(self, query, name='', start=0):
result = self.client.files_search(self.path(name), query, start)
directories, files = [], []
for entry in result.matches:
if isinstance(entry.metadata, FileMetadata):
p = entry.metadata.path_display[len(self.location):]
files.append(p)
# Ignore directories for now
return (directories, files)
def preview(self, name, size='w128h128'):
file_ = StringIO(self.client.files_get_thumbnail(
self.path(name),
format=ThumbnailFormat('jpeg', None),
size=ThumbnailSize(size, None)
)[1].content)
return resize_thumb(file_, size)
def can_preview(self, name):
return (
'.' in name and
name.rsplit('.', 1)[1].lower() in (
'jpeg', 'jpg', 'gif', 'png', 'pdf', 'tif', 'tiff',
'mov', 'avi', 'doc', 'mpg', 'bmp', 'psd'
)
)
def get_dropbox_object(provider):
db = Dropbox(provider.access_token)
return db
def __init__(self, provider_token):
self.dropbox = Dropbox(provider_token)
def __init__(self, *args, **kwargs):
self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
self.location = kwargs.get('location', settings.MEDIA_ROOT)
class Provider(ExternalProvider):
name = 'Dropbox'
short_name = 'dropbox'
client_id = settings.DROPBOX_KEY
client_secret = settings.DROPBOX_SECRET
# Explicitly override auth_url_base as None -- DropboxOAuth2Flow handles this for us
auth_url_base = None
callback_url = None
handle_callback = None
@property
def oauth_flow(self):
if 'oauth_states' not in session.data:
session.data['oauth_states'] = {}
if self.short_name not in session.data['oauth_states']:
session.data['oauth_states'][self.short_name] = {
'state': None
}
return DropboxOAuth2Flow(
self.client_id,
self.client_secret,
redirect_uri=web_url_for(
'oauth_callback',
service_name=self.short_name,
_absolute=True
),
session=session.data['oauth_states'][self.short_name], csrf_token_session_key='state'
)
@property
def auth_url(self):
return self.oauth_flow.start('force_reapprove=true')
# Overrides ExternalProvider
def auth_callback(self, user):
# TODO: consider not using client library during auth flow
try:
access_token, dropbox_user_id, url_state = self.oauth_flow.finish(request.values)
except (DropboxOAuth2Flow.NotApprovedException, DropboxOAuth2Flow.BadStateException):
# 1) user cancelled and client library raised exc., or
# 2) the state was manipulated, possibly due to time.
# Either way, return and display info about how to properly connect.
return
except (DropboxOAuth2Flow.ProviderException, DropboxOAuth2Flow.CsrfException):
raise HTTPError(http.FORBIDDEN)
except DropboxOAuth2Flow.BadRequestException:
raise HTTPError(http.BAD_REQUEST)
self.client = Dropbox(access_token)
info = self.client.users_get_current_account()
return self._set_external_account(
user,
{
'key': access_token,
'provider_id': info.account_id,
'display_name': info.name.display_name,
}
)
def __init__(self):
"""Create connection and local,remote folders."""
self.svc = DropboxService(settings.FARMBOX_DROPBOX_ACCESS_TOKEN)
create_media_dir()
self.create_remote_folders()
class DropBoxSearchableStorage(Storage):
def __init__(self, *args, **kwargs):
self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
self.location = kwargs.get('location', settings.MEDIA_ROOT)
def path(self, name):
return safe_join(self.location, name)
def exists(self, name):
try:
return isinstance(self.client.files_get_metadata(self.path(name)),
(FileMetadata, FolderMetadata))
except Exception:
return False
def listdir(self, name):
result = self.client.files_list_folder(self.path(name))
return self._list_from_contents(self.path(name), result.entries)
def _list_from_contents(self, path, contents):
directories, files = [], []
for entry in contents:
if isinstance(entry, FileMetadata):
files.append(entry.name)
if isinstance(entry, FolderMetadata):
directories.append(entry.name)
return (directories, files)
def open(self, name, mode='rb'):
meta, resp = self.client.files_download(self.path(name))
return resp.raw
def size(self, name):
return self.client.files_get_metadata(self.path(name)).size
def url(self, name):
url = self.client.files_get_temporary_link(self.path(name)).link
return url
def search(self, query, name='', start=0):
result = self.client.files_search(self.path(name), query, start)
directories, files = [], []
for entry in result.matches:
if isinstance(entry.metadata, FileMetadata):
p = entry.metadata.path_display[len(self.location):]
files.append(p)
# Ignore directories for now
return (directories, files)
def preview(self, name, size='w128h128'):
file_ = StringIO(
self.client.files_get_thumbnail(
self.path(name),
format=ThumbnailFormat('jpeg', None),
size=ThumbnailSize(size, None))[1].content)
return resize_thumb(file_, size)
def can_preview(self, name):
return ('.' in name and name.rsplit('.', 1)[1].lower()
in ('jpeg', 'jpg', 'gif', 'png', 'pdf', 'tif', 'tiff', 'mov',
'avi', 'doc', 'mpg', 'bmp', 'psd'))
def __init__(self, *args, **kwargs):
self.client = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
self.location = kwargs.get('location', settings.MEDIA_ROOT)
class FileManager(object):
NAME = 'dropbox'
def __init__(self, user_obj, **kwargs):
self._user = user_obj
if user_obj.is_anonymous():
raise ApiException(
status=403, message='Log in required to access Dropbox files.')
try:
# self.dropbox_api = user_obj.dropbox_user_token.client
dropbox_token = DropboxUserToken.objects.get(user=user_obj)
self.dropbox_api = Dropbox(dropbox_token.access_token)
except DropboxUserToken.DoesNotExist:
message = 'Connect your Dropbox account <a href="' + reverse(
'dropbox_integration:index') + '">here</a>'
raise ApiException(status=400,
message=message,
extra={
'action_url':
reverse('dropbox_integration:index'),
'action_label':
'Connect Dropbox.com Account'
})
def parse_file_id(self, path):
if path == '/' or path == '':
file_type, path = 'folder', ''
else:
try:
file_type, path = DropboxFile.parse_file_id(path)
except AssertionError:
# file path is hierarchical; need to find the DropboxObject here
if path[:1] != '/':
path = '/' + path
try:
dropbox_item = DropboxFile(
self.dropbox_api.files_list_folder(path))
return dropbox_item.type, path
except ApiError as e:
if e.error.get_path().is_not_found():
raise ApiException(
'The Dropbox path "{0}" does not exist.'.format(
path),
status=404)
return file_type, path
def listing(self, file_id='', **kwargs):
"""
Lists contents of a folder or details of a file.
Args:
file_id: The type/path of the Dropbox Object. This should be formatted {type}/{path}
where {type} is one of ['folder', 'file'] and {path} is the numeric Dropbox path for
the object.
Returns:
Dictionary with two keys;
- resource: File System resource that we're listing
- files: Array of Api file-like objects
"""
default_pems = 'ALL'
try:
file_type, path = self.parse_file_id(file_id)
dropbox_item = self.dropbox_api.files_list_folder(path)
children = []
if file_type == 'folder':
has_more = dropbox_item.has_more
cursor = dropbox_item.cursor
entries = dropbox_item.entries
while True:
children.extend([
DropboxFile(item,
item.path_display,
parent=dropbox_item).to_dict(
default_pems=default_pems)
for item in entries
])
if has_more:
folder = self.dropbox_api.files_list_folder_continue(
cursor)
entries = folder.entries
has_more = folder.has_more
cursor = folder.cursor
else:
break
else:
children = None
list_data = DropboxFile(dropbox_item,
path).to_dict(default_pems=default_pems)
if children:
list_data['children'] = children
return list_data
except AssertionError:
raise ApiException(
status=404,
message='The file/folder you requested does not exist.')
except AuthError:
# user needs to reconnect with Dropbox
message = 'While you previously granted this application access to Dropbox, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \
'to continue using Dropbox data.' % reverse('dropbox_integration:index')
raise ApiException(status=403, message=message)
except ApiError as e:
if e.error.get_path().is_not_folder():
dropbox_item = self.dropbox_api.files_get_metadata(path)
list_data = DropboxFile(dropbox_item, path)
return list_data
message = 'Unable to communicate with Dropbox: %s' % e.message
raise ApiException(status=500, message=message)
def file(self, file_id, action, path=None, **kwargs):
pass
def is_shared(self, *args, **kwargs):
return False
def is_search(self, *args, **kwargs):
return False
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(
event_type='data',
status=Notification.INFO,
operation='dropbox_download_start',
message='Starting download file %s from dropbox.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
file_type, path = self.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(path, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
path, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='dropbox_download_end',
message='File %s has been copied from dropbox successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except:
logger.exception('Unexpected task failure: dropbox_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='dropbox_download_error',
message='We were unable to get the specified file from dropbox. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def move(self, file_id, **kwargs):
raise ApiException('Moving Dropbox files is not supported.',
status=400,
extra={
'file_id': file_id,
'kwargs': kwargs
})
def get_preview_url(self, file_id, **kwargs):
file_type, path = self.parse_file_id(file_id)
if file_type == 'file':
shared_link = self.dropbox_api.sharing_create_shared_link(path).url
embed_url = re.sub('dl=0$', 'raw=1', shared_link)
return {'href': embed_url}
return None
def preview(self, file_id, file_mgr_name, **kwargs):
try:
file_type, path = self.parse_file_id(file_id)
dropbox_item = self.dropbox_api.files_get_metadata(path)
dropbox_file = DropboxFile(dropbox_item, path)
if dropbox_file.previewable:
preview_url = reverse('designsafe_api:box_files_media',
args=[file_mgr_name,
file_id.strip('/')])
return JsonResponse(
{'href': '{}?preview=true'.format(preview_url)})
else:
return HttpResponseBadRequest(
'Preview not available for this item.')
except HTTPError as e:
logger.exception('Unable to preview file')
return HttpResponseBadRequest(e.response.text)
def get_download_url(self, file_id, **kwargs):
file_type, path = self.parse_file_id(file_id)
if file_type == 'file':
download_url = self.dropbox_api.files_get_temporary_link(path)
return {'href': download_url.link}
return None
#def import_file(self, file_id, from_resource, import_file_id, **kwargs):
# dropbox_upload.apply_async(args=(self._user.username,
# file_id,
# from_resource,
# import_file_id),
# countdown=10)
# return {'message': 'Your file(s) have been scheduled for upload to box.'}
def download_file(self, dropbox_file_path, download_directory_path):
"""
Downloads the file for dropbox_file_path to the given download_path.
:param dropbox_file_path:
:param download_directory_path:
:return: the full path to the downloaded file
"""
dropbox_file = self.dropbox_api.files_get_metadata(dropbox_file_path)
# convert utf-8 chars
safe_filename = dropbox_file.name.encode(sys.getfilesystemencoding(),
'ignore')
file_download_path = os.path.join(download_directory_path,
safe_filename)
logger.debug('Download file %s <= dropbox://file/%s',
file_download_path, dropbox_file_path)
self.dropbox_api.files_download_to_file(file_download_path,
dropbox_file_path)
return file_download_path
def download_folder(self, path, download_path):
"""
Recursively download the folder for path, and all of its contents, to the given
download_path.
:param path:
:param download_path:
:return:
"""
dropbox_folder = self.dropbox_api.files_list_folder(path)
has_more = dropbox_folder.has_more
cursor = dropbox_folder.cursor
dropbox_folder_metadata = self.dropbox_api.files_alpha_get_metadata(
path)
# convert utf-8 chars
safe_dirname = dropbox_folder_metadata.name.encode(
sys.getfilesystemencoding(), 'ignore')
directory_path = os.path.join(download_path, safe_dirname)
logger.debug('Creating directory %s <= dropbox://folder/%s',
directory_path, path)
try:
os.mkdir(directory_path, 0o0755)
except OSError as e:
if e.errno == 17: # directory already exists?
pass
else:
logger.exception('Error creating directory: %s',
directory_path)
raise
items = dropbox_folder.entries
while True:
for item in items:
if type(item) == FileMetadata:
self.download_file(item.path_lower, directory_path)
elif type(item) == FolderMetadata:
self.download_folder(item.path_lower, directory_path)
if has_more:
folder = self.dropbox_api.files_list_folder_continue(cursor)
items = folder.entries
has_more = items.has_more
cursor = items.cursor
else:
break
return directory_path
def upload(self, username, src_file_id, dest_file_id):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='dropbox_upload_start',
message='Starting uploading file %s to dropbox.' %
(src_file_id, ),
user=username,
extra={})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split src ination file path
src_file_path_comps = src_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = src_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if src_file_path_comps[1:]:
src_real_path = os.path.join(*src_file_path_comps[1:])
else:
src_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
src_real_path = os.path.join(base_mounted_path, project_dir,
src_real_path.strip('/'))
else:
src_real_path = os.path.join(base_mounted_path,
src_real_path.strip('/'))
logger.debug('src_real_path: {}'.format(src_real_path))
file_type, path = self.parse_file_id(dest_file_id)
if os.path.isfile(src_real_path):
self.upload_file(path, src_real_path)
elif os.path.isdir(src_real_path):
self.upload_directory(path, src_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
src_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='dropbox_upload_end',
message='File %s has been copied to dropbox successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
except Exception as err:
logger.exception('Unexpected task failure: dropbox_upload',
extra={
'username': username,
'src_file_id': src_file_id,
'dst_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='dropbox_upload_error',
message='We were unable to get the specified file from dropbox. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def upload_file(self, dropbox_path, file_real_path):
file_path, file_name = os.path.split(file_real_path)
with open(file_real_path, 'rb') as file_handle:
f = file_handle.read()
file_size = os.path.getsize(file_path)
CHUNK_SIZE = 4 * 1024 * 1024 # 4MB
if file_size <= CHUNK_SIZE:
logger.debug('dropbox_path: %s, file_name: %s', dropbox_path,
file_name),
self.dropbox_api.files_upload(
f, '%s/%s' % (dropbox_path, file_name))
else:
upload_session_start_result = self.dropbox_api.files_upload_session_start(
f.read(CHUNK_SIZE))
cursor = dropbox.files.UploadSessionCursor(
session_id=upload_session_start_result.session_id,
offset=f.tell())
commit = dropbox.files.CommitInfo(path=file_path)
while f.tell() < file_size:
if ((file_size - f.tell()) <= CHUNK_SIZE):
print(
self.dropbox_api.files_upload_session_finish(
f.read(CHUNK_SIZE), cursor, commit))
else:
self.dropbox_api.files_upload_session_append(
f.read(CHUNK_SIZE), cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
logger.info('Successfully uploaded %s to dropbox:folder/%s',
file_real_path, file_path)
def upload_directory(self, dropbox_parent_folder, dir_real_path):
"""
Recursively uploads the directory and all of its contents (subdirectories and files)
to the dropbox folder specified by dropbox_parent_folder.
:param dropbox_parent_folder: The dropbox folder to upload the directory to.
:param dir_real_path: The real path on the filesystem of the directory to upload.
:return:
"""
dirparentpath, dirname = os.path.split(dir_real_path)
# dropbox_parent_folder = self.dropbox_api.folder(dropbox_parent_folder)
logger.info('Create directory %s in dropbox folder/%s', dirname,
dropbox_parent_folder)
# box_folder = box_parent_folder.create_subfolder(dirname)
for dirpath, subdirnames, filenames in os.walk(dir_real_path):
# upload all the files
for filename in filenames:
filepath = os.path.join(dirpath, filename)
self.upload_file('%s/%s' % (dropbox_parent_folder, dirname),
filepath)
# upload all the subdirectories
for subdirname in subdirnames:
subdirpath = os.path.join(dirpath, subdirname)
self.upload_directory(
'%s/%s' % (dropbox_parent_folder, dirname), subdirpath)
# prevent further walk, because recursion
subdirnames[:] = []
class DropboxApp:
"""
File operations for dropbox.
See:
https://dropbox-sdk-python.readthedocs.io/en/latest/index.html
"""
svc: DropboxService
def __init__(self):
"""Create connection and local,remote folders."""
self.svc = DropboxService(settings.FARMBOX_DROPBOX_ACCESS_TOKEN)
create_media_dir()
self.create_remote_folders()
def create_remote_folders(self):
"""Create root subfolders if they don't exist."""
subfolder_exists = {}
for sub in settings.CLOUD_SUBFOLDERS:
subfolder_exists[sub] = False
for c in self.list_contents():
if isinstance(c, FolderMetadata) and c.name == sub:
subfolder_exists[sub] = True
for k, v in subfolder_exists.items():
if not v:
self.svc.files_create_folder_v2(f"/{k}")
def remove_remote_db(self) -> bool:
path = settings.DATABASES["sqlite"]["NAME"]
dest = f"/{os.path.join(settings.BACKUP_DB_FOLDER,os.path.basename(path))}"
self.remove(dest)
return True
def backup_db(self):
"""Backup local db dump."""
src = settings.DATABASES["sqlite"]["NAME"]
dest = f"/{settings.BACKUP_DB_FOLDER}"
self.upload(src, dest)
def restore_db(self):
"""Restore remote db dump"""
dest = settings.DATABASES["sqlite"]["NAME"]
src = f"/{os.path.join(settings.BACKUP_DB_FOLDER,os.path.basename(dest))}"
self.svc.files_download_to_file(dest, src, rev=None)
def upload(self, src, dest=""):
"""Upload bytes to dropbox."""
filename = os.path.basename(src)
dest = f"{dest}/{filename}"
with open(src, "rb") as f:
self.svc.files_upload(f.read(), dest, mute=True)
def list_contents(self, path=""):
"""Lists folder contents.
:param str path: If empty, it's the
default dropbox app folder.
`/processed` is the default app's
`processed` subfolder
"""
return self.svc.files_list_folder(path).entries
def download_all_as_zip(self, local_path,
remote_path) -> List[FileMetadata]:
"""Downloads all files to instance storage"""
self.svc.files_download_zip_to_file(local_path, remote_path)
return self.list_contents(remote_path)
def remove(self, path: str):
self.svc.files_delete_v2(path)
def move_to_processed_dir(self, ext_ref, dest_path):
pass
