def _init_impl(self):
with self.update_lock:
# Create a pool of at most that many connections
session = create_session(50)
scope = as_list(self.config.default_scope, ',')
config = {
'session': session,
'user_agent': self.config.user_agent,
'oauth2_access_token': self.server.decrypt(self.config.secret),
'oauth2_access_token_expiration': int(self.config.oauth2_access_token_expiration or 0),
'scope': scope,
'max_retries_on_error': int(self.config.max_retries_on_error or 0),
'max_retries_on_rate_limit': int(self.config.max_retries_on_rate_limit or 0),
'timeout': int(self.config.timeout),
'headers': parse_extra_into_dict(self.config.http_headers),
}
# Create the actual connection object
self._impl = DropboxClient(**config)
# Confirm the connection was established
self.ping()
# We can assume we are connected now
self.is_connected = True
def checkFolder(self, accessToken, folder):
try:
client = Dropbox(accessToken, session=create_session(proxies=self.__proxies))
self.__createFolder(client, folder + "/entries/deleted")
self.__createFolder(client, folder + "/photos/deleted")
except ApiError:
return False
return True
def session_instance(self, mocker):
session_obj = create_session()
post_response = mock.MagicMock(status_code=200)
post_response.json.return_value = {
"access_token": ACCESS_TOKEN,
"expires_in": EXPIRES_IN
}
mocker.patch.object(session_obj, 'post', return_value=post_response)
return session_obj
def __init__(self, accessToken, folder, proxyHost=None, proxyPort=None, proxyUser=None, proxyPassword=None):
proxies = _proxies(proxyHost, proxyPort, proxyUser, proxyPassword)
self.__token = accessToken
self.__basePath = folder
self.__notesPath = folder + "/entries"
self.__removedNotesPath = self.__notesPath + "/deleted"
self.__photosPath = folder + "/photos"
self.__client = Dropbox(self.__token, session=create_session(proxies=proxies))
self.__notesCache = {}
self.__dayOneFlavor = folder == SyncFolder.DayOne
def __init__(self):
Analyzer.__init__(self)
self.polling_interval = self.get_param('config.polling_interval', 60)
self.proxies = self.get_param('config.proxy', None)
self.dbx_token = self.get_param('config.dropbox_token', None,
'Missing Dropbox QAuth Token')
self.zip_password = self.get_param('config.zip_password', None,
'Missing ZIP password')
self.dbx_session = dropbox.create_session(8, proxies=self.proxies)
self.dbx = dropbox.Dropbox(self.dbx_token, session=self.dbx_session)
def _create_session(self):
session = dropbox.create_session()
old_session_post = session.post
def new_session_post(*n, **kw):
r = old_session_post(*n, **kw)
self._local.r = r
return r
session.post = new_session_post
return session
def main():
parser = ArgumentParser()
parser.add_argument('mountpoint', type=str,
help='Where to mount the file system')
parser.add_argument('token', type=str,
help='Token of dropbox app')
parser.add_argument('--debug', action='store_true', default=False,
help='Enable debugging output')
parser.add_argument('--debug-fuse', action='store_true', default=False,
help='Enable FUSE debugging output')
parser.add_argument('--tmpdir', type=str, default='/tmp/fusedive',
help='Temporary local path')
options = parser.parse_args()
subprocess.Popen(('mkdir -p %s' % (options.tmpdir)).split())
init_logging(options.debug)
pros = {
'http': "socks5://127.0.0.1:1080",
'https': "socks5://127.0.0.1:1080"
}
sess = dropbox.create_session(max_connections=3, proxies=pros)
dbx = dropbox.Dropbox(options.token, session=sess)
operations = DropboxOperations(dbx, options.tmpdir)
fuse_options = set(llfuse.default_options)
fuse_options.add('fsname=dropboxfs')
fuse_options.discard('default_permissions')
if options.debug_fuse:
fuse_options.add('debug')
llfuse.init(operations, options.mountpoint, fuse_options)
# sqlite3 does not support multithreading
try:
llfuse.main(workers=1)
except:
subprocess.Popen(('rm -rf %s' % (options.tmpdir)).split())
llfuse.close()
raise
subprocess.Popen(('rm -rf %s' % (options.tmpdir)).split())
llfuse.close()
def make_fs(self):
# Return an instance of your FS object here
self.access_token = 'Ozdb24UtqKAAAAAAAAAAC5-zHhrmCXEmdFWu9Dmj0PJrvWn-FCG23zLpt5k6OiGu'
if "DEV" in os.environ:
proxies = {
'http': 'http://127.0.0.1:1087',
'https': 'http://127.0.0.1:1087'
}
sess = create_session(8, proxies=proxies)
else:
sess = None
fs = DropboxFS(self.access_token, session=sess)
for f in fs.listdir('/'):
f = fs.fix_path(f)
fs.dropbox.files_delete_v2(f)
return fs
def make_fs(self):
# Return an instance of your FS object here
self.access_token = "olshmk5XitgAAAAAAAAJnRpQ3mrGJq5kmsI6QvycvC2kT8p18SsajOlhWV511oAd"
if "DEV" in os.environ:
proxies = {
"http": "http://127.0.0.1:1087",
"https": "http://127.0.0.1:1087",
}
sess = create_session(8, proxies=proxies)
else:
sess = None
dfs = DropboxFS(self.access_token, session=sess)
if dfs.exists(TEST_PATH):
dfs.removetree(TEST_PATH)
dfs.makedir(TEST_PATH)
fs2 = fs.subfs.SubFS(dfs, TEST_PATH)
return fs2
def make_fs(self):
# Return an instance of your FS object here
self.access_token = DROPBOX_ACCESS_TOKEN
self.access_token = "GiQj7BV19aAAAAAAAAAACAevudx3Rxyca3vKenwRV9suPJ2sWKw3Bm6rC9CpxDM2"
if "DEV" in os.environ:
proxies = {
"http": "http://127.0.0.1:1087",
"https": "http://127.0.0.1:1087",
}
sess = create_session(8, proxies=proxies)
else:
sess = None
fs = DropboxFS(self.access_token, session=sess)
for f in fs.listdir("/"):
f = fs.fix_path(f)
fs.dropbox.files_delete_v2(f)
return fs
def invalid_grant_session_instance(self, mocker):
session_obj = create_session()
post_response = mock.MagicMock(status_code=400)
post_response.json.return_value = {"error": "invalid_grant"}
mocker.patch.object(session_obj, 'post', return_value=post_response)
return session_obj
class DropboxUploaderOperation(Operation):
NAME = "Dropbox Uploader"
SESSION = dropbox.create_session(max_connections=QueueManager.MAX_JOBS)
CHUNK_SIZE = 1024 * 1024 # 1MB
def __init__(self, *args, **kwargs):
Operation.__init__(self, *args, **kwargs)
self._dropbox = None
self._space_thread = None
self._grid = Gtk.Grid(
border_width=5,
row_spacing=5,
column_spacing=5,
halign=Gtk.Align.FILL,
valign=Gtk.Align.CENTER,
hexpand=True,
vexpand=False,
)
self.add(self._grid)
# In the initial version, use only access-key and folder name
self._grid.attach(
Gtk.Label(
label="Destination folder",
halign=Gtk.Align.START,
valign=Gtk.Align.CENTER,
hexpand=False,
vexpand=False,
),
0,
0,
1,
1,
)
widget = self.register_widget(
Gtk.Entry(
halign=Gtk.Align.FILL,
valign=Gtk.Align.CENTER,
hexpand=True,
vexpand=False,
),
"destination_folder",
)
self._grid.attach(widget, 1, 0, 2, 1)
self._grid.attach(
Gtk.Label(
label="Email address",
halign=Gtk.Align.START,
valign=Gtk.Align.CENTER,
hexpand=False,
vexpand=False,
),
0,
1,
1,
1,
)
self._email_entry = self.register_widget(
Gtk.Entry(
placeholder_text="Address used for registering with Dropbox",
halign=Gtk.Align.FILL,
valign=Gtk.Align.CENTER,
hexpand=True,
vexpand=False,
),
"email",
)
self._grid.attach(self._email_entry, 1, 1, 1, 1)
self._email_entry.connect("changed", self._email_entry_changed_cb)
button = Gtk.Button(
label="Validate",
halign=Gtk.Align.END,
valign=Gtk.Align.CENTER,
hexpand=False,
vexpand=False,
)
self._grid.attach(button, 2, 1, 1, 1)
button.connect("clicked", self._validate_button_clicked_cb)
self._space_label = Gtk.Label(
label="Space Usage: not available",
use_markup=True,
halign=Gtk.Align.START,
valign=Gtk.Align.CENTER,
hexpand=True,
vexpand=True,
)
self._grid.attach(self._space_label, 0, 2, 3, 1)
# check space usage every 60 seconds and update the corresponding label
GLib.timeout_add_seconds(
60,
self._launch_space_usage_thread,
False,
priority=GLib.PRIORITY_DEFAULT_IDLE,
)
def _launch_space_usage_thread(self, kill):
space_thread = DropboxSpaceCheckerThread(self)
space_thread.start()
if kill:
return GLib.SOURCE_REMOVE
else:
return GLib.SOURCE_CONTINUE
def _update_space_usage(self, usage):
self._space_label.props.label = f"Space Usage: {str(usage)}"
return GLib.SOURCE_REMOVE
def _email_entry_changed_cb(self, entry):
# any changes to the email address reset the Dropbox client
self._dropbox = None
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
def _validate_button_clicked_cb(self, button):
# first confirm that we have a proper email address
if not re.fullmatch(r"[^@]+@[^@]+\.[^@]+", self.params.email):
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text=f"{self.params.email} is not a vaild email address",
)
dialog.run()
dialog.destroy()
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
return
# check keyring
try:
refresh_token = keyring.get_password(
"RFI-File-Monitor-Dropbox", self.params.email.lower()
)
except keyring.errors.KeyringError as e:
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text="Error accessing keyring",
secondary_text=str(e),
)
dialog.run()
dialog.destroy()
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
return
if refresh_token:
# use refresh token to launch dropbox session
self._dropbox = dropbox.Dropbox(
oauth2_refresh_token=refresh_token,
session=self.SESSION,
app_key=APP_KEY,
)
exc = None
try:
account_info = self._dropbox.users_get_current_account()
except dropbox.exceptions.AuthError as e:
if e.error.is_invalid_access_token():
logger.info("Dropbox token has been revoked!")
self._dropbox = None
# delete keyring password
keyring.delete_password(
"RFI-File-Monitor-Dropbox", self.params.email.lower()
)
self._validate_button_clicked_cb(button)
return
exc = e
except Exception as e:
exc = e
if exc:
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text="Could not get user information",
secondary_text=str(exc),
)
dialog.run()
dialog.destroy()
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
self._dropbox = None
return
if account_info.email.lower() != self.params.email.lower():
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text="Email address does not match Dropbox user records",
)
dialog.run()
dialog.destroy()
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
self._dropbox = None
else:
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-default",
)
GLib.idle_add(
self._launch_space_usage_thread,
True,
priority=GLib.PRIORITY_DEFAULT_IDLE,
)
return
# without refresh token -> link app
dbx_dialog = DropboxLinkDialog(self.appwindow)
if dbx_dialog.run() != Gtk.ResponseType.OK:
# not OK means that the auth flow was not started or aborted
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
self._dropbox = None
dbx_dialog.destroy()
return
authorization_code = dbx_dialog.authorization_code
auth_flow = dbx_dialog.auth_flow
dbx_dialog.destroy()
try:
res = auth_flow.finish(authorization_code)
except Exception as e:
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text="Could not link RFI-File-Monitor to Dropbox",
secondary_text=str(e),
)
dialog.run()
dialog.destroy()
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY,
icon_name="emblem-unreadable",
)
self._dropbox = None
return
refresh_token = res.refresh_token
self._dropbox = dropbox.Dropbox(
oauth2_refresh_token=refresh_token,
session=self.SESSION,
app_key=APP_KEY,
)
self._email_entry.set_icon_from_icon_name(
icon_pos=Gtk.EntryIconPosition.SECONDARY, icon_name="emblem-default"
)
GLib.idle_add(
self._launch_space_usage_thread,
True,
priority=GLib.PRIORITY_DEFAULT_IDLE,
)
# save token in keyring
try:
keyring.set_password(
"RFI-File-Monitor-Dropbox",
self.params.email.lower(),
refresh_token,
)
except keyring.errors.KeyringError as e:
dialog = Gtk.MessageDialog(
transient_for=self.appwindow,
modal=True,
destroy_with_parent=True,
message_type=Gtk.MessageType.ERROR,
buttons=Gtk.ButtonsType.CLOSE,
text="Error accessing keyring. Dropbox should still work in this session though.",
secondary_text=str(e),
)
dialog.run()
dialog.destroy()
def preflight_check(self):
if not self.params.destination_folder:
raise Exception("Destination folder cannot be an empty string")
if not self._dropbox:
raise Exception(
"Validate the email address and link the Dropbox account"
)
self._dropbox.check_user("test-user")
# create folder and upload test file
self._base_folder = f"/{self.params.destination_folder}"
try:
self._dropbox.files_create_folder_v2(self._base_folder)
except dropbox.exceptions.ApiError:
# an exception is thrown if the directory already exists
pass
test_filename = (
f"{self._base_folder}/test-file-{get_random_string(6)}.txt"
)
self._dropbox.files_upload(b"Dummy contents", path=test_filename)
# delete file
self._dropbox.files_delete_v2(test_filename)
@add_directory_support
def run(self, file: File):
# check first if file already exists
# if it does, then first get its size
# if the size matches, calculate checksum.
# if checksums match, SKIP
# else: remove file and upload new version
dbx_filename = str(
PurePosixPath(self._base_folder, *file.relative_filename.parts)
)
size = os.path.getsize(file.filename)
try:
metadata = self._dropbox.files_get_metadata(dbx_filename)
except dropbox.exceptions.ApiError:
pass
else:
if size == metadata.size:
# calculate content hash of local file
hasher = DropboxContentHasher()
with open(file.filename, "rb") as f:
while True:
chunk = f.read(4096)
if len(chunk) == 0:
break
hasher.update(chunk)
if hasher.hexdigest() == metadata.content_hash:
raise SkippedOperation(
"File has already been uploaded to Dropbox"
)
else:
# delete remote file
self._dropbox.files_delete_v2(dbx_filename)
else:
# delete remote file
self._dropbox.files_delete_v2(dbx_filename)
# the following was inspired by Maestral
if size <= self.CHUNK_SIZE:
# upload small file
with open(file.filename, "rb") as f:
self._dropbox.files_upload(f.read(), dbx_filename)
else:
# upload large file
with open(file.filename, "rb") as f:
session_start = self._dropbox.files_upload_session_start(
f.read(self.CHUNK_SIZE)
)
uploaded = f.tell()
cursor = dropbox.files.UploadSessionCursor(
session_id=session_start.session_id, offset=uploaded
)
commit = dropbox.files.CommitInfo(path=dbx_filename)
while True:
try:
if size - f.tell() <= self.CHUNK_SIZE:
md = self._dropbox.files_upload_session_finish(
f.read(self.CHUNK_SIZE), cursor, commit
)
else:
self._dropbox.files_upload_session_append_v2(
f.read(self.CHUNK_SIZE), cursor
)
md = None
# housekeeping
uploaded = f.tell()
# upload progressbar
file.update_progressbar(
self.index, 100 * uploaded / size
)
if md:
break
else:
cursor.offset = uploaded
except dropbox.exceptions.DropboxException as exc:
error = getattr(exc, "error", None)
if (
isinstance(
error, dropbox.files.UploadSessionFinishError
)
and error.is_lookup_failed()
):
session_lookup_error = error.get_lookup_failed()
elif isinstance(
error, dropbox.files.UploadSessionLookupError
):
session_lookup_error = error
else:
return str(exc)
if session_lookup_error.is_incorrect_offset():
o = (
session_lookup_error.get_incorrect_offset().correct_offset
)
# reset position in file
f.seek(o)
cursor.offset = f.tell()
else:
return str(exc)
return None
