def revision(): dbx = Dropbox(session['access_token']) f = dbx.files_download(request.args['path'], request.args['rev']) resp = make_response(f[1].content) resp.headers["Content-Disposition"] = "attachment; filename=" + f[0].name return resp
def download_file(dbx: dropbox.Dropbox, local_report_file_path: Path,
remote_report_file_path: str):
try:
metadata, response = dbx.files_download(remote_report_file_path)
with local_report_file_path.open(mode='wb') as report_file:
report_file.write(response.content)
except ApiError as e:
sys.exit(f"Error: {e}")
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.error['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.error, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = DUMMY_PAYLOAD
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(DUMMY_PAYLOAD, resp.content)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
def test_bad_upload_types(self):
with self.assertRaises(TypeError):
self.dbx.files_upload(BytesIO(b'test'), '/Test')
@require_team_token
def test_team(self, token):
dbxt = DropboxTeam(token)
dbxt.team_groups_list()
r = dbxt.team_members_list()
if r.members:
# Only test assuming a member if there is a member
dbxt.as_user(
r.members[0].profile.team_member_id).files_list_folder('')
def pull_from_dropbox(branch_name, symbol, gui):
global settings # Read
if settings["CONFIRM"]:
confirm = simpledialog.askstring(
"Confirm",
"Type in 'YES' if you wish to proceed. This will override existing worlds"
" if a conflict is found")
if not confirm == "YES":
return "Pull cancelled"
saves_path = settings["SAVES_DIR"]
temp_dir = settings["TEMP_DIR"]
source = "/" + branch_name + "/"
if settings["OAUTH"] == 'null':
return "Please type in /login to use this feature"
println("Starting download... ", gui)
println(
"Do not close the app until 'done downloading' message is shown on the console",
gui)
# clear temp_dir
for path_temp in listdir(temp_dir):
if path.isdir(path.join(temp_dir, path_temp)):
rmtree(path.join(temp_dir, path_temp))
else:
remove(path.join(temp_dir, path_temp))
# download zip files
dbx = Dropbox(settings["OAUTH"].access_token)
for entry in dbx.files_list_folder(source).entries:
file = source + entry.name
with open(path.join(temp_dir, entry.name), "wb") as f:
metadata, res = dbx.files_download(path=file)
f.write(res.content)
f.close()
for path_temp in listdir(temp_dir):
file = path.join(temp_dir, path_temp)
name, extension = path.splitext(file)
file_name, ext = path.splitext(path_temp)
if file_name[0] == symbol and extension == ".zip":
with zipfile.ZipFile(file, 'r') as zip_ref:
z = path.join(saves_path, file_name)
zip_ref.extractall(z)
remove(file)
save(settings)
return "Done Downloading"
def debug():
"""Debug todo synchronization code"""
dropbox = db.session.query(models.Dropbox).first() # type: models.Dropbox
try:
dbx = Dropbox(dropbox.access_token)
md, res = dbx.files_download(path=dropbox.file_location)
except ApiError as err:
if err.error.is_path() and err.error.get_path().is_not_found():
return 'File not found: ' + dropbox.file_location
return 'Other error occurred'
update_todos(content=res.content)
return redirect(url_for('show_todos'))
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.error['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.error, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = string.ascii_letters
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(string.ascii_letters, resp.text)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
@require_team_token
def test_team(self, token):
dbxt = DropboxTeam(token)
dbxt.team_groups_list()
r = dbxt.team_members_list()
if r.members:
# Only test assuming a member if there is a member
dbxt.as_user(r.members[0].profile.team_member_id).files_list_folder('')
def chunk(self, path, filename, size, offset=0):
"""
return one chunk of file
:param str path: path on server
:param str filename: name of file
:param int size: chunk-size
:param int offset: bits from the beginning
:return: tuple(File obj, content)
"""
p_session = session()
dbx_p = Dropbox(oauth2_access_token=self._access_token, headers={
"Range": "bytes=" + str(offset) + "-" + str(offset + size - 1)}, session=p_session) # fetch chunks from dropbox
meta, response = dbx_p.files_download(path+"/"+filename)
f = File(meta.name, meta.path_lower, meta.client_modified, meta.client_modified)
p_session.close()
return f, response.content
def sync():
challenge = request.args.get('challenge')
if challenge is not None:
return challenge
"""Synchronize database with todo.txt"""
dropbox = db.session.query(models.Dropbox).first() # type: models.Dropbox
# Make sure this is a valid request from Dropbox
signature = request.headers.get('X-Dropbox-Signature')
if not hmac.compare_digest(signature, hmac.new(dropbox.secret.encode(), request.data, sha256).hexdigest()):
app.logger.warn('Invalid sync request attempted')
abort(403)
dbx = Dropbox(dropbox.access_token)
if dropbox.cursor is None:
result = dbx.files_list_folder(path=os.path.dirname(dropbox.file_location))
else:
result = dbx.files_list_folder_continue(cursor=dropbox.cursor)
# Check if todo.txt was changed
found = False
for metadata in result.entries: # type: Metadata
if metadata.path_lower == dropbox.file_location.lower():
found = True
break
if not found:
dropbox.cursor = result.cursor
db.session.merge(dropbox)
db.session.commit()
return ''
app.logger.info('Sync request made')
try:
md, res = dbx.files_download(path=dropbox.file_location)
except ApiError as err:
if err.error.is_path() and err.error.get_path().is_not_found():
return 'File not found: ' + dropbox.file_location
return 'Other error occurred'
update_todos(content=res.content)
dropbox.cursor = result.cursor
db.session.merge(dropbox)
db.session.commit()
return ''
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.reason['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.reason, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = string.ascii_letters
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(string.ascii_letters, resp.text)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
def convert_from_dropbox():
"""Convert an image from dropbox."""
access_token = request.headers.get('Dropbox-Token')
# Check if the dropbox access token was sent
if access_token is None:
abort(403, description='No dropbox access token sent')
dbx = Dropbox(access_token)
path = request.form.get('path')
# Check if the path was sent in the request
if path is None:
abort(400, description='No path sent')
format = request.form.get('format')
# Check if the format was sent in the request
if format is None:
abort(400, description='No format sent')
# Check if the format sent is accepted
if format.lower() not in FORMATS:
abort(400, description='The format sent is invalid')
try:
metadata, response = dbx.files_download(path)
except:
abort(404, description='The file was not found')
# Create a byte object to save the image
file = BytesIO()
with Image.open(BytesIO(response.content)) as image:
image.save(file, format)
file.seek(0)
return send_file(file, mimetype='image/{}'.format(format))
def process_user(account):
'''Call /files/list_folder for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', account)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
dbx = Dropbox(token)
has_more = True
while has_more:
if cursor is None:
result = dbx.files_list_folder(path='')
else:
result = dbx.files_list_folder_continue(cursor)
for entry in result.entries:
# Ignore deleted files, folders, and non-markdown files
if (isinstance(entry, DeletedMetadata)
or isinstance(entry, FolderMetadata)
or not entry.path_lower.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
_, resp = dbx.files_download(entry.path_lower)
html = markdown(resp.content.decode("utf-8"))
dbx.files_upload(bytes(html, encoding='utf-8'),
entry.path_lower[:-3] + '.html',
mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
def process_user(account):
'''
Call /files/list_folder for the given user ID and process any changes.
'''
print("*" * 30, " - PROCESS_USER - ", "*" * 30)
# OAuth token for the user
token = redis_client.hget('tokens', account)
print("token: ", token)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
print("cursor: ", cursor)
output_file_extension = ".csv"
input_file_extensions = [
".textgrid",
".TextGrid",
]
processed_marker = "_cleaned"
dbx = Dropbox(token.decode())
has_more = True
while has_more:
print("there's more!")
if cursor is None:
print("cursor is 'None'!")
result = dbx.files_list_folder(path='')
print("result: ", result)
else:
print("entering files_list_folder_continue...")
result = dbx.files_list_folder_continue(cursor.decode())
print("result: ", result)
for entry in result.entries:
print("entry: ", entry)
# yapf: disable
if (
isinstance(entry, DeletedMetadata) or
isinstance(entry, FolderMetadata) or
not any(entry.path_lower.endswith(e)
for e in input_file_extensions)
):
# yapf: enable
print("skipping.")
continue
_, resp = dbx.files_download(entry.path_lower)
print("processing data...")
processed_data = clean_data(resp.content)
dbx.files_upload(processed_data,
entry.path_lower[:-4] + processed_marker +
output_file_extension,
mode=WriteMode('add'))
#mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
class DropboxNoteProvider(RemoteNoteProvider):
__DAY_ONE_EXTENSION = ".doentry"
def __init__(self, accessToken, folder, proxyHost=None, proxyPort=None, proxyUser=None, proxyPassword=None):
proxies = _proxies(proxyHost, proxyPort, proxyUser, proxyPassword)
self.__token = accessToken
self.__basePath = folder
self.__notesPath = folder + "/entries"
self.__removedNotesPath = self.__notesPath + "/deleted"
self.__photosPath = folder + "/photos"
self.__client = Dropbox(self.__token, session=create_session(proxies=proxies))
self.__notesCache = {}
self.__dayOneFlavor = folder == SyncFolder.DayOne
@online
@expires
def sync(self):
_LOG.info("Listing all notes and photos")
folder = self.__client.files_list_folder(self.__basePath, recursive=True)
files = list(filter(lambda e: e is not None, map(FileEntry.fromMetadata, folder.entries)))
while folder.has_more:
folder = self.__client.files_list_folder_continue(folder.cursor)
files.extend(filter(lambda e: e is not None, map(FileEntry.fromMetadata, folder.entries)))
notes = {}
for file in filter(lambda f: f.folder == self.__notesPath and isUuid(self.__normalizeNoteName(f.name)), files):
uuid = self.__normalizeNoteName(file.name)
notes[uuid] = NoteStatus(uuid, file.lastModified)
for file in filter(lambda f: f.folder == self.__photosPath and f.name.endswith(".jpg"), files):
uuid = file.name[:-4]
if uuid in notes:
notes[uuid].hasPhoto = True
for file in filter(lambda f: f.folder == self.__removedNotesPath and isUuid(self.__normalizeNoteName(f.name)),
files):
uuid = self.__normalizeNoteName(file.name)
if uuid in notes:
if file.lastModified >= notes[uuid].lastModified:
_LOG.warning("Sync integrity check deleting note: %s", uuid)
try:
self.__client.files_delete(self.__notePath(uuid))
except ApiError:
_LOG.warning("Note %s not found", uuid)
if notes[uuid].hasPhoto:
_LOG.warning("Sync integrity check deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
del notes[uuid]
else:
_LOG.warning("Sync integrity check deleting REMOVED note: %s", uuid)
try:
self.__client.files_delete(self.__removedNotePath(uuid))
except ApiError:
_LOG.warning("REMOVED note %s not found", uuid)
continue
notes[uuid] = NoteStatus(uuid, file.lastModified, True)
self.__notesCache = notes
return notes
@online
@expires
def get(self, uuid):
_LOG.info("Getting note: %s", uuid)
metadata, response = self.__client.files_download(self.__notePath(uuid))
with response:
note = unmarshalNote(response.content, metadata.client_modified)
if uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
_LOG.info("Getting photo: %s", uuid)
try:
with self.__client.files_download(self.__photoPath(uuid))[1] as response:
note.photo = response.content
except ApiError as e:
if e.error.is_path() and e.error.get_path().is_not_found():
_LOG.warning("Photo %s does not exist", uuid)
else:
raise e
return renderHtml(note)
@online
@expires
def add(self, note):
uuid = note.uuid
_LOG.info("Adding note: %s", uuid)
self.__uploadFile(self.__notePath(uuid), note.lastModified, marshalNote(note), overwrite=False)
if note.photo:
_LOG.info("Adding photo: %s", uuid)
self.__uploadFile(self.__photoPath(uuid), note.lastModified, note.photo)
elif uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
# Clean removed note if exists
if uuid in self.__notesCache and self.__notesCache[uuid].removed:
_LOG.info("Deleting REMOVED note: %s", uuid)
try:
self.__client.files_delete(self.__removedNotePath(uuid))
except ApiError:
_LOG.warning("REMOVED note %s not found", uuid)
@online
@expires
def update(self, note):
uuid = note.uuid
# Check if note exists
if self.__notesCache and (uuid not in self.__notesCache or self.__notesCache[uuid].removed):
raise RuntimeError("Note[uuid=%s] does not exist" % uuid)
_LOG.info("Updating note: %s", uuid)
self.__uploadFile(self.__notePath(uuid), note.lastModified, marshalNote(note))
if note.photo:
_LOG.info("Updating photo: %s", uuid)
self.__uploadFile(self.__photoPath(uuid), note.lastModified, note.photo)
elif uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
@online
@expires
def remove(self, note):
uuid = note.uuid
# Remove note if exists
if uuid in self.__notesCache and not self.__notesCache[uuid].removed:
_LOG.info("Deleting note: %s", uuid)
try:
self.__client.files_delete(self.__notePath(uuid))
except ApiError:
_LOG.warning("Note %s not found", uuid)
# Remove photo if exists
if uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
_LOG.info("Adding REMOVED note: %s", uuid)
self.__uploadFile(self.__removedNotePath(uuid), note.lastModified, b"")
def __uploadFile(self, path, lastModified, content, overwrite=True):
mode = WriteMode.overwrite if overwrite else WriteMode.add
self.__client.files_upload(content, path, mode=mode, client_modified=lastModified)
def __normalizeNoteName(self, name):
if self.__dayOneFlavor and name.endswith(self.__DAY_ONE_EXTENSION):
name = name[:-(len(self.__DAY_ONE_EXTENSION))]
return name
def __buildNotePath(self, parentPath, uuid):
path = parentPath + "/" + uuid
if self.__dayOneFlavor:
path += self.__DAY_ONE_EXTENSION
return path
def __notePath(self, uuid):
return self.__buildNotePath(self.__notesPath, uuid)
def __removedNotePath(self, uuid):
return self.__buildNotePath(self.__removedNotesPath, uuid)
def __photoPath(self, uuid):
return self.__photosPath + "/" + uuid + ".jpg"
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def user_authenticated(self):
try:
account = self.api_client.users_get_current_account()
log.Debug("User authenticated as ,%s" % account)
return True
except:
log.Debug('User not authenticated')
return False
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException(
'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException(
'dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError(
'dpbx error: cannot interact, but need human attention',
log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
authresult = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' %
e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(authresult.access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info(
"dpbx: Authentication failed. Trying to obtain new access token"
)
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception.
# So this line should not be reached
raise BackendException(
"dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" %
self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
progress.report_transfer(0, file_size)
if file_size < DPBX_UPLOAD_CHUNK_SIZE:
# Upload whole file at once to avoid extra server request
res_metadata = self.put_file_small(source_path, remote_path)
else:
res_metadata = self.put_file_chunked(source_path, remote_path)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException(
'dpbx: result path mismatch: %s (expected: %s)' %
(res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException(
'dpbx: result size mismatch: %s (expected: %s)' %
(res_metadata.size, file_size))
def put_file_small(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
log.Debug('dpbx,files_upload(%s, [%d bytes])' %
(remote_path, file_size))
res_metadata = self.api_client.files_upload(
f.read(),
remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
log.Debug('dpbx,files_upload(): %s' % res_metadata)
progress.report_transfer(file_size, file_size)
return res_metadata
finally:
f.close()
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug(
'dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id,
f.tell())
commit_info = CommitInfo(remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug(
'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(
buf, upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug(
'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(
buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' %
(upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError
) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset(
).correct_offset
log.Debug(
'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
% (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException(
"dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' %
(remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata
# again. Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" %
(local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
if not self.user_authenticated():
self.login()
remote_dir = '/' + urllib.unquote(
self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
res = []
try:
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
except ApiError as e:
if (isinstance(e.error, ListFolderError) and e.error.is_path()
and e.error.get_path().is_not_found()):
log.Debug('dpbx.list(%s): ignore missing folder (%s)' %
(remote_dir, e))
else:
raise
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [
x for x in file_list
if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn(
'Warning! It looks like there are automatically renamed files on backend'
)
log.Warn(
'They were probably created when using older version of duplicity.'
)
log.Warn('')
log.Warn(
'Please check your backup consistency. Most likely you will need to choose'
)
log.Warn(
'largest file from duplicity-* (number).gpg and remove brackets from its name.'
)
log.Warn('')
log.Warn(
'These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException('dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
access_token, _ = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' % e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info("dpbx: Authentication failed. Trying to obtain new access token")
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception. So this line should not be reached
raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
progress.report_transfer(0, file_size)
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed chunk
while (f.tell() < file_size) or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if len(buf) != 0:
log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset)
else:
log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
# Server reports that we should send another chunk. Most likely this is caused by
# network error during previous upload attempt. In such case we'll get expected offset
# from server and it's enough to just seek() and retry again
new_offset = error.get_incorrect_offset().correct_offset
log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something strange and no safe way to recover
raise BackendException("dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if retry_number == 0:
raise
# We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size))
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata again.
# Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
res = []
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn('Warning! It looks like there are automatically renamed files on backend')
log.Warn('They were probably created when using older version of duplicity.')
log.Warn('')
log.Warn('Please check your backup consistency. Most likely you will need to choose')
log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.')
log.Warn('')
log.Warn('These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = DROPBOX_OAUTH2_TOKEN
self.root_path = DROPBOX_ROOT_PATH
if oauth2_access_token is None:
raise ImproperlyConfigured("Você deve configurar um token em DROPBOX_OAUTH2_TOKEN ou em settings.py")
self.dbx = Dropbox(oauth2_access_token)
def user_profile(self):
self.dt = self.dbx.users_get_current_account()
print(self.dt)
def list_dir_and_files_all(self):
try:
self.dt = self.dbx.files_list_folder(self.root_path)
print('DIRETÓRIOS\n')
self.list_subdirs(self.dt)
except:
self.dt = self.dbx.files_get_metadata(self.root_path)
print('ARQUIVOS\n')
if (isinstance(self.dt, dropbox.files.FileMetadata)):
print('FUNCIONA')
file = self.list_files(self.dt)
return file
def list_subdirs(self, dt):
for entry in self.dt.entries:
i = entry.path_display
print(i)
def list_files(self):
self.dt = self.dbx.files_get_metadata(self.root_path)
file = self.dt.path_display
print(file)
return file
def upload_file(self):
#print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW)
time = datetime.datetime.now()
time = time.strftime("%Y%m%d%H%M%S")
FILEPATH = self.simple_backup()
with open(FILEPATH, 'rb') as f:
self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump')
link = self.dbx.sharing_create_shared_link_with_settings(DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump')
url = link.url
dl_url = re.sub(r"\?dl\=0", "?dl=1", url)
return dl_url
def upload_file_compress(self, filename=''):
print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW)
t = datetime.datetime.now()
t.strftime("%Y%m%d%H%M%S")
new_basename = os.path.basename(filename)
with open(filename, 'rb') as f:
self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + new_basename)
link = self.dbx.sharing_create_shared_link_with_settings(
DROPBOX_ROOT_PATH_NEW + '/' + new_basename)
url = link.url
dl_url = re.sub(r"\?dl\=0", "?dl=1", url)
return dl_url
def download_file(self, file=''):
self.file = (DROPBOX_ROOT_PATH + '/' + file)
file_name = self.file.replace('/sistemaweb/backup/', '')
print('\nDownloading... /data/backup/' + file_name)
try:
metadata, res = self.dbx.files_download(self.file)
except:
pass
metadata, res = self.dbx.files_download(file)
final_path = ROOT_DIR + '/data/backup/' + file_name
f = open(final_path, "wb")
f.write(res.content)
f.close()
if '.zip' in final_path or '.gz' in final_path:
print('Arquivo compactado...efetuando descompressão de dados.')
self.uncompress_file(final_path)
new_basename = os.path.basename(final_path).replace('.gz', '')
return new_basename
return final_path
def list_dirs_root_path(self):
self.dt = self.dbx.files_list_folder(self.root_path)
#print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n')
dir = self.download_file(self.dt.entries[-1].path_display) #self.list_files_root_path(self.dt)
return dir
def list_files_all(self):
self.dt = self.dbx.files_list_folder(self.root_path)
#print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n')
self.data = []
for entry in self.dt.entries:
data = {}
data['backup_link']
data['client_modified'] = entry.client_modified
data['size'] = str(entry.size)+" bytes"
t = entry.client_modified
time = datetime.timedelta(hours=2)
hora = datetime.datetime.strptime(str(t), '%Y-%m-%d %H:%M:%S')
now = hora - time
size = entry.size
size = str(size)+' bytes'
display = entry.path_display
print(display, now , size)
self.data.append(data)
return self.data
def simple_backup(self):
g = dbbackup.get_connector()
execute_from_command_line(["manage.py", "dbbackup", "-v", "1"])
filename = (ROOT_DIR + '/data/backup/' + g.generate_filename())
#print(filename)
return filename
def compress_file(self, filename='', n=''):
if n == '':
self.compress_all(filename)
elif n == '1':
file = self.compress_all(filename)
action = self.upload_file_compress(file)
return action
def compress_all(self, filename):
if 'C:' in filename:
new_basename = os.path.basename(filename)
file = (ROOT_DIR + '/data/backup/' + new_basename + '.gz')
f = open(filename, 'rb')
data = f.read()
f = gzip.open(file, 'wb')
f.write(data)
f.close()
#print('Arquivo compactado com sucesso!!!')
return file
else:
filepath = (ROOT_DIR + '/data/backup/' + filename)
file = (ROOT_DIR + '/data/backup/' + filename + '.gz')
f = open(filepath, 'rb')
data = f.read()
f = gzip.open(file, 'wb')
f.write(data)
f.close()
#print('Arquivo compactado com sucesso!!!')
return file
def uncompress_file(self, filename):
if 'C:' in filename:
new_basename = os.path.basename(filename).replace('.gz', '')
local_file = (ROOT_DIR + '/data/backup/' + new_basename)
f = gzip.open(filename, 'rb')
data = f.read()
f.close()
f = open(local_file, 'wb')
f.write(data)
f.close()
print('Arquivo descomprimido com sucesso!!!')
else:
local_file = (ROOT_DIR + '/data/backup/' + filename)
new_basename = os.path.basename(filename).replace('.gz', '')
file = (ROOT_DIR + '/data/backup/' + new_basename)
f = gzip.open(local_file, 'rb')
data = f.read()
f.close()
f = open(file, 'wb')
f.write(data)
f.close()
print('Arquivo descomprimido com sucesso!!!')
def restore_db(self, filepath=''):
print("VEJA O FILEPATH: ",filepath)
execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", filepath])
def restore(self): # faz a restauração do banco de dados a partir de um backup salvo na dropbox.
file_name = DropBoxStorage().list_dirs_root_path()
new_basename = file_name
if '/' in file_name:
basename = shutil.copy(file_name, ROOT_DIR + '/data/backup/')
new_basename = os.path.basename(basename)
DropBoxStorage().restore_db(new_basename)
from django.core.management import call_command
import django
django.setup()
call_command('dbrestore', '-v', '1', "--noinput", "-i", new_basename)
#execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", new_basename])
"""
def download(token, file_name, path='/'):
dbx = Dropbox(token)
meta, data = dbx.files_download(path+file_name)
return data.content
def read_dbx_file(dbx_file_path):
dbx_oauth = 'xXh2jnMU2-EAAAAAAAAZENzE7qh9xg_t45npARYvQY4sifKDsUWdq6R3_x2Lco2e'
# dbx_oauth = input('Paster your HMS dropbox OAuth here: ')
dbx = Dropbox(dbx_oauth)
_, dbx_file = dbx.files_download(dbx_file_path)
return StringIO(dbx_file.text)
class dropboxClient(object):
''' a class of methods to manage file storage on Dropbox API '''
# https://www.dropbox.com/developers/documentation/http/documentation
_class_fields = {
'schema': {
'access_token': '',
'collection_name': 'labPack',
'record_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.ogg',
'record_key_path': '/home/user/.config/collective-acuity-labpack/user-data/obs/terminal',
'record_key_comp': 'obs',
'previous_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.yaml',
'secret_key': '6tZ0rUexOiBcOse2-dgDkbeY',
'prefix': 'obs/terminal',
'delimiter': '2016-03-17T17-24-51-687845Z.yaml',
'max_results': 1
},
'components': {
'.collection_name': {
'max_length': 255,
'must_not_contain': ['/', '^\\.']
},
'.record_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.record_key_path': {
'max_length': 32767
},
'.record_key_comp': {
'max_length': 255
},
'.secret_key': {
'must_not_contain': [ '[\\t\\n\\r]' ]
},
'.max_results': {
'min_value': 1,
'integer_data': True
},
'.previous_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.prefix': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
}
},
'metadata': {
'record_optimal_bytes': 10000 * 1024,
'record_max_bytes': 150000 * 1024
}
}
def __init__(self, access_token, collection_name=''):
'''
a method to initialize the dropboxClient class
:param access_token: string with oauth2 access token for users account
'''
title = '%s.__init__' % self.__class__.__name__
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'access_token': access_token,
'collection_name': collection_name
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from dropbox import Dropbox
from dropbox.files import FileMetadata, WriteMode, DeleteArg
from dropbox.exceptions import ApiError
sys_path.insert(0, sys_path.pop())
# construct dropbox client
from labpack.compilers.objects import _method_constructor
self.dropbox = Dropbox(oauth2_access_token=access_token)
# construct dropbox objects
self.objects = _method_constructor({
'FileMetadata': FileMetadata,
'ApiError': ApiError,
'WriteMode': WriteMode,
'DeleteArg': DeleteArg
})
# construct collection name
self.collection_name = collection_name
def _import(self, record_key, record_data, overwrite=True, last_modified=0.0, **kwargs):
'''
a helper method for other storage clients to import into appdata
:param record_key: string with key for record
:param record_data: byte data for body of record
:param overwrite: [optional] boolean to overwrite existing records
:param last_modified: [optional] float to record last modified date
:param kwargs: [optional] keyword arguments from other import methods
:return: boolean indicating whether record was imported
'''
title = '%s._import' % self.__class__.__name__
# check overwrite
if not overwrite:
if self.exists(record_key):
return False
# check max size
import sys
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
# TODO: apply session upload for files greater than record_max
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', record_key):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
elif last_modified:
from labpack.records.time import labDT
mod_time = labDT.fromEpoch(last_modified)
upload_kwargs['client_modified'] = mod_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return True
def _walk(self, root_path=''):
''' an iterator method which walks the file structure of the dropbox collection '''
title = '%s._walk' % self.__class__.__name__
if root_path:
root_path = '/%s' % root_path
try:
response = self.dropbox.files_list_folder(path=root_path, recursive=True)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
if response.has_more:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
except:
raise DropboxConnectionError(title)
def exists(self, record_key):
'''
a method to determine if a record exists in collection
:param record_key: string with key of record
:return: boolean reporting status
'''
title = '%s.exists' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# send get metadata request
file_path = '/%s' % record_key
try:
self.dropbox.files_get_metadata(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
return False
else:
raise DropboxConnectionError(title)
return True
def save(self, record_key, record_data, overwrite=True, secret_key=''):
'''
a method to create a record in the collection folder
:param record_key: string with name to assign to record (see NOTES below)
:param record_data: byte data for record body
:param overwrite: [optional] boolean to overwrite records with same name
:param secret_key: [optional] string with key to encrypt data
:return: string with name of record
NOTE: record_key may only contain alphanumeric, /, _, . or -
characters and may not begin with the . or / character.
NOTE: using one or more / characters splits the key into
separate segments. these segments will appear as a
sub directories inside the record collection and each
segment is used as a separate index for that record
when using the list method
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
'''
title = '%s.save' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate byte data
if not isinstance(record_data, bytes):
raise ValueError('%s(record_data=b"...") must be byte data.' % title)
# construct and validate file path
file_root, file_name = os.path.split(record_key)
self.fields.validate(file_name, '.record_key_comp')
while file_root:
file_root, path_node = os.path.split(file_root)
self.fields.validate(path_node, '.record_key_comp')
# check overwrite exception
if not overwrite:
if self.exists(record_key):
raise Exception('%s(record_key="%s") already exists. To overwrite, set overwrite=True' % (title, record_key))
# check size of file
import sys
record_optimal = self.fields.metadata['record_optimal_bytes']
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
elif record_size > record_optimal:
print('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))
# TODO add upload session for support of files over 150MB
# http://dropbox-sdk-python.readthedocs.io/en/latest/moduledoc.html#dropbox.dropbox.Dropbox.files_upload_session_start
# encrypt data
if secret_key:
from labpack.encryption import cryptolab
record_data, secret_key = cryptolab.encrypt(record_data, secret_key)
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', file_name):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return record_key
def load(self, record_key, secret_key=''):
'''
a method to retrieve byte data of appdata record
:param record_key: string with name of record
:param secret_key: [optional] string used to decrypt data
:return: byte data for record body
'''
title = '%s.load' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct file path
file_path = '/%s' % record_key
# request file data
try:
metadata, response = self.dropbox.files_download(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
raise Exception('%s(record_key=%s) does not exist.' % (title, record_key))
else:
raise DropboxConnectionError(title)
record_data = response.content
# decrypt (if necessary)
if secret_key:
from labpack.encryption import cryptolab
record_data = cryptolab.decrypt(record_data, secret_key)
return record_data
def conditional_filter(self, path_filters):
''' a method to construct a conditional filter function for list method
:param path_filters: dictionary or list of dictionaries with query criteria
:return: filter_function object
path_filters:
[ { 0: { conditional operators }, 1: { conditional_operators }, ... } ]
conditional operators:
"byte_data": false,
"discrete_values": [ "" ],
"excluded_values": [ "" ],
"greater_than": "",
"less_than": "",
"max_length": 0,
"max_value": "",
"min_length": 0,
"min_value": "",
"must_contain": [ "" ],
"must_not_contain": [ "" ],
"contains_either": [ "" ]
'''
title = '%s.conditional_filter' % self.__class__.__name__
from labpack.compilers.filters import positional_filter
filter_function = positional_filter(path_filters, title)
return filter_function
def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
'''
a method to list keys in the dropbox collection
:param prefix: string with prefix value to filter results
:param delimiter: string with value which results must not contain (after prefix)
:param filter_function: (positional arguments) function used to filter results
:param max_results: integer with maximum number of results to return
:param previous_key: string with key in collection to begin search after
:return: list of key strings
NOTE: each key string can be divided into one or more segments
based upon the / characters which occur in the key string as
well as its file extension type. if the key string represents
a file path, then each directory in the path, the file name
and the file extension are all separate indexed values.
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
it is possible to filter the records in the collection according
to one or more of these path segments using a filter_function.
NOTE: the filter_function must be able to accept an array of positional
arguments and return a value that can evaluate to true or false.
while searching the records, list produces an array of strings
which represent the directory structure in relative path of each
key string. if a filter_function is provided, this list of strings
is fed to the filter function. if the function evaluates this input
and returns a true value the file will be included in the list
results.
'''
title = '%s.list' % self.__class__.__name__
# validate input
input_fields = {
'prefix': prefix,
'delimiter': delimiter,
'max_results': max_results,
'previous_key': previous_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate filter function
if filter_function:
try:
path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
filter_function(*path_segments)
except:
err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
raise TypeError('%s must accept positional arguments.' % err_msg)
# construct empty results list
results_list = []
check_key = True
if previous_key:
check_key = False
# determine root path
root_path = ''
if prefix:
from os import path
root_path, file_name = path.split(prefix)
# iterate over dropbox files
for file_path in self._walk(root_path):
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
if record_key == previous_key:
check_key = True
# find starting point
if not check_key:
continue
# apply prefix filter
partial_key = record_key
if prefix:
if record_key.find(prefix) == 0:
partial_key = record_key[len(prefix):]
else:
continue
# apply delimiter filter
if delimiter:
if partial_key.find(delimiter) > -1:
continue
# apply filter function
if filter_function:
if filter_function(*path_segments):
results_list.append(record_key)
else:
results_list.append(record_key)
# return results list
if len(results_list) == max_results:
return results_list
return results_list
def delete(self, record_key):
''' a method to delete a file
:param record_key: string with name of file
:return: string reporting outcome
'''
title = '%s.delete' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate existence of file
if not self.exists(record_key):
exit_msg = '%s does not exist.' % record_key
return exit_msg
# remove file
current_dir = os.path.split(record_key)[0]
try:
file_path = '/%s' % record_key
self.dropbox.files_delete(file_path)
except:
raise DropboxConnectionError(title)
# remove empty directories in path to file
try:
while current_dir:
folder_path = '/%s' % current_dir
response = self.dropbox.files_list_folder(folder_path)
if not response.entries:
self.dropbox.files_delete(folder_path)
current_dir = os.path.split(current_dir)[0]
else:
break
except:
raise DropboxConnectionError(title)
exit_msg = '%s has been deleted.' % record_key
return exit_msg
def remove(self):
'''
a method to remove all records in the collection
NOTE: this method removes all the files in the collection, but the
collection folder itself created by oauth2 cannot be removed.
only the user can remove the app folder
:return: string with confirmation of deletion
'''
title = '%s.remove' % self.__class__.__name__
# get contents in root
try:
response = self.dropbox.files_list_folder(path='')
except:
raise DropboxConnectionError(title)
# populate delete list
delete_list = []
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
# continue retrieval if folder is large
if response.has_more:
try:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
except:
raise DropboxConnectionError(title)
# send batch delete request
try:
self.dropbox.files_delete_batch(delete_list)
except:
raise DropboxConnectionError(title)
# return outcome
insert = 'collection'
if self.collection_name:
insert = self.collection_name
exit_msg = 'Contents of %s will been removed from Dropbox.' % insert
return exit_msg
def export(self, storage_client, overwrite=True):
'''
a method to export all the records in collection to another platform
:param storage_client: class object with storage client methods
:return: string with exit message
'''
title = '%s.export' % self.__class__.__name__
# validate storage client
method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ]
for method in method_list:
if not getattr(storage_client, method, None):
from labpack.parsing.grammar import join_words
raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list)))
# walk collection folder to find files
import os
count = 0
skipped = 0
for file_path in self._walk():
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
file_path = '/%s' % file_path
# retrieve data and metadata
try:
metadata, response = self.dropbox.files_download(file_path)
except:
raise DropboxConnectionError(title)
record_data = response.content
client_modified = metadata.client_modified
# import record into storage client
last_modified = 0.0
if client_modified:
from dateutil.tz import tzutc
from labpack.records.time import labDT
last_modified = labDT.fromPython(client_modified.replace(tzinfo=tzutc())).epoch()
outcome = storage_client._import(record_key, record_data, overwrite=overwrite, last_modified=last_modified)
if outcome:
count += 1
else:
skipped += 1
# report outcome
plural = ''
skip_insert = ''
new_folder = storage_client.collection_name
if count != 1:
plural = 's'
if skipped > 0:
skip_plural = ''
if skipped > 1:
skip_plural = 's'
skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural)
exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert)
return exit_msg
class DropboxConnection(object):
class Parameters(object):
def __init__(self, auth_token, remote_dir_path):
self._auth_token = auth_token
self._remote_dir_path = remote_dir_path
@property
def auth_token(self):
return self._auth_token
@property
def remote_dir_path(self):
return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path
def __str__(self):
return """
remote_dir_path: {}
""".format(self.remote_dir_path)
@classmethod
def get_client_from_params(cls, dropbox_parameters):
#Dropbox connection placeholder
dropbox = None
if dropbox_parameters:
dropbox_params = DropboxConnection.Parameters(
dropbox_parameters[0], dropbox_parameters[1])
dropbox = DropboxConnection(dropbox_params)
return dropbox
@classmethod
def get_client(cls, auth_token, remote_dir_path):
#Initialize the paramters
params = DropboxConnection.Parameters(auth_token, remote_dir_path)
#Create dropbox client
client = DropboxConnection(params)
return client
def __init__(self, params):
#Required parameters
self._params = params
#Derived parameters
self._client = Dropbox(self._params.auth_token)
#Logging
self._logger = logging.get_logger(__name__)
def upload(self, source_file_path):
"""It upload the source files to the dropbox.
Arguments:
source_file_path {string} -- The source file path.
Raises:
ValueError -- It raise value error for invalid files.
"""
#Validate parameters
if not source_file_path:
raise ValueError("Invalid source file path")
with open(source_file_path, 'rb') as handle:
#Source file name
source_file_name = Path(source_file_path).name
#Remote path
remote_file_path = (self._params.remote_dir_path /
source_file_name).as_posix()
#File size
upload_size = path.getsize(source_file_path)
#Upload the files based on the upload size
if upload_size <= constants.DROPBOX_CHUNK_SIZE:
self._logger.info(
'Preparing to upload small file: %s with size: %d to: %s',
source_file_path, upload_size, remote_file_path)
self._upload_small_file(handle, remote_file_path)
else:
self._logger.info(
'Preparing to upload large file: %s with size: %d to: %s',
source_file_path, upload_size, remote_file_path)
self._upload_large_file(handle, upload_size, remote_file_path)
self._logger.info('Uploaded: %s', source_file_path)
def _upload_small_file(self, handle, remote_file_path):
"""It uploads a small source files to the dropbox.
Arguments:
handle {A File handle} -- The source file handle.
remote_file_path {string} -- The destination path of the file.
"""
self._client.files_upload(handle.read(),
remote_file_path,
mode=Dropbox_WriteMode.overwrite)
def _upload_large_file(self, handle, upload_size, remote_file_path):
"""It uploads a large source files to the dropbox.
Arguments:
handle {A File handle} -- The source file handle.
upload_size {int} -- The number of bytes to be uploaded.
remote_file_path {string} -- The destination path of the file.
"""
#Upload session
session = self._client.files_upload_session_start(
handle.read(constants.DROPBOX_CHUNK_SIZE))
cursor = Dropbox_UploadSessionCursor(session_id=session.session_id,
offset=handle.tell())
#Upload look
with tqdm(desc='Uploading: {}'.format(remote_file_path),
total=upload_size) as pbar:
#Update the progress bar for the session start reads
pbar.update(handle.tell())
while handle.tell() < upload_size:
#Calculate remaining bytes
remaining_bytes = upload_size - handle.tell()
#If it is the last chunk, finalize the upload
if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE:
#Commit info
commit = Dropbox_CommitInfo(
path=remote_file_path,
mode=Dropbox_WriteMode.overwrite)
#Finish upload
self._client.files_upload_session_finish(
handle.read(remaining_bytes), cursor, commit)
#Update progress
pbar.update(remaining_bytes)
#More than chunk size remaining to upload
else:
self._client.files_upload_session_append_v2(
handle.read(constants.DROPBOX_CHUNK_SIZE), cursor)
#Update the cursor
cursor.offset = handle.tell()
#Update the progress
pbar.update(constants.DROPBOX_CHUNK_SIZE)
#Refresh the progress bar
pbar.refresh()
def download(self, remote_file_path):
"""It downloads the remote files from the dropbox.
Arguments:
remote_file_path {Path} -- The path to the remote file.
Raises:
ValueError -- It raise value error for invalid file name.
"""
#Validate parameters
if not remote_file_path:
raise ValueError("Invalid remote file path")
#Destination file path
dest_file_path = remote_file_path
#Full remote file path
remote_file_path = self._params.remote_dir_path / remote_file_path
#Download file size placeholder
download_size = 0
try:
download_size = self._client.files_get_metadata(
remote_file_path.as_posix()).size
except ApiError as e:
raise FileNotFoundError(
'File: {} is not found'.format(remote_file_path.as_posix()), e)
self._logger.info('Preparing file download: %s with size: %d to: %s',
remote_file_path, download_size, dest_file_path)
#Download the file
self._download_file(dest_file_path, remote_file_path, download_size)
self._logger.info('Completed the file download: %s to: %s',
remote_file_path, dest_file_path)
def _download_file(self, dest_file_path, remote_file_path, download_size):
"""It downloads the remote files from the dropbox.
Arguments:
remote_file_path {A Path object} -- The path of the remote file.
dest_file_path {string} -- The destination file path.
download_size {int} -- The number of bytes to be downloaded.
"""
#Download
_, result = self._client.files_download(remote_file_path.as_posix())
#Temporary dest_file_name
tmp_dest_file_path = "{}.tmp".format(dest_file_path)
with open(tmp_dest_file_path, 'wb') as handle:
with tqdm(desc='Downloading: {}'.format(
remote_file_path.as_posix()),
total=download_size) as pbar:
for bytes_read in result.iter_content(
constants.DROPBOX_CHUNK_SIZE):
handle.write(bytes_read)
#Update the progress
pbar.update(len(bytes_read))
if Path(tmp_dest_file_path).exists():
rename(tmp_dest_file_path, dest_file_path)
def list(self, dir_path=Path(), file_name_prefix=''):
"""It lists the files in the dropbox folder that starts with the given prefix.
Arguments:
file_name_prefix {string} -- The prefix to filter the results.
"""
#Candidate directory whose contents are to be listed
candidate_dir_path = self._params.remote_dir_path / dir_path
self._logger.info('Enumerating: %s with file_name_prefix: %s',
candidate_dir_path, file_name_prefix)
#Call the downstream API
response = self._client.files_list_folder(
candidate_dir_path.as_posix())
#Output list placeholder
files = []
sizes = []
if response.entries:
#Log the response summary
self._logger.info('Got %d files in: %s', len(response.entries),
candidate_dir_path)
#Extract the name of files satisfying the input criteria from the response entries.
file_infos = [(dir_path / entry.name, entry.size)
for entry in response.entries
if entry.name.startswith(file_name_prefix)]
files, sizes = zip(*file_infos)
return files, sizes
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(
self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None,
):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {"mode": WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
"recursive": True,
"include_non_downloadable_files": False,
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as e:
if _is_file_not_found_error(e):
raise KeyError(f"Key doesn't exist: {k}")
raise
if not contents_response.status_code:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
return contents_response.content
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {'mode': WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
'recursive': True,
'include_non_downloadable_files': False
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as err:
if _is_file_not_found_error(err):
raise KeyError(f"Key doesn't exist: {k}")
else:
raise ValueError(
"Some unknown error happened (sorry, the lazy dev didn't tell me more than that)."
)
if contents_response.status_code:
return contents_response.content
else:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
# def _entry_is_dir(entry):
# return not hasattr(entry, 'is_downloadable')
#
#
# def _entry_is_file(entry):
# return hasattr(entry, 'is_downloadable')
#
#
# def _extend_path(path, extension):
# extend_path = '/' + path + '/' + extension + '/'
# extend_path.replace('//', '/')
# return extend_path
#
#
# class DropboxLinkPersister(DropboxPersister):
# def __init__(self, url, oauth2_access_token):
# self._con = Dropbox(oauth2_access_token)
# self.url = url
# self.shared_link = SharedLink(url=url)
#
# def _yield_from_files_list_folder(self, path, path_gen):
# """
# yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
# in a depth search manner.
# """
# for x in path_gen.entries:
# try:
# if _entry_is_file(x):
# yield x.name
# else:
# folder_path = _extend_path(path, x.name)
# yield from self._get_path_gen_from_path(path=folder_path)
# except Exception as e:
# print(e)
# if path_gen.has_more:
# yield from self._get_path_gen_from_cursor(path_gen.cursor, path=path)
#
# def _get_path_gen_from_path(self, path):
# path_gen = self._con.files_list_folder(path=path, recursive=False, shared_link=self.shared_link)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def _get_path_gen_from_cursor(self, cursor, path):
# path_gen = self._con.files_list_folder_continue(cursor)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def __iter__(self):
# yield from self._get_path_gen_from_path(path='')
