class DropboxLinkReaderWithToken(ReadOnlyMixin, DropboxPersister):
def __init__(self, url, oauth2_access_token):
self._con = Dropbox(oauth2_access_token)
self.url = url
self.shared_link = SharedLink(url=url)
def _yield_from_files_list_folder(self, path, path_gen):
"""
yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
in a depth search manner.
"""
for x in path_gen.entries:
if _entry_is_file(x):
yield x.path_display
else:
folder_path = _extend_path(path, x.name)
yield from self._get_path_gen_from_path(path=folder_path)
if path_gen.has_more:
yield from self._get_path_gen_from_cursor(path_gen.cursor,
path=path)
def _get_path_gen_from_path(self, path):
path_gen = self._con.files_list_folder(path=path,
recursive=False,
shared_link=self.shared_link)
yield from self._yield_from_files_list_folder(path, path_gen)
def _get_path_gen_from_cursor(self, cursor, path):
path_gen = self._con.files_list_folder_continue(cursor)
yield from self._yield_from_files_list_folder(path, path_gen)
def __iter__(self):
yield from self._get_path_gen_from_path(path="")
def list_files(dbx: dropbox.Dropbox, lookup_folder: list):
all_folder_entries = []
for folder in lookup_folder:
response = dbx.files_list_folder(folder, recursive=True)
entries = response.entries
while response.has_more:
print(dt.now(), 'Listing files... Found so far:', len(entries))
response = dbx.files_list_folder_continue(response.cursor)
entries += response.entries
all_folder_entries += entries
return all_folder_entries
def sync():
challenge = request.args.get('challenge')
if challenge is not None:
return challenge
"""Synchronize database with todo.txt"""
dropbox = db.session.query(models.Dropbox).first() # type: models.Dropbox
# Make sure this is a valid request from Dropbox
signature = request.headers.get('X-Dropbox-Signature')
if not hmac.compare_digest(signature, hmac.new(dropbox.secret.encode(), request.data, sha256).hexdigest()):
app.logger.warn('Invalid sync request attempted')
abort(403)
dbx = Dropbox(dropbox.access_token)
if dropbox.cursor is None:
result = dbx.files_list_folder(path=os.path.dirname(dropbox.file_location))
else:
result = dbx.files_list_folder_continue(cursor=dropbox.cursor)
# Check if todo.txt was changed
found = False
for metadata in result.entries: # type: Metadata
if metadata.path_lower == dropbox.file_location.lower():
found = True
break
if not found:
dropbox.cursor = result.cursor
db.session.merge(dropbox)
db.session.commit()
return ''
app.logger.info('Sync request made')
try:
md, res = dbx.files_download(path=dropbox.file_location)
except ApiError as err:
if err.error.is_path() and err.error.get_path().is_not_found():
return 'File not found: ' + dropbox.file_location
return 'Other error occurred'
update_todos(content=res.content)
dropbox.cursor = result.cursor
db.session.merge(dropbox)
db.session.commit()
return ''
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
dbx = Dropbox(token)
has_more = True
trello_client = trello.TrelloClient(TRELLO_API_KEY, token=TRELLO_API_TOKEN)
while has_more:
if cursor is None:
result = dbx.files_list_folder(path='/remote_workspace')
else:
result = dbx.files_list_folder_continue(cursor)
for entry in result.entries:
# Ignore deleted files, folders, and non-markdown files
if (isinstance(entry, DeletedMetadata) or isinstance(entry, FolderMetadata)):
continue
card = get_card_by_name(trello_client, entry.name.encode('utf-8'))
if(card == False):
trello_post(trello_client, entry.name.encode('utf-8'))
continue
card.set_pos("top")
card.comment("update! revision: %s" % entry.rev)
revs = dbx.files_list_revisions(entry.path_lower)
if(card.list_id == "577db30f129e87073996cc1a" and len(revs.entries) >= 2):
card.change_list("577db3127b9a95030e956ab8")
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result.has_more
def process_user(account):
'''Call /files/list_folder for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', account)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
dbx = Dropbox(token)
has_more = True
while has_more:
if cursor is None:
result = dbx.files_list_folder(path='')
else:
result = dbx.files_list_folder_continue(cursor)
for entry in result.entries:
# Ignore deleted files, folders, and non-markdown files
if (isinstance(entry, DeletedMetadata)
or isinstance(entry, FolderMetadata)
or not entry.path_lower.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
_, resp = dbx.files_download(entry.path_lower)
html = markdown(resp.content.decode("utf-8"))
dbx.files_upload(bytes(html, encoding='utf-8'),
entry.path_lower[:-3] + '.html',
mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException('dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
access_token, _ = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' % e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info("dpbx: Authentication failed. Trying to obtain new access token")
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception. So this line should not be reached
raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
progress.report_transfer(0, file_size)
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed chunk
while (f.tell() < file_size) or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if len(buf) != 0:
log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset)
else:
log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
# Server reports that we should send another chunk. Most likely this is caused by
# network error during previous upload attempt. In such case we'll get expected offset
# from server and it's enough to just seek() and retry again
new_offset = error.get_incorrect_offset().correct_offset
log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something strange and no safe way to recover
raise BackendException("dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if retry_number == 0:
raise
# We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size))
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata again.
# Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
res = []
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn('Warning! It looks like there are automatically renamed files on backend')
log.Warn('They were probably created when using older version of duplicity.')
log.Warn('')
log.Warn('Please check your backup consistency. Most likely you will need to choose')
log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.')
log.Warn('')
log.Warn('These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
class dropboxClient(object):
''' a class of methods to manage file storage on Dropbox API '''
# https://www.dropbox.com/developers/documentation/http/documentation
_class_fields = {
'schema': {
'access_token': '',
'collection_name': 'labPack',
'record_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.ogg',
'record_key_path': '/home/user/.config/collective-acuity-labpack/user-data/obs/terminal',
'record_key_comp': 'obs',
'previous_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.yaml',
'secret_key': '6tZ0rUexOiBcOse2-dgDkbeY',
'prefix': 'obs/terminal',
'delimiter': '2016-03-17T17-24-51-687845Z.yaml',
'max_results': 1
},
'components': {
'.collection_name': {
'max_length': 255,
'must_not_contain': ['/', '^\\.']
},
'.record_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.record_key_path': {
'max_length': 32767
},
'.record_key_comp': {
'max_length': 255
},
'.secret_key': {
'must_not_contain': [ '[\\t\\n\\r]' ]
},
'.max_results': {
'min_value': 1,
'integer_data': True
},
'.previous_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.prefix': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
}
},
'metadata': {
'record_optimal_bytes': 10000 * 1024,
'record_max_bytes': 150000 * 1024
}
}
def __init__(self, access_token, collection_name=''):
'''
a method to initialize the dropboxClient class
:param access_token: string with oauth2 access token for users account
'''
title = '%s.__init__' % self.__class__.__name__
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'access_token': access_token,
'collection_name': collection_name
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from dropbox import Dropbox
from dropbox.files import FileMetadata, WriteMode, DeleteArg
from dropbox.exceptions import ApiError
sys_path.insert(0, sys_path.pop())
# construct dropbox client
from labpack.compilers.objects import _method_constructor
self.dropbox = Dropbox(oauth2_access_token=access_token)
# construct dropbox objects
self.objects = _method_constructor({
'FileMetadata': FileMetadata,
'ApiError': ApiError,
'WriteMode': WriteMode,
'DeleteArg': DeleteArg
})
# construct collection name
self.collection_name = collection_name
def _import(self, record_key, record_data, overwrite=True, last_modified=0.0, **kwargs):
'''
a helper method for other storage clients to import into appdata
:param record_key: string with key for record
:param record_data: byte data for body of record
:param overwrite: [optional] boolean to overwrite existing records
:param last_modified: [optional] float to record last modified date
:param kwargs: [optional] keyword arguments from other import methods
:return: boolean indicating whether record was imported
'''
title = '%s._import' % self.__class__.__name__
# check overwrite
if not overwrite:
if self.exists(record_key):
return False
# check max size
import sys
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
# TODO: apply session upload for files greater than record_max
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', record_key):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
elif last_modified:
from labpack.records.time import labDT
mod_time = labDT.fromEpoch(last_modified)
upload_kwargs['client_modified'] = mod_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return True
def _walk(self, root_path=''):
''' an iterator method which walks the file structure of the dropbox collection '''
title = '%s._walk' % self.__class__.__name__
if root_path:
root_path = '/%s' % root_path
try:
response = self.dropbox.files_list_folder(path=root_path, recursive=True)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
if response.has_more:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
except:
raise DropboxConnectionError(title)
def exists(self, record_key):
'''
a method to determine if a record exists in collection
:param record_key: string with key of record
:return: boolean reporting status
'''
title = '%s.exists' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# send get metadata request
file_path = '/%s' % record_key
try:
self.dropbox.files_get_metadata(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
return False
else:
raise DropboxConnectionError(title)
return True
def save(self, record_key, record_data, overwrite=True, secret_key=''):
'''
a method to create a record in the collection folder
:param record_key: string with name to assign to record (see NOTES below)
:param record_data: byte data for record body
:param overwrite: [optional] boolean to overwrite records with same name
:param secret_key: [optional] string with key to encrypt data
:return: string with name of record
NOTE: record_key may only contain alphanumeric, /, _, . or -
characters and may not begin with the . or / character.
NOTE: using one or more / characters splits the key into
separate segments. these segments will appear as a
sub directories inside the record collection and each
segment is used as a separate index for that record
when using the list method
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
'''
title = '%s.save' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate byte data
if not isinstance(record_data, bytes):
raise ValueError('%s(record_data=b"...") must be byte data.' % title)
# construct and validate file path
file_root, file_name = os.path.split(record_key)
self.fields.validate(file_name, '.record_key_comp')
while file_root:
file_root, path_node = os.path.split(file_root)
self.fields.validate(path_node, '.record_key_comp')
# check overwrite exception
if not overwrite:
if self.exists(record_key):
raise Exception('%s(record_key="%s") already exists. To overwrite, set overwrite=True' % (title, record_key))
# check size of file
import sys
record_optimal = self.fields.metadata['record_optimal_bytes']
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
elif record_size > record_optimal:
print('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))
# TODO add upload session for support of files over 150MB
# http://dropbox-sdk-python.readthedocs.io/en/latest/moduledoc.html#dropbox.dropbox.Dropbox.files_upload_session_start
# encrypt data
if secret_key:
from labpack.encryption import cryptolab
record_data, secret_key = cryptolab.encrypt(record_data, secret_key)
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', file_name):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return record_key
def load(self, record_key, secret_key=''):
'''
a method to retrieve byte data of appdata record
:param record_key: string with name of record
:param secret_key: [optional] string used to decrypt data
:return: byte data for record body
'''
title = '%s.load' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct file path
file_path = '/%s' % record_key
# request file data
try:
metadata, response = self.dropbox.files_download(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
raise Exception('%s(record_key=%s) does not exist.' % (title, record_key))
else:
raise DropboxConnectionError(title)
record_data = response.content
# decrypt (if necessary)
if secret_key:
from labpack.encryption import cryptolab
record_data = cryptolab.decrypt(record_data, secret_key)
return record_data
def conditional_filter(self, path_filters):
''' a method to construct a conditional filter function for list method
:param path_filters: dictionary or list of dictionaries with query criteria
:return: filter_function object
path_filters:
[ { 0: { conditional operators }, 1: { conditional_operators }, ... } ]
conditional operators:
"byte_data": false,
"discrete_values": [ "" ],
"excluded_values": [ "" ],
"greater_than": "",
"less_than": "",
"max_length": 0,
"max_value": "",
"min_length": 0,
"min_value": "",
"must_contain": [ "" ],
"must_not_contain": [ "" ],
"contains_either": [ "" ]
'''
title = '%s.conditional_filter' % self.__class__.__name__
from labpack.compilers.filters import positional_filter
filter_function = positional_filter(path_filters, title)
return filter_function
def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
'''
a method to list keys in the dropbox collection
:param prefix: string with prefix value to filter results
:param delimiter: string with value which results must not contain (after prefix)
:param filter_function: (positional arguments) function used to filter results
:param max_results: integer with maximum number of results to return
:param previous_key: string with key in collection to begin search after
:return: list of key strings
NOTE: each key string can be divided into one or more segments
based upon the / characters which occur in the key string as
well as its file extension type. if the key string represents
a file path, then each directory in the path, the file name
and the file extension are all separate indexed values.
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
it is possible to filter the records in the collection according
to one or more of these path segments using a filter_function.
NOTE: the filter_function must be able to accept an array of positional
arguments and return a value that can evaluate to true or false.
while searching the records, list produces an array of strings
which represent the directory structure in relative path of each
key string. if a filter_function is provided, this list of strings
is fed to the filter function. if the function evaluates this input
and returns a true value the file will be included in the list
results.
'''
title = '%s.list' % self.__class__.__name__
# validate input
input_fields = {
'prefix': prefix,
'delimiter': delimiter,
'max_results': max_results,
'previous_key': previous_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate filter function
if filter_function:
try:
path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
filter_function(*path_segments)
except:
err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
raise TypeError('%s must accept positional arguments.' % err_msg)
# construct empty results list
results_list = []
check_key = True
if previous_key:
check_key = False
# determine root path
root_path = ''
if prefix:
from os import path
root_path, file_name = path.split(prefix)
# iterate over dropbox files
for file_path in self._walk(root_path):
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
if record_key == previous_key:
check_key = True
# find starting point
if not check_key:
continue
# apply prefix filter
partial_key = record_key
if prefix:
if record_key.find(prefix) == 0:
partial_key = record_key[len(prefix):]
else:
continue
# apply delimiter filter
if delimiter:
if partial_key.find(delimiter) > -1:
continue
# apply filter function
if filter_function:
if filter_function(*path_segments):
results_list.append(record_key)
else:
results_list.append(record_key)
# return results list
if len(results_list) == max_results:
return results_list
return results_list
def delete(self, record_key):
''' a method to delete a file
:param record_key: string with name of file
:return: string reporting outcome
'''
title = '%s.delete' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate existence of file
if not self.exists(record_key):
exit_msg = '%s does not exist.' % record_key
return exit_msg
# remove file
current_dir = os.path.split(record_key)[0]
try:
file_path = '/%s' % record_key
self.dropbox.files_delete(file_path)
except:
raise DropboxConnectionError(title)
# remove empty directories in path to file
try:
while current_dir:
folder_path = '/%s' % current_dir
response = self.dropbox.files_list_folder(folder_path)
if not response.entries:
self.dropbox.files_delete(folder_path)
current_dir = os.path.split(current_dir)[0]
else:
break
except:
raise DropboxConnectionError(title)
exit_msg = '%s has been deleted.' % record_key
return exit_msg
def remove(self):
'''
a method to remove all records in the collection
NOTE: this method removes all the files in the collection, but the
collection folder itself created by oauth2 cannot be removed.
only the user can remove the app folder
:return: string with confirmation of deletion
'''
title = '%s.remove' % self.__class__.__name__
# get contents in root
try:
response = self.dropbox.files_list_folder(path='')
except:
raise DropboxConnectionError(title)
# populate delete list
delete_list = []
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
# continue retrieval if folder is large
if response.has_more:
try:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
except:
raise DropboxConnectionError(title)
# send batch delete request
try:
self.dropbox.files_delete_batch(delete_list)
except:
raise DropboxConnectionError(title)
# return outcome
insert = 'collection'
if self.collection_name:
insert = self.collection_name
exit_msg = 'Contents of %s will been removed from Dropbox.' % insert
return exit_msg
def export(self, storage_client, overwrite=True):
'''
a method to export all the records in collection to another platform
:param storage_client: class object with storage client methods
:return: string with exit message
'''
title = '%s.export' % self.__class__.__name__
# validate storage client
method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ]
for method in method_list:
if not getattr(storage_client, method, None):
from labpack.parsing.grammar import join_words
raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list)))
# walk collection folder to find files
import os
count = 0
skipped = 0
for file_path in self._walk():
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
file_path = '/%s' % file_path
# retrieve data and metadata
try:
metadata, response = self.dropbox.files_download(file_path)
except:
raise DropboxConnectionError(title)
record_data = response.content
client_modified = metadata.client_modified
# import record into storage client
last_modified = 0.0
if client_modified:
from dateutil.tz import tzutc
from labpack.records.time import labDT
last_modified = labDT.fromPython(client_modified.replace(tzinfo=tzutc())).epoch()
outcome = storage_client._import(record_key, record_data, overwrite=overwrite, last_modified=last_modified)
if outcome:
count += 1
else:
skipped += 1
# report outcome
plural = ''
skip_insert = ''
new_folder = storage_client.collection_name
if count != 1:
plural = 's'
if skipped > 0:
skip_plural = ''
if skipped > 1:
skip_plural = 's'
skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural)
exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert)
return exit_msg
def process_user(account):
'''
Call /files/list_folder for the given user ID and process any changes.
'''
print("*" * 30, " - PROCESS_USER - ", "*" * 30)
# OAuth token for the user
token = redis_client.hget('tokens', account)
print("token: ", token)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
print("cursor: ", cursor)
output_file_extension = ".csv"
input_file_extensions = [
".textgrid",
".TextGrid",
]
processed_marker = "_cleaned"
dbx = Dropbox(token.decode())
has_more = True
while has_more:
print("there's more!")
if cursor is None:
print("cursor is 'None'!")
result = dbx.files_list_folder(path='')
print("result: ", result)
else:
print("entering files_list_folder_continue...")
result = dbx.files_list_folder_continue(cursor.decode())
print("result: ", result)
for entry in result.entries:
print("entry: ", entry)
# yapf: disable
if (
isinstance(entry, DeletedMetadata) or
isinstance(entry, FolderMetadata) or
not any(entry.path_lower.endswith(e)
for e in input_file_extensions)
):
# yapf: enable
print("skipping.")
continue
_, resp = dbx.files_download(entry.path_lower)
print("processing data...")
processed_data = clean_data(resp.content)
dbx.files_upload(processed_data,
entry.path_lower[:-4] + processed_marker +
output_file_extension,
mode=WriteMode('add'))
#mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {'mode': WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
'recursive': True,
'include_non_downloadable_files': False
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as err:
if _is_file_not_found_error(err):
raise KeyError(f"Key doesn't exist: {k}")
else:
raise ValueError(
"Some unknown error happened (sorry, the lazy dev didn't tell me more than that)."
)
if contents_response.status_code:
return contents_response.content
else:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
# def _entry_is_dir(entry):
# return not hasattr(entry, 'is_downloadable')
#
#
# def _entry_is_file(entry):
# return hasattr(entry, 'is_downloadable')
#
#
# def _extend_path(path, extension):
# extend_path = '/' + path + '/' + extension + '/'
# extend_path.replace('//', '/')
# return extend_path
#
#
# class DropboxLinkPersister(DropboxPersister):
# def __init__(self, url, oauth2_access_token):
# self._con = Dropbox(oauth2_access_token)
# self.url = url
# self.shared_link = SharedLink(url=url)
#
# def _yield_from_files_list_folder(self, path, path_gen):
# """
# yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
# in a depth search manner.
# """
# for x in path_gen.entries:
# try:
# if _entry_is_file(x):
# yield x.name
# else:
# folder_path = _extend_path(path, x.name)
# yield from self._get_path_gen_from_path(path=folder_path)
# except Exception as e:
# print(e)
# if path_gen.has_more:
# yield from self._get_path_gen_from_cursor(path_gen.cursor, path=path)
#
# def _get_path_gen_from_path(self, path):
# path_gen = self._con.files_list_folder(path=path, recursive=False, shared_link=self.shared_link)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def _get_path_gen_from_cursor(self, cursor, path):
# path_gen = self._con.files_list_folder_continue(cursor)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def __iter__(self):
# yield from self._get_path_gen_from_path(path='')
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(
self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None,
):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {"mode": WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
"recursive": True,
"include_non_downloadable_files": False,
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as e:
if _is_file_not_found_error(e):
raise KeyError(f"Key doesn't exist: {k}")
raise
if not contents_response.status_code:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
return contents_response.content
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
class rpiImageDbxClass(rpiBaseClass):
"""
Implements the rpiImageDb class to manage images in a remote directory (dropbox).
"""
def __init__(self, name, rpi_apscheduler, rpi_events, rpi_config, cam_rpififo=None):
### Get the Dbx error event
#self._eventDbErr = rpi_events.eventErrList["DBXJob"]
### Get the custom config parameters
self._config = rpi_config
### Get FIFO buffer for images from the camera (deque)
self._imageFIFO = cam_rpififo
### The FIFO buffer for the uploaded images (deque)
self.imageUpldFIFO = rpififo.rpiFIFOClass([], 576)
self.imageUpldFIFO.crtSubDir = ''
### Init base class
super().__init__(name, rpi_apscheduler, rpi_events)
def __repr__(self):
return "<%s (name=%s, rpi_apscheduler=%s, rpi_events=dict(), rpi_config=%s, dbuff_rpififo=%s)>" % (self.__class__.__name__, self.name, self._sched, self._config, self._imageFIFO)
def __str__(self):
msg = super().__str__()
return "%s::: dbinfo: %s, config: %s\nimageUpldFIFO: %s\n%s" % \
(self.name, self.dbinfo, self._config, self.imageUpldFIFO, msg)
def __del__(self):
### Clean base class
super().__del__()
#
# Main interface methods
#
def jobRun(self):
try:
# Lock the buffer
self._imageFIFO.acquireSemaphore()
# Get the current images in the FIFO
# Refresh the last remote image when available
if len(self._imageFIFO):
# Update remote cam image with the current (last) image
if not (self._imageFIFO[-1] == self.crt_image_snap):
self._putImage(self._imageFIFO[-1], self._config['image_snap'], True)
self.crt_image_snap = self._imageFIFO[-1]
self.numImgUpdDb += 1
logging.info("Updated remote %s with %s" % (self._config['image_snap'], self._imageFIFO[-1]) )
# Lock the upload buffer
self.imageUpldFIFO.acquireSemaphore()
# Check if a new upload sub-folder has to be used
if not (self.imageUpldFIFO.crtSubDir == self._imageFIFO.crtSubDir):
self.imageUpldFIFO.crtSubDir = self._imageFIFO.crtSubDir
self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
self._mkdirImage(self.upldir)
# Upload only images in the FIFO which have not been uploaded yet
for img in self._imageFIFO:
if not img in self.imageUpldFIFO:
self._putImage(img, os.path.join(self.upldir, os.path.basename(img)))
logging.info("Uploaded %s" % img )
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
# Update status
self.statusUpdate = (self.name, self.numImgUpdDb)
else:
# Update status
self.statusUpdate = (self.name, ERRNONE)
logging.info('Nothing to upload')
# Handle exceptions, mostly HTTP/SSL related!
except exceptions.Timeout as e:
# Catching this error will catch both ReadTimeout and ConnectTimeout.
raise rpiBaseClassError("%s::: jobRun(): Connect/ReadTimeoutError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.ConnectionError as e:
# A Connection error occurred.
raise rpiBaseClassError("%s::: jobRun(): ConnectionError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.HTTPError as e:
# An HTTP error occurred.
raise rpiBaseClassError("%s::: jobRun(): HTTPError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.RequestException as e:
# There was an ambiguous exception that occurred while handling your request.
raise rpiBaseClassError("%s::: jobRun(): RequestException:\n%s" % (self.name, str(e)), ERRLEV2)
# except BadStatusLine as e:
# self.eventErr_set('run()')
# logging.debug("BadStatusLine:\n%s" % str(e))
# pass
except rpiBaseClassError as e:
if e.errval == ERRCRIT:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): %s" % (self.name, e.errmsg), e.errval)
except RuntimeError as e:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): RuntimeError:\n%s" % (self.name, str(e)), ERRCRIT)
except:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): Unhandled Exception:\n%s" % (self.name, str(sys.exc_info())), ERRCRIT)
finally:
# Release the buffer
self._imageFIFO.releaseSemaphore()
def initClass(self):
""""
(re)Initialize the class.
"""
#self.imageDbHash = None
self._imageDbCursor = None
self.imageDbList = []
self.numImgUpdDb = 0
self.crt_image_snap = None
self.imgid = self._imageFIFO.camID + '.jpg'
self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
self.logfile = './upldlog.json'
### When there are already images listed in the upload log file, then
# make sure we don't upload them to the remote folder again
# Else, create the file with an empty list; to be updated in endDayOAM()
try:
self.imageUpldFIFO.acquireSemaphore()
self.imageUpldFIFO.clear()
if os.path.isfile(self.logfile):
with open(self.logfile,'r') as logf:
upldimg = json.load(logf)
for img in upldimg:
self.imageUpldFIFO.append(img)
del upldimg
logging.info("%s::: Local log file %s found and loaded." % (self.name, self.logfile))
else:
with open(self.logfile,'w') as logf:
json.dump([], logf)
logging.info("%s::: Local log file %s initialized." % (self.name, self.logfile))
except IOError:
raise rpiBaseClassError("%s::: initClass(): Local log file %s was not found or could not be created." % (self.name, self.logfile), ERRCRIT)
finally:
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
### Init Dropbox API client
self._token_file = self._config['token_file']
self._dbx = None
self.dbinfo = None
try:
with open(self._token_file, 'r') as token:
self._dbx = Dropbox(token.read())
info = self._dbx.users_get_current_account()
# info._all_field_names_ =
# {'account_id', 'is_paired', 'locale', 'email', 'name', 'team', 'country', 'account_type', 'referral_link'}
self.dbinfo ={'email': info.email, 'referral_link': info.referral_link}
logging.info("%s::: Loaded access token from ''%s''" % (self.name, self._token_file) )
### Create remote root folder (relative to app root) if it does not exist yet
self._mkdirImage(os.path.normpath(self._config['image_dir']))
except rpiBaseClassError as e:
if e.errval == ERRCRIT:
self.endDayOAM()
raise rpiBaseClassError("initClass(): %s" % e.errmsg, e.errval)
except IOError:
self.endDayOAM()
raise rpiBaseClassError("initClass(): Token file ''%s'' could not be read." % (self.name, self._token_file), ERRCRIT)
except AuthError as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): AuthError:\n%s" % e.error, ERRCRIT)
except DropboxException as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): DropboxException:\n%s" % str(e), ERRCRIT)
except InternalServerError as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): InternalServerError:\n%s" % str(e.status_code), ERRCRIT)
def endDayOAM(self):
"""
End-of-Day Operation and Maintenance sequence.
"""
self._lsImage(self.upldir)
logging.info("%s::: %d images in the remote folder %s" % (self.name, len(self.imageDbList), self.upldir))
# Lock the uplaod buffer
self.imageUpldFIFO.acquireSemaphore()
try:
upldimg=[]
for img in self.imageUpldFIFO:
upldimg.append(img)
with open(self.logfile,'w') as logf:
json.dump(upldimg, logf)
del upldimg
logging.info("%s::: Local log file %s updated." % (self.name, self.logfile))
except IOError:
raise rpiBaseClassError("endDayOAM(): Local log file %s was not found." % self.logfile, ERRCRIT)
finally:
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
# def endOAM(self):
# """
# End OAM procedure.
# """
@atexit.register
def atexitend():
self.endDayOAM()
def _lsImage(self,from_path):
"""
List the image/video files in the remote directory.
Stores the found file names in self.imageDbList.
"""
try:
if self._imageDbCursor is None:
self.ls_ref = self._dbx.files_list_folder('/' + os.path.normpath(from_path), recursive=False, include_media_info=True )
else:
new_ls = self._dbx.files_list_folder_continue(self._imageDbCursor)
if new_ls.entries == []:
logging.debug("%s::: _lsImage():: No changes on the server." % self.name)
else:
self.ls_ref = new_ls
# Select only images and only the ones for the current imgid (camid)
foundImg = False
for f in self.ls_ref.entries:
if 'media_info' in f._all_field_names_ and \
f.media_info is not None:
if self.imgid in f.path_lower:
img = '.%s' % f.path_lower
foundImg = True
if not img in self.imageDbList:
self.imageDbList.append(img)
if not foundImg:
self.imageDbList = []
### Store the hash of the folder
self._imageDbCursor = self.ls_ref.cursor
if len(self.imageDbList) > 0:
logging.debug("%s::: _lsImage():: imageDbList[0..%d]: %s .. %s" % (self.name, len(self.imageDbList)-1, self.imageDbList[0], self.imageDbList[-1]) )
else:
logging.debug("%s::: _lsImage():: imageDbList[]: empty" % self.name)
except ApiError as e:
raise rpiBaseClassError("_lsImage(): %s" % e.error, ERRLEV2)
def _putImage(self, from_path, to_path, overwrite=False):
"""
Copy local file to remote file.
Stores the uploaded files names in self.imageUpldFIFO.
Examples:
_putImage('./path/test.jpg', '/path/dropbox-upload-test.jpg')
"""
try:
mode = (WriteMode.overwrite if overwrite else WriteMode.add)
with open(from_path, "rb") as from_file:
self._dbx.files_upload( from_file, '/' + os.path.normpath(to_path), mode)
if not overwrite:
self.imageUpldFIFO.append(from_path)
logging.debug("%s::: _putImage(): Uploaded file from %s to remote %s" % (self.name, from_path, to_path))
except IOError:
raise rpiBaseClassError("_putImage(): Local img file %s could not be opened." % from_path, ERRCRIT)
except ApiError as e:
raise rpiBaseClassError("_putImage(): %s" % e.error, ERRLEV2)
def _mkdirImage(self, path):
"""
Create a new remote directory.
Examples:
_mkdirImage('/dropbox_dir_test')
"""
try:
self._dbx.files_create_folder('/' + os.path.normpath(path))
logging.debug("%s::: Remote output folder /%s created." % (self.name, path))
except ApiError as e:
noerr = False
# dropbox.files.CreateFolderError
if e.error.is_path():
# dropbox.files.WriteError
we = e.error.get_path()
if we.is_conflict():
# dropbox.files.WriteConflictError
wce = we.get_conflict()
# union tag is 'folder'
if wce.is_folder():
logging.info("%s::: Remote output folder /%s already exist!" % (self.name, path))
noerr = True
if not noerr:
raise rpiBaseClassError("_mkdirImage(): Remote output folder /%s was not created! %s" % (path, e.error), ERRCRIT)
else:
pass
def _mvImage(self, from_path, to_path):
"""
Move/rename a remote file or directory.
Examples:
_mvImage('./path1/dropbox-move-test.jpg', '/path2/dropbox-move-test.jpg')
"""
try:
self._dbx.files_move( '/' + os.path.normpath(from_path), '/' + os.path.normpath(to_path) )
logging.debug("%s::: _mvImage(): Moved file from %s to %s" % (self.name, from_path, to_path))
except ApiError as e:
raise rpiBaseClassError("_mvImage(): Image %s could not be moved to %s! %s" % (from_path, to_path, e.error), ERRLEV2)
class DropboxFS(FS):
_meta = {
"case_insensitive": False,
"invalid_path_chars": "\0",
"network": True,
"read_only": False,
"thread_safe": True,
"unicode_paths": True,
"virtual": False,
}
def __init__(self, accessToken, session=None):
super(DropboxFS, self).__init__()
self._lock = threading.RLock()
self.dropbox = Dropbox(accessToken, session=session)
def fix_path(self, path):
if isinstance(path, bytes):
try:
path = path.decode("utf-8")
except AttributeError:
pass
if not path.startswith("/"):
path = "/" + path
if path == "." or path == "./":
path = "/"
path = self.validatepath(path)
return path
def __repr__(self):
return "<DropboxDriveFS>"
def _infoFromMetadata(self, metadata):
rawInfo = {
"basic": {
"name": metadata.name,
"is_dir": isinstance(metadata, FolderMetadata),
}
}
if isinstance(metadata, FileMetadata):
rawInfo.update(
{"details": {"size": metadata.size, "type": ResourceType.file}}
)
else:
rawInfo.update({"details": {"type": ResourceType.directory}})
return Info(rawInfo)
def getinfo(self, path, namespaces=None):
_path = self.fix_path(path)
if _path == "/":
info_dict = {
"basic": {"name": "", "is_dir": True},
"details": {"type": ResourceType.directory},
}
return Info(info_dict)
try:
metadata = self.dropbox.files_get_metadata(
_path, include_media_info=True
)
except ApiError as e:
raise errors.ResourceNotFound(path=path, exc=e)
return self._infoFromMetadata(metadata)
def setinfo(self, path, info):
if not self.exists(path):
raise errors.ResourceNotFound(path)
def listdir(self, path):
_path = self.fix_path(path)
if _path == "/":
_path = ""
if not self.exists(_path):
raise errors.ResourceNotFound(path)
meta = self.getinfo(_path)
if meta.is_file:
raise errors.DirectoryExpected(path)
result = self.dropbox.files_list_folder(_path, include_media_info=True)
allEntries = result.entries
while result.has_more:
result = self.dropbox.files_list_folder_continue(result.cursor)
allEntries += result.entries
return [x.name for x in allEntries]
def makedir(self, path, permissions=None, recreate=False):
path = self.fix_path(path)
if self.exists(path) and not recreate:
raise errors.DirectoryExists(path)
if path == "/":
return SubFS(self, path)
if self.exists(path):
meta = self.getinfo(path)
if meta.is_dir:
if recreate == False:
raise errors.DirectoryExists(path)
else:
return SubFS(self, path)
if meta.is_file:
raise errors.DirectoryExpected(path)
ppath = self.get_parent(path)
if not self.exists(ppath):
raise errors.ResourceNotFound(ppath)
try:
folderMetadata = self.dropbox.files_create_folder_v2(path)
except ApiError as e:
raise errors.DirectoryExpected(path=path)
return SubFS(self, path)
def openbin(self, path, mode="r", buffering=-1, **options):
path = self.fix_path(path)
_mode = Mode(mode)
mode = _mode
_mode.validate_bin()
_path = self.validatepath(path)
log.debug("openbin: %s, %s", path, mode)
with self._lock:
try:
info = self.getinfo(_path)
log.debug("Info: %s", info)
except errors.ResourceNotFound:
if not _mode.create:
raise errors.ResourceNotFound(path)
# Check the parent is an existing directory
if not self.getinfo(self.get_parent(_path)).is_dir:
raise errors.DirectoryExpected(path)
else:
if info.is_dir:
raise errors.FileExpected(path)
if _mode.exclusive:
raise errors.FileExists(path)
return DropboxFile(self.dropbox, path, mode)
def remove(self, path):
_path = self.fix_path(path)
try:
info = self.getinfo(path)
if info.is_dir:
raise errors.FileExpected(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
log.debug(e)
raise errors.FileExpected(path=path, exc=e)
def removedir(self, path):
_path = self.fix_path(path)
if _path == "/":
raise errors.RemoveRootError()
try:
info = self.getinfo(path)
if not info.is_dir:
raise errors.DirectoryExpected(path=path)
if len(self.listdir(path)) > 0:
raise errors.DirectoryNotEmpty(path=path)
self.dropbox.files_delete_v2(_path)
except ApiError as e:
if isinstance(e.error._value, LookupError):
raise errors.ResourceNotFound(path=path)
raise errors.FileExpected(path=path, exc=e)
def copy(self, src_path, dst_path, overwrite=False):
src_path = self.fix_path(src_path)
dst_path = self.fix_path(dst_path)
try:
src_meta = self.getinfo(src_path)
if src_meta.is_dir:
raise errors.FileExpected(src_path)
except ApiError as e:
raise errors.ResourceNotFound
dst_meta = None
try:
dst_meta = self.getinfo(dst_path)
except Exception as e:
pass
if dst_meta is not None:
if overwrite == True:
self.remove(dst_path)
else:
raise errors.DestinationExists(dst_path)
parent_path = self.get_parent(dst_path)
if not self.exists(parent_path):
raise errors.ResourceNotFound(dst_path)
self.dropbox.files_copy_v2(src_path, dst_path)
def get_parent(self, dst_path):
import os
parent_path = os.path.abspath(os.path.join(dst_path, ".."))
return parent_path
def exists(self, path):
path = self.fix_path(path)
try:
self.getinfo(path)
return True
except Exception as e:
return False
def move(self, src_path, dst_path, overwrite=False):
_src_path = self.fix_path(src_path)
_dst_path = self.fix_path(dst_path)
if not self.getinfo(_src_path).is_file:
raise errors.FileExpected(src_path)
if not overwrite and self.exists(_dst_path):
raise errors.DestinationExists(dst_path)
if "/" in dst_path and not self.exists(self.get_parent(_dst_path)):
raise errors.ResourceNotFound(src_path)
with self._lock:
try:
if overwrite:
try:
# remove file anyways
self.dropbox.files_delete_v2(_dst_path)
except Exception as e:
pass
self.dropbox.files_move_v2(_src_path, _dst_path)
except ApiError as e:
raise errors.ResourceNotFound(src_path, exc=e)
def apierror_map(self, error):
log.debug(error)
def geturl(self, path, purpose='download'):
url = self.dropbox.sharing_create_shared_link(path).url
url = url.replace('?dl=0', '?raw=1')
return url
class Dropbox(BlackboxStorage):
"""Storage handler that uploads backups to Dropbox."""
required_fields = ("access_token", )
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.upload_base = self.config.get("upload_directory") or "/"
self.client = DropboxClient(self.config["access_token"])
self.valid = self._validate_token()
def _validate_token(self):
"""Check if dropbox token is valid."""
try:
return self.client.check_user("test").result == "test"
except AuthError:
return False
def sync(self, file_path: Path) -> None:
"""Sync a file to Dropbox."""
# Check if Dropbox token is valid.
if self.valid is False:
error = "Dropbox token is invalid!"
self.success = False
self.output = error
log.error(error)
return None
# This is size what can be uploaded as one chunk.
# When file is bigger than that, this will be uploaded
# in multiple parts.
chunk_size = 4 * 1024 * 1024
temp_file, recompressed = self.compress(file_path)
upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}"
try:
with temp_file as f:
file_size = os.stat(f.name).st_size
log.debug(file_size)
if file_size <= chunk_size:
self.client.files_upload(f.read(), upload_path,
WriteMode.overwrite)
else:
session_start = self.client.files_upload_session_start(
f.read(chunk_size))
cursor = UploadSessionCursor(session_start.session_id,
offset=f.tell())
# Commit contains path in Dropbox and write mode about file
commit = CommitInfo(upload_path, WriteMode.overwrite)
while f.tell() < file_size:
if (file_size - f.tell()) <= chunk_size:
self.client.files_upload_session_finish(
f.read(chunk_size), cursor, commit)
else:
self.client.files_upload_session_append(
f.read(chunk_size), cursor.session_id,
cursor.offset)
cursor.offset = f.tell()
self.success = True
except (ApiError, HttpError) as e:
log.error(e)
self.success = False
self.output = str(e)
def rotate(self, database_id: str) -> None:
"""
Rotate the files in the Dropbox directory.
All files in base directory of backups will be deleted when they
are older than `retention_days`, and because of this,
it's better to have backups in isolated folder.
"""
# Check if Dropbox token is valid.
if self.valid is False:
log.error("Dropbox token is invalid - Can't delete old backups!")
return None
# Let's rotate only this type of database
db_type_regex = rf"{database_id}_blackbox_\d{{2}}_\d{{2}}_\d{{4}}.+"
# Receive first batch of files.
files_result = self.client.files_list_folder(
self.upload_base if self.upload_base != "/" else "")
entries = [
entry for entry in files_result.entries
if self._is_backup_file(entry, db_type_regex)
]
# If there is more files, receive all of them.
while files_result.has_more:
cursor = files_result.cursor
files_result = self.client.files_list_folder_continue(cursor)
entries += [
entry for entry in files_result.entries
if self._is_backup_file(entry, db_type_regex)
]
retention_days = 7
if Blackbox.retention_days:
retention_days = Blackbox.retention_days
# Find all old files and delete them.
for item in entries:
last_modified = item.server_modified
now = datetime.now(tz=last_modified.tzinfo)
delta = now - last_modified
if delta.days >= retention_days:
self.client.files_delete(item.path_lower)
@staticmethod
def _is_backup_file(entry, db_type_regex) -> bool:
"""Check if file is actually this kind of database backup."""
return isinstance(entry, FileMetadata) and re.match(
db_type_regex, entry.name)
class DropboxNoteProvider(RemoteNoteProvider):
__DAY_ONE_EXTENSION = ".doentry"
def __init__(self, accessToken, folder, proxyHost=None, proxyPort=None, proxyUser=None, proxyPassword=None):
proxies = _proxies(proxyHost, proxyPort, proxyUser, proxyPassword)
self.__token = accessToken
self.__basePath = folder
self.__notesPath = folder + "/entries"
self.__removedNotesPath = self.__notesPath + "/deleted"
self.__photosPath = folder + "/photos"
self.__client = Dropbox(self.__token, session=create_session(proxies=proxies))
self.__notesCache = {}
self.__dayOneFlavor = folder == SyncFolder.DayOne
@online
@expires
def sync(self):
_LOG.info("Listing all notes and photos")
folder = self.__client.files_list_folder(self.__basePath, recursive=True)
files = list(filter(lambda e: e is not None, map(FileEntry.fromMetadata, folder.entries)))
while folder.has_more:
folder = self.__client.files_list_folder_continue(folder.cursor)
files.extend(filter(lambda e: e is not None, map(FileEntry.fromMetadata, folder.entries)))
notes = {}
for file in filter(lambda f: f.folder == self.__notesPath and isUuid(self.__normalizeNoteName(f.name)), files):
uuid = self.__normalizeNoteName(file.name)
notes[uuid] = NoteStatus(uuid, file.lastModified)
for file in filter(lambda f: f.folder == self.__photosPath and f.name.endswith(".jpg"), files):
uuid = file.name[:-4]
if uuid in notes:
notes[uuid].hasPhoto = True
for file in filter(lambda f: f.folder == self.__removedNotesPath and isUuid(self.__normalizeNoteName(f.name)),
files):
uuid = self.__normalizeNoteName(file.name)
if uuid in notes:
if file.lastModified >= notes[uuid].lastModified:
_LOG.warning("Sync integrity check deleting note: %s", uuid)
try:
self.__client.files_delete(self.__notePath(uuid))
except ApiError:
_LOG.warning("Note %s not found", uuid)
if notes[uuid].hasPhoto:
_LOG.warning("Sync integrity check deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
del notes[uuid]
else:
_LOG.warning("Sync integrity check deleting REMOVED note: %s", uuid)
try:
self.__client.files_delete(self.__removedNotePath(uuid))
except ApiError:
_LOG.warning("REMOVED note %s not found", uuid)
continue
notes[uuid] = NoteStatus(uuid, file.lastModified, True)
self.__notesCache = notes
return notes
@online
@expires
def get(self, uuid):
_LOG.info("Getting note: %s", uuid)
metadata, response = self.__client.files_download(self.__notePath(uuid))
with response:
note = unmarshalNote(response.content, metadata.client_modified)
if uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
_LOG.info("Getting photo: %s", uuid)
try:
with self.__client.files_download(self.__photoPath(uuid))[1] as response:
note.photo = response.content
except ApiError as e:
if e.error.is_path() and e.error.get_path().is_not_found():
_LOG.warning("Photo %s does not exist", uuid)
else:
raise e
return renderHtml(note)
@online
@expires
def add(self, note):
uuid = note.uuid
_LOG.info("Adding note: %s", uuid)
self.__uploadFile(self.__notePath(uuid), note.lastModified, marshalNote(note), overwrite=False)
if note.photo:
_LOG.info("Adding photo: %s", uuid)
self.__uploadFile(self.__photoPath(uuid), note.lastModified, note.photo)
elif uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
# Clean removed note if exists
if uuid in self.__notesCache and self.__notesCache[uuid].removed:
_LOG.info("Deleting REMOVED note: %s", uuid)
try:
self.__client.files_delete(self.__removedNotePath(uuid))
except ApiError:
_LOG.warning("REMOVED note %s not found", uuid)
@online
@expires
def update(self, note):
uuid = note.uuid
# Check if note exists
if self.__notesCache and (uuid not in self.__notesCache or self.__notesCache[uuid].removed):
raise RuntimeError("Note[uuid=%s] does not exist" % uuid)
_LOG.info("Updating note: %s", uuid)
self.__uploadFile(self.__notePath(uuid), note.lastModified, marshalNote(note))
if note.photo:
_LOG.info("Updating photo: %s", uuid)
self.__uploadFile(self.__photoPath(uuid), note.lastModified, note.photo)
elif uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
@online
@expires
def remove(self, note):
uuid = note.uuid
# Remove note if exists
if uuid in self.__notesCache and not self.__notesCache[uuid].removed:
_LOG.info("Deleting note: %s", uuid)
try:
self.__client.files_delete(self.__notePath(uuid))
except ApiError:
_LOG.warning("Note %s not found", uuid)
# Remove photo if exists
if uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
_LOG.info("Deleting photo: %s", uuid)
try:
self.__client.files_delete(self.__photoPath(uuid))
except ApiError:
_LOG.warning("Photo %s not found", uuid)
_LOG.info("Adding REMOVED note: %s", uuid)
self.__uploadFile(self.__removedNotePath(uuid), note.lastModified, b"")
def __uploadFile(self, path, lastModified, content, overwrite=True):
mode = WriteMode.overwrite if overwrite else WriteMode.add
self.__client.files_upload(content, path, mode=mode, client_modified=lastModified)
def __normalizeNoteName(self, name):
if self.__dayOneFlavor and name.endswith(self.__DAY_ONE_EXTENSION):
name = name[:-(len(self.__DAY_ONE_EXTENSION))]
return name
def __buildNotePath(self, parentPath, uuid):
path = parentPath + "/" + uuid
if self.__dayOneFlavor:
path += self.__DAY_ONE_EXTENSION
return path
def __notePath(self, uuid):
return self.__buildNotePath(self.__notesPath, uuid)
def __removedNotePath(self, uuid):
return self.__buildNotePath(self.__removedNotesPath, uuid)
def __photoPath(self, uuid):
return self.__photosPath + "/" + uuid + ".jpg"
class DropboxFS(FS):
def __init__(self, accessToken):
super().__init__()
self.dropbox = Dropbox(accessToken)
_meta = self._meta = {
"case_insensitive": False, # I think?
"invalid_path_chars": ":", # not sure what else
"max_path_length": None, # don't know what the limit is
"max_sys_path_length": None, # there's no syspath
"network": True,
"read_only": False,
"supports_rename": False # since we don't have a syspath...
}
def __repr__(self):
return "<DropboxDriveFS>"
def _infoFromMetadata(self, metadata): # pylint: disable=no-self-use
rawInfo = {
"basic": {
"name": metadata.name,
"is_dir": isinstance(metadata, FolderMetadata),
}
}
if isinstance(metadata, FileMetadata):
rawInfo.update({
"details": {
"accessed": None, # not supported by Dropbox API
"created": None, # not supported by Dropbox API?,
"metadata_changed": None, # not supported by Dropbox
"modified": datetime_to_epoch(
metadata.server_modified
), # API documentation says that this is reliable
"size": metadata.size,
"type": 0
},
"dropbox": {
"content_hash": metadata.
content_hash, # see https://www.dropbox.com/developers/reference/content-hash
"rev": metadata.rev,
"client_modified": metadata.
client_modified # unverified value coming from dropbox clients
}
})
if metadata.media_info is not None and metadata.media_info.is_metadata(
) is True:
media_info_metadata = metadata.media_info.get_metadata()
if media_info_metadata.time_taken is not None:
rawInfo.update({
"media_info": {
"taken_date_time":
datetime_to_epoch(media_info_metadata.time_taken)
}
})
if media_info_metadata.location is not None:
rawInfo.update({
"media_info": {
"location_latitude":
media_info_metadata.location.latitude,
"location_longitude":
media_info_metadata.location.longitude
}
})
# Dropbox doesn't parse some jpgs properly
if media_info_metadata.dimensions is not None:
rawInfo.update({
"media_info": {
"dimensions_height":
media_info_metadata.dimensions.height,
"dimensions_width":
media_info_metadata.dimensions.width
}
})
elif isinstance(metadata, FolderMetadata):
rawInfo.update({
"details": {
"accessed": None, # not supported by Dropbox API
"created": None, # not supported by Dropbox API,
"metadata_changed": None, # not supported by Dropbox
"modified": None, # not supported for folders
"size": None, # not supported for folders
"type": 1
}
})
else:
assert False, f"{metadata.name}, {metadata}, {type(metadata)}"
return Info(rawInfo)
def getinfo(self, path, namespaces=None):
if path == "/":
return Info({"basic": {"name": "", "is_dir": True}})
try:
if not path.startswith("/"):
path = "/" + path
metadata = self.dropbox.files_get_metadata(path,
include_media_info=True)
except ApiError as e:
raise ResourceNotFound(path=path, exc=e)
return self._infoFromMetadata(metadata)
def setinfo(self, path, info): # pylint: disable=too-many-branches
# dropbox doesn't support changing any of the metadata values
pass
def listdir(self, path):
return [x.name for x in self.scandir(path)]
def makedir(self, path, permissions=None, recreate=False):
try:
folderMetadata = self.dropbox.files_create_folder(path)
except ApiError as e:
assert isinstance(e.reason, CreateFolderError)
# TODO - there are other possibilities
raise DirectoryExpected(path=path)
# don't need to close this filesystem so we return the non-closing version
return SubFS(self, path)
def openbin(self, path, mode="r", buffering=-1, **options):
mode = Mode(mode)
exists = True
isDir = False
try:
isDir = self.getinfo(path).is_dir
except ResourceNotFound:
exists = False
if mode.exclusive and exists:
raise FileExists(path)
elif mode.reading and not mode.create and not exists:
raise ResourceNotFound(path)
elif isDir:
raise FileExpected(path)
return DropboxFile(self.dropbox, path, mode)
def remove(self, path):
try:
self.dropbox.files_delete(path)
except ApiError as e:
raise FileExpected(path=path, exc=e)
def removedir(self, path):
try:
self.dropbox.files_delete(path)
except ApiError as e:
assert e.reason is DeleteError
raise DirectoryExpected(path=path, exc=e)
# non-essential method - for speeding up walk
def scandir(self, path, namespaces=None, page=None):
#
if path == "/":
path = ""
# get all the avaliable metadata since it's cheap
# TODO - this call has a recursive flag so we can either use that and cache OR override walk
result = self.dropbox.files_list_folder(path, include_media_info=True)
allEntries = result.entries
while result.has_more:
result = self.dropbox.files_list_folder_continue(result.cursor)
allEntries += result.entries
return [self._infoFromMetadata(x) for x in allEntries]
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def user_authenticated(self):
try:
account = self.api_client.users_get_current_account()
log.Debug("User authenticated as ,%s" % account)
return True
except:
log.Debug('User not authenticated')
return False
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException(
'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException(
'dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError(
'dpbx error: cannot interact, but need human attention',
log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
authresult = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' %
e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(authresult.access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info(
"dpbx: Authentication failed. Trying to obtain new access token"
)
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception.
# So this line should not be reached
raise BackendException(
"dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" %
self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
progress.report_transfer(0, file_size)
if file_size < DPBX_UPLOAD_CHUNK_SIZE:
# Upload whole file at once to avoid extra server request
res_metadata = self.put_file_small(source_path, remote_path)
else:
res_metadata = self.put_file_chunked(source_path, remote_path)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException(
'dpbx: result path mismatch: %s (expected: %s)' %
(res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException(
'dpbx: result size mismatch: %s (expected: %s)' %
(res_metadata.size, file_size))
def put_file_small(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
log.Debug('dpbx,files_upload(%s, [%d bytes])' %
(remote_path, file_size))
res_metadata = self.api_client.files_upload(
f.read(),
remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
log.Debug('dpbx,files_upload(): %s' % res_metadata)
progress.report_transfer(file_size, file_size)
return res_metadata
finally:
f.close()
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug(
'dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id,
f.tell())
commit_info = CommitInfo(remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug(
'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(
buf, upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug(
'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(
buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' %
(upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError
) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset(
).correct_offset
log.Debug(
'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
% (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException(
"dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' %
(remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata
# again. Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" %
(local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
if not self.user_authenticated():
self.login()
remote_dir = '/' + urllib.unquote(
self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
res = []
try:
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
except ApiError as e:
if (isinstance(e.error, ListFolderError) and e.error.is_path()
and e.error.get_path().is_not_found()):
log.Debug('dpbx.list(%s): ignore missing folder (%s)' %
(remote_dir, e))
else:
raise
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [
x for x in file_list
if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn(
'Warning! It looks like there are automatically renamed files on backend'
)
log.Warn(
'They were probably created when using older version of duplicity.'
)
log.Warn('')
log.Warn(
'Please check your backup consistency. Most likely you will need to choose'
)
log.Warn(
'largest file from duplicity-* (number).gpg and remove brackets from its name.'
)
log.Warn('')
log.Warn(
'These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
#!python3
import sys
from dropbox import Dropbox
def print_result(sufix, date):
print("dropbox_duplicati_latest_file_%s %d" % (sufix, date))
if len(sys.argv)!=4:
print("Usage: %s DROPBOX_API_TOKEN PROMETHEUS_ALIAS PATH_ON_DROPBOX" % (sys.argv[0]))
sys.exit(1)
token = sys.argv[1]
alias = sys.argv[2]
path = sys.argv[3]
dbx = Dropbox(token)
try:
files = dbx.files_list_folder(path=path)
while files.has_more:
files = dbx.files_list_folder_continue(cursor=files.cursor)
last_file=files.entries[-1]
except Exception as e:
print_result(alias, 0)
exit(1)
print_result(alias, last_file.server_modified.timestamp())
