def do_upload_image():
logging.debug("Uploading image to Dropbox, file: %r", upload_file)
try:
dropbox = Dropbox(self._token)
dropbox.files_upload(img, upload_file)
except (DropboxException, RequestException):
logging.exception("Dropbox failure")
class DropboxHelper(object):
def __init__(self, access_token):
self.dropbox = Dropbox(oauth2_access_token=access_token)
def upload(self, filename, file_path):
with open(file_path, 'rb') as f:
try:
self.dropbox.files_upload(f.read(), '/' + filename)
except Exception:
os.remove(file_path)
raise CommandError(
'Unable to upload file to Dropbox. Maybe access token is invalid.'
)
def delete_all_files(self):
for i in self.dropbox.files_list_folder('').entries:
self.dropbox.files_delete(i.path_lower)
def download_last_backup(self, dir_path):
entries = self.dropbox.files_list_folder('').entries
if len(entries) == 0:
raise CommandError('We could not find any backup.')
entry = entries[-1]
full_path = dir_path + entry.path_lower
self.dropbox.files_download_to_file(full_path, entry.path_lower)
return full_path, entry.content_hash
def upload_to_dropbox(
ctx,
token,
folder_name):
filename = ctx.obj['filename']
filepath = ctx.obj['path-to-picture']
dropbox = Dropbox(token)
targetfile = ('/' + folder_name + '/' + filename)
image = Image.open(filepath)
imageIO = io.BytesIO()
image.save(imageIO, format='JPEG')
try:
dropbox.files_upload(imageIO.getvalue(), targetfile, mode=WriteMode('overwrite'))
except ApiError as err:
# This checks for the specific error where a user doesn't have enough Dropbox space quota to upload this file
if (err.error.is_path() and
err.error.get_path().error.is_insufficient_space()):
sys.exit("ERROR: Cannot back up; insufficient space.")
elif err.user_message_text:
print(err.user_message_text)
sys.exit()
else:
print(err)
sys.exit()
# create a shared link
link = dropbox.sharing_create_shared_link(targetfile)
url = link.url
# link which directly downloads by replacing ?dl=0 with ?dl=1
dl_url = re.sub(r"\?dl\=0", "?dl=1", url)
print('done uploading file ' + filename + ' to ' + dl_url)
def main():
(user, pasw) = foo('dropbox')
lroot = "/home/whatwelo/local/usr/ssl/LetsEncrypt" # local
rroot = "/Chris/ssl/LetsEncrypt" # remote (dropbox)
files = (
"certs/whatwelove.org.crt",
"certs/whatwelove.org.key",
"ca/signing-ca-chain1.pem",
"ca/signing-ca-chain2.pem",
"ca/signing-ca-chain.pem" # copy as dropbox does not support symlinks
)
dbx = Dropbox(pasw)
print("Attempting to upload...")
for filen in files:
try:
base = basename(filen)
dirn = dirname(filen)
src = path_join(lroot, dirn, base)
dst = path_join(rroot, dirn, base)
print('Uploading %s to dropbox:%s' % (src, dst))
with open(src, 'rb') as f:
data = f.read()
with stopwatch('upload %d bytes' % len(data)):
dbx.files_upload(data,
dst,
mode=WriteMode('overwrite', None),
autorename=False,
mute=True)
except Exception as err:
print(("Failed to upload %s\n%s" % (filen, err)))
print("Finished upload.")
class DropboxHelper(object):
def __init__(self, access_token):
self.dropbox = Dropbox(oauth2_access_token=access_token)
def upload(self, filename, file_path):
with open(file_path, 'rb') as f:
try:
self.dropbox.files_upload(f.read(), '/' + filename)
except Exception:
os.remove(file_path)
raise CommandError('Unable to upload file to Dropbox. Maybe access token is invalid.')
def delete_all_files(self):
for i in self.dropbox.files_list_folder('').entries:
self.dropbox.files_delete(i.path_lower)
def download_last_backup(self, dir_path):
entries = self.dropbox.files_list_folder('').entries
if len(entries) == 0:
raise CommandError('We could not find any backup.')
entry = entries[-1]
full_path = dir_path + entry.path_lower
self.dropbox.files_download_to_file(full_path, entry.path_lower)
return full_path, entry.content_hash
def upload_db():
dbx = Dropbox(token)
file_name = getcwd().rstrip('utilities') + '/costrajectory.db'
f = open(file_name, 'rb')
dbx.files_upload(f.read(),
'/costrajectory.db',
mode=WriteMode('overwrite'))
print('Remote database updated successfully.')
def backup(): # no test coverage
log.info("Backing up database")
cmd = f"pg_dump --no-owner --dbname={config.db_uri}"
result = subprocess.check_output(cmd, shell=True)
dbx = Dropbox(config.dropbox_token)
date = pendulum.now().date()
filename = f"{config.db_name}_{date.year}_{date.month}_{date.day}.sql"
path = config.dbx_db_backup_folder / filename
dbx.files_upload(f=result, path=path.as_posix(), autorename=True)
def dropBox (self, FILE, token, account_id):
try:
dbx = Dropbox(token)
head, tails = os.path.split(FILE)
with open(FILE, 'r') as f_in:
mode = WriteMode('overwrite', None) #ADD SOME EXCEPTIONS HERE TO CATCH IF IT DOESNT UPLAD
dbx.files_upload(f_in, '/'+tails, mode=mode)
except Exception as e:
return str(e)
class DropBoxDataProvider(DataProviderBase):
smoke_url = DROPBOX_SMOKE_URL
def __init__(self, acs_token):
self.dbx = Dropbox(acs_token)
def api_smoke(self) -> int:
return len(self.dbx.files_list_folder('').entries)
def get_list_of_objects(self, dbx_folder='') -> list:
result = namedtuple('Result', ['filename', 'filepatch'])
return [
result(el.name, el.path_lower)
for el in self.dbx.files_list_folder(dbx_folder).entries
]
def file_delete(self, dbx_file) -> str:
return self.dbx.files_delete_v2(dbx_file).metadata.path_lower
def file_download(self, local_file, dbx_file) -> str:
return self.dbx.files_download_to_file(local_file, dbx_file).path_lower
def file_upload(self, local_file, dbx_file) -> str:
if isinstance(local_file, str):
if local_file.startswith("https://"):
waiting_time = 0.1
waiting_attempt = 100
url_result = self.dbx.files_save_url(dbx_file, local_file)
job_id = url_result.get_async_job_id()
while waiting_attempt > 0:
st = self.dbx.files_save_url_check_job_status(job_id)
if st.is_complete():
return st.get_complete().path_lower
sleep(waiting_time)
waiting_attempt -= 1
else:
with open(local_file, 'rb') as f:
return self.dbx.files_upload(
f.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
else:
return self.dbx.files_upload(local_file.read(),
dbx_file,
autorename=True,
strict_conflict=True).path_lower
def file_move(self, dbx_file_from, dbx_file_to) -> str:
return self.dbx.files_move_v2(dbx_file_from,
dbx_file_to).metadata.path_lower
def create_folder(self, dbx_folder) -> str:
return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower
def get_file_tmp_link(self, dbx_path) -> str:
return self.dbx.files_get_temporary_link(dbx_path).link
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.error['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.error, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = DUMMY_PAYLOAD
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(DUMMY_PAYLOAD, resp.content)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
def test_bad_upload_types(self):
with self.assertRaises(TypeError):
self.dbx.files_upload(BytesIO(b'test'), '/Test')
@require_team_token
def test_team(self, token):
dbxt = DropboxTeam(token)
dbxt.team_groups_list()
r = dbxt.team_members_list()
if r.members:
# Only test assuming a member if there is a member
dbxt.as_user(
r.members[0].profile.team_member_id).files_list_folder('')
def _upload_small_file(dbx: dropbox.Dropbox, path: Path,
chunk_size: int) -> Iterator[Tuple[int, int]]:
yield 0, 1
dbx.files_upload(
f=path.read_bytes(),
path=_upload_path(path),
mode=dropbox.files.WriteMode("overwrite"),
)
logger.debug(f"Uploaded {path}")
yield 1, 1
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.error['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.error, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = string.ascii_letters
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(string.ascii_letters, resp.text)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
@require_team_token
def test_team(self, token):
dbxt = DropboxTeam(token)
dbxt.team_groups_list()
r = dbxt.team_members_list()
if r.members:
# Only test assuming a member if there is a member
dbxt.as_user(r.members[0].profile.team_member_id).files_list_folder('')
def upload_to_dropbox(video_file_path):
"""
Dropbox is an unoffical feature - This code worked at one point,
but official support was removed for several reasons.
"""
try:
file_size = os.path.getsize(video_file_path)
if file_size <= FILE_CHUNK_SIZE:
dbx = Dropbox(DROPBOX_API_KEY)
f = open(video_file_path, 'rb')
dbx.files_upload(f, video_file_path)
else:
upload_dropbox_file_chucks(video_file_path, file_size)
except Exception as e:
print('Unhandled exception while uploading files - {}'.format(e))
def upload_chunked(dropbox_client: dropbox.Dropbox, local_file_path: str,
remote_file_path: str) -> None:
""" Uploads a file in chucks to Dropbox, allowing it to resume on (connection) failure. """
logger.info('Dropbox: Syncing file %s', remote_file_path)
write_mode = dropbox.files.WriteMode.overwrite
file_handle = open(local_file_path, 'rb')
file_size = os.path.getsize(local_file_path)
# Many thanks to https://stackoverflow.com/documentation/dropbox-api/409/uploading-a-file/1927/uploading-a-file-usin
# g-the-dropbox-python-sdk#t=201610181733061624381
CHUNK_SIZE = 2 * 1024 * 1024
# Small uploads should be transfers at one go.
if file_size <= CHUNK_SIZE:
dropbox_client.files_upload(file_handle.read(),
remote_file_path,
mode=write_mode)
# Large uploads can be sent in chunks, by creating a session allowing multiple separate uploads.
else:
upload_session_start_result = dropbox_client.files_upload_session_start(
file_handle.read(CHUNK_SIZE))
cursor = dropbox.files.UploadSessionCursor(
session_id=upload_session_start_result.session_id,
offset=file_handle.tell())
commit = dropbox.files.CommitInfo(path=remote_file_path,
mode=write_mode)
# We keep sending the data in chunks, until we reach the last one, then we instruct Dropbox to finish the upload
# by combining all the chunks sent previously.
while file_handle.tell() < file_size:
if (file_size - file_handle.tell()) <= CHUNK_SIZE:
dropbox_client.files_upload_session_finish(
file_handle.read(CHUNK_SIZE), cursor, commit)
else:
dropbox_client.files_upload_session_append_v2(
file_handle.read(CHUNK_SIZE), cursor)
cursor.offset = file_handle.tell()
file_handle.close()
class TestDropbox(unittest.TestCase):
def setUp(self):
self.dbx = Dropbox(oauth2_token)
def test_bad_auth(self):
# Test malformed token
malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
with self.assertRaises(BadInputError) as cm:
malformed_token_dbx.files_list_folder('')
self.assertIn('token is malformed', cm.exception.message)
# Test reasonable-looking invalid token
invalid_token_dbx = Dropbox(INVALID_TOKEN)
with self.assertRaises(AuthError) as cm:
invalid_token_dbx.files_list_folder('')
self.assertEqual(cm.exception.reason['error']['.tag'],
'invalid_access_token')
def test_rpc(self):
self.dbx.files_list_folder('')
# Test API error
random_folder_path = '/' + \
''.join(random.sample(string.ascii_letters, 15))
with self.assertRaises(ApiError) as cm:
self.dbx.files_list_folder(random_folder_path)
self.assertIsInstance(cm.exception.reason, ListFolderError)
def test_upload_download(self):
# Upload file
timestamp = str(datetime.datetime.utcnow())
random_filename = ''.join(random.sample(string.ascii_letters, 15))
random_path = '/Test/%s/%s' % (timestamp, random_filename)
test_contents = string.ascii_letters
self.dbx.files_upload(test_contents, random_path)
# Download file
metadata, resp = self.dbx.files_download(random_path)
self.assertEqual(string.ascii_letters, resp.text)
# Cleanup folder
self.dbx.files_delete('/Test/%s' % timestamp)
def upload_file(dbx: dropbox.Dropbox, local_report_file_path: Path,
remote_report_file_path: str):
try:
with local_report_file_path.open(mode='rb') as report_file:
upload_result = dbx.files_upload(
report_file.read(),
remote_report_file_path,
mode=dropbox.files.WriteMode.overwrite)
except ApiError as e:
sys.exit(f"Error: {e}")
def handle(self, *args, **options):
dropbox_destination = settings.PROJECT_NAME
client = Dropbox(settings.DROPBOX_TOKEN)
local_paths = []
for relative_path in settings.BACKUP_FILES:
file_or_directory = os.path.join(settings.BASE_DIR, relative_path)
if os.path.isdir(file_or_directory):
for root, dirs, files in os.walk(file_or_directory):
for filename in files:
local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, settings.BASE_DIR)
dropbox_path = os.path.join(dropbox_destination, relative_path)
local_paths.append((local_path, dropbox_path))
else:
dropbox_path = os.path.join(dropbox_destination, relative_path)
local_paths.append((file_or_directory, dropbox_path))
for local_path, dropbox_path in local_paths:
dropbox_path = '/%s' % dropbox_path
print u'Uploading %s to %s...' % (local_path, dropbox_path)
with open(local_path, 'rb') as f:
client.files_upload(f, dropbox_path, WriteMode.overwrite)
def process_user(account):
'''Call /files/list_folder for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', account)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
dbx = Dropbox(token)
has_more = True
while has_more:
if cursor is None:
result = dbx.files_list_folder(path='')
else:
result = dbx.files_list_folder_continue(cursor)
for entry in result.entries:
# Ignore deleted files, folders, and non-markdown files
if (isinstance(entry, DeletedMetadata)
or isinstance(entry, FolderMetadata)
or not entry.path_lower.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
_, resp = dbx.files_download(entry.path_lower)
html = markdown(resp.content.decode("utf-8"))
dbx.files_upload(bytes(html, encoding='utf-8'),
entry.path_lower[:-3] + '.html',
mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
def upload(dbx: dropbox.Dropbox, src: str, dst: str, overwrite: bool = False):
mode = (dropbox.files.WriteMode.overwrite
if overwrite
else dropbox.files.WriteMode.add)
mtime = os.path.getmtime(src)
with open(src, 'rb') as f:
data = f.read()
try:
res = dbx.files_upload(
data, dst, mode,
client_modified=datetime.datetime(*time.gmtime(mtime)[:6]),
mute=True)
except dropbox.exceptions.ApiError as err:
print('*** API error', err)
return None
print('upload as', res.name.encode('utf8'))
return res
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
location = setting('DROPBOX_ROOT_PATH', '/')
oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN')
timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE)
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout,
write_mode=write_mode):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure an auth token at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.root_path = root_path
self.write_mode = write_mode
self.client = Dropbox(oauth2_access_token, timeout=timeout)
def _full_path(self, name):
if name == '/':
name = ''
# If the machine is windows do not append the drive letter to file path
if os.name == 'nt':
final_path = os.path.join(self.root_path, name).replace('\\', '/')
# Separator on linux system
sep = '//'
base_path = self.root_path
if (not os.path.normcase(final_path).startswith(os.path.normcase(base_path + sep)) and
os.path.normcase(final_path) != os.path.normcase(base_path) and
os.path.dirname(os.path.normcase(base_path)) != os.path.normcase(base_path)):
raise SuspiciousFileOperation(
'The joined path ({}) is located outside of the base path '
'component ({})'.format(final_path, base_path))
return final_path
else:
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
if full_path == '/':
full_path = ''
metadata = self.client.files_list_folder(full_path)
for entry in metadata.entries:
if isinstance(entry, FolderMetadata):
directories.append(entry.name)
else:
files.append(entry.name)
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.size
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.server_modified
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.client_modified
def url(self, name):
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
content.open()
if content.size <= self.CHUNK_SIZE:
self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode))
else:
self._chunked_upload(content, self._full_path(name))
content.close()
return name
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE)
)
cursor = UploadSessionCursor(
session_id=upload_session.session_id,
offset=content.tell()
)
commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode))
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit
)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor
)
cursor.offset = content.tell()
def get_available_name(self, name, max_length=None):
"""Overwrite existing file with the same name."""
name = self._full_path(name)
if self.write_mode == 'overwrite':
return get_available_overwrite_name(name, max_length)
return super().get_available_name(name, max_length)
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox folder.
"""
def __init__(self, token=ACCESS_TOKEN, location=ROOT_FOLDER):
if not token:
raise ImproperlyConfigured("You must configure an access token at "
"'settings.DROPBOX_ACCESS_TOKEN'.")
self.client = Dropbox(token)
self.account_info = self.client.users_get_current_account()
self.location = location or DEFAULT_ROOT_FOLDER
self.base_url = 'https://dl.dropboxusercontent.com/'
def _get_abs_path(self, name):
return os.path.realpath(os.path.join(self.location, name))
def _open(self, name, mode='rb'):
name = self._get_abs_path(name)
remote_file = DropboxFile(name, self, mode=mode)
return remote_file
def _save(self, name, content):
name = self._get_abs_path(name)
directory = os.path.dirname(name)
if not self.exists(directory) and directory:
self.client.files_create_folder(directory)
# response = self.client.files_get_metadata(directory)
# if not response['is_dir']:
# raise IOError("%s exists and is not a directory." % directory)
abs_name = os.path.realpath(os.path.join(self.location, name))
self.client.files_upload(content.read(), abs_name)
return name
def delete(self, name):
name = self._get_abs_path(name)
try:
self.client.files_delete(name)
except ApiError as e:
if isinstance(e.error, DeleteError)\
and e.error.is_path_lookup()\
and e.error.get_path_lookup().is_not_found():
# not found
return False
# error
raise e
# deleted
return True
def exists(self, name):
name = self._get_abs_path(name)
try:
self.client.files_get_metadata(name)
except ApiError as e:
if hasattr(e.error, 'is_path')\
and e.error.is_path()\
and e.error.get_path().is_not_found():
# not found
return False
# error
raise e
# found
return True
def listdir(self, path):
path = self._get_abs_path(path)
response = self.client.files_list_folder(path)
directories = []
files = []
for entry in response.entries:
if isinstance(entry, FolderMetadata):
directories.append(os.path.basename(entry.path_display))
elif isinstance(entry, FileMetadata):
files.append(os.path.basename(entry.path_display))
return directories, files
def size(self, name):
name = self._get_abs_path(name)
return self.client.files_get_metadata(name).size
def url(self, name):
name = self._get_abs_path(name)
return self.client.files_get_temporary_link(name).link
def modified_time(self, name):
name = self._get_abs_path(name)
return self.client.files_get_metadata(name).server_modified
def accessed_time(self, name):
name = self._get_abs_path(name)
# Note to the unwary, this is actually an mtime
return self.client.files_get_metadata(name).client_modified
def get_available_name(self, name, max_length=None):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
name = self._get_abs_path(name)
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a number (before
# the file extension, if one exists) to the filename until the generated
# filename doesn't exist.
count = itertools.count(1)
while self.exists(name):
# file_ext includes the dot.
_fn = "%s_%s%s" % (file_root, count.next(), file_ext)
name = os.path.join(dir_name, _fn)
return name
class DropBoxStorage(Storage):
"""
The default Storage base for all file storing of the Chat app
"""
def __init__(self, oauth2_access_token: str = None, root_path: str = None):
"""
The access token and root path may be provided here as well,
if they are not provided here, program will check settings for environment variables
:param oauth2_access_token: The OAUTH2 access token for the DropBox api
:param root_path: The root path for storing the files in the DropBox storage, defaults '/'
"""
oauth2_access_token = oauth2_access_token or settings.DROPBOX_OAUTH2_TOKEN
self.root_path = root_path or settings.DROPBOX_ROOT_PATH or '/'
if oauth2_access_token is None:
raise ImproperlyConfigured(
"You must configure an OATH2 access token ENV named "
"'DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def delete(self, name: str):
"""
Deletes the specified file from the storage system.
"""
self.client.files_delete_v2(join(self.root_path, basename(name)))
def exists(self, name: str):
"""
Returns True if a file referenced by the given name already exists in the
storage system, or False if the name is available for a new file.
"""
try:
return bool(
self.client.files_get_metadata(
join(self.root_path, basename(name))))
except ApiError:
return False
def url(self, name: str):
"""
Returns an absolute URL where the file's contents can be accessed
directly by a Web browser.
"""
media = self.client.files_get_temporary_link(
join(self.root_path, basename(name)))
return media.link
def _open(self, name: str, mode: str = 'rb'):
"""
Call DropBoxStorage.open(...) instead
"""
file = DropBoxFile(join(self.root_path, basename(name)), self.client)
return file
def _save(self, name: str, content: File):
"""
Call DropBoxStorage.save(...) instead
"""
self.client.files_upload(content.read(),
join(self.root_path, basename(name)))
return name
#!/usr/bin/python
from ipaddress import ip_address
from os import environ
from subprocess import check_output
from dropbox import Dropbox
from dropbox.files import WriteMode
startIP=ip_address(u'80.73.48.0')
for ipCounter in range(0,4096):
ip = (startIP+ipCounter).__str__()
print(ip)
xmlMessage = check_output("nmap -oX - -F -sV "+ip,shell=True)
# read the dropbox access key from the environment variable DBX_KEY
dbxAccessKey = environ["DBX_KEY"]
# connect to dropbox
dbx=Dropbox(dbxAccessKey)
# upload scan file
dbx.files_upload(xmlMessage, "/NewScans/SCAN_"+ip+".xml", WriteMode.overwrite, True, None , True)
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN')
self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def _full_path(self, name):
if name == '/':
name = ''
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
metadata = self.client.files_get_metadata(full_path)
for entry in metadata['contents']:
entry['path'] = entry['path'].replace(full_path, '', 1)
entry['path'] = entry['path'].replace('/', '', 1)
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
content.open()
if content.size <= self.CHUNK_SIZE:
self.client.files_upload(content.read(), self._full_path(name))
else:
self._chunked_upload(content, self._full_path(name))
content.close()
return name
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE)
)
cursor = UploadSessionCursor(
session_id=upload_session.session_id,
offset=content.tell()
)
commit = CommitInfo(path=dest_path)
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit
)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor
)
cursor.offset = content.tell()
class dropboxClient(object):
''' a class of methods to manage file storage on Dropbox API '''
# https://www.dropbox.com/developers/documentation/http/documentation
_class_fields = {
'schema': {
'access_token': '',
'collection_name': 'labPack',
'record_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.ogg',
'record_key_path': '/home/user/.config/collective-acuity-labpack/user-data/obs/terminal',
'record_key_comp': 'obs',
'previous_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.yaml',
'secret_key': '6tZ0rUexOiBcOse2-dgDkbeY',
'prefix': 'obs/terminal',
'delimiter': '2016-03-17T17-24-51-687845Z.yaml',
'max_results': 1
},
'components': {
'.collection_name': {
'max_length': 255,
'must_not_contain': ['/', '^\\.']
},
'.record_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.record_key_path': {
'max_length': 32767
},
'.record_key_comp': {
'max_length': 255
},
'.secret_key': {
'must_not_contain': [ '[\\t\\n\\r]' ]
},
'.max_results': {
'min_value': 1,
'integer_data': True
},
'.previous_key': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
},
'.prefix': {
'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
}
},
'metadata': {
'record_optimal_bytes': 10000 * 1024,
'record_max_bytes': 150000 * 1024
}
}
def __init__(self, access_token, collection_name=''):
'''
a method to initialize the dropboxClient class
:param access_token: string with oauth2 access token for users account
'''
title = '%s.__init__' % self.__class__.__name__
# construct input validation model
self.fields = jsonModel(self._class_fields)
# validate inputs
input_fields = {
'access_token': access_token,
'collection_name': collection_name
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# workaround for module namespace conflict
from sys import path as sys_path
sys_path.append(sys_path.pop(0))
from dropbox import Dropbox
from dropbox.files import FileMetadata, WriteMode, DeleteArg
from dropbox.exceptions import ApiError
sys_path.insert(0, sys_path.pop())
# construct dropbox client
from labpack.compilers.objects import _method_constructor
self.dropbox = Dropbox(oauth2_access_token=access_token)
# construct dropbox objects
self.objects = _method_constructor({
'FileMetadata': FileMetadata,
'ApiError': ApiError,
'WriteMode': WriteMode,
'DeleteArg': DeleteArg
})
# construct collection name
self.collection_name = collection_name
def _import(self, record_key, record_data, overwrite=True, last_modified=0.0, **kwargs):
'''
a helper method for other storage clients to import into appdata
:param record_key: string with key for record
:param record_data: byte data for body of record
:param overwrite: [optional] boolean to overwrite existing records
:param last_modified: [optional] float to record last modified date
:param kwargs: [optional] keyword arguments from other import methods
:return: boolean indicating whether record was imported
'''
title = '%s._import' % self.__class__.__name__
# check overwrite
if not overwrite:
if self.exists(record_key):
return False
# check max size
import sys
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
# TODO: apply session upload for files greater than record_max
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', record_key):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
elif last_modified:
from labpack.records.time import labDT
mod_time = labDT.fromEpoch(last_modified)
upload_kwargs['client_modified'] = mod_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return True
def _walk(self, root_path=''):
''' an iterator method which walks the file structure of the dropbox collection '''
title = '%s._walk' % self.__class__.__name__
if root_path:
root_path = '/%s' % root_path
try:
response = self.dropbox.files_list_folder(path=root_path, recursive=True)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
if response.has_more:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for record in response.entries:
if not isinstance(record, self.objects.FileMetadata):
continue
yield record.path_display[1:]
except:
raise DropboxConnectionError(title)
def exists(self, record_key):
'''
a method to determine if a record exists in collection
:param record_key: string with key of record
:return: boolean reporting status
'''
title = '%s.exists' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# send get metadata request
file_path = '/%s' % record_key
try:
self.dropbox.files_get_metadata(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
return False
else:
raise DropboxConnectionError(title)
return True
def save(self, record_key, record_data, overwrite=True, secret_key=''):
'''
a method to create a record in the collection folder
:param record_key: string with name to assign to record (see NOTES below)
:param record_data: byte data for record body
:param overwrite: [optional] boolean to overwrite records with same name
:param secret_key: [optional] string with key to encrypt data
:return: string with name of record
NOTE: record_key may only contain alphanumeric, /, _, . or -
characters and may not begin with the . or / character.
NOTE: using one or more / characters splits the key into
separate segments. these segments will appear as a
sub directories inside the record collection and each
segment is used as a separate index for that record
when using the list method
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
'''
title = '%s.save' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate byte data
if not isinstance(record_data, bytes):
raise ValueError('%s(record_data=b"...") must be byte data.' % title)
# construct and validate file path
file_root, file_name = os.path.split(record_key)
self.fields.validate(file_name, '.record_key_comp')
while file_root:
file_root, path_node = os.path.split(file_root)
self.fields.validate(path_node, '.record_key_comp')
# check overwrite exception
if not overwrite:
if self.exists(record_key):
raise Exception('%s(record_key="%s") already exists. To overwrite, set overwrite=True' % (title, record_key))
# check size of file
import sys
record_optimal = self.fields.metadata['record_optimal_bytes']
record_max = self.fields.metadata['record_max_bytes']
record_size = sys.getsizeof(record_data)
error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
if record_size > record_max:
raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
elif record_size > record_optimal:
print('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))
# TODO add upload session for support of files over 150MB
# http://dropbox-sdk-python.readthedocs.io/en/latest/moduledoc.html#dropbox.dropbox.Dropbox.files_upload_session_start
# encrypt data
if secret_key:
from labpack.encryption import cryptolab
record_data, secret_key = cryptolab.encrypt(record_data, secret_key)
# construct upload kwargs
upload_kwargs = {
'f': record_data,
'path': '/%s' % record_key,
'mute': True,
'mode': self.objects.WriteMode.overwrite
}
# modify file time
import re
if re.search('\\.drep$', file_name):
from labpack.records.time import labDT
drep_time = labDT.fromEpoch(1)
upload_kwargs['client_modified'] = drep_time
# send upload request
try:
self.dropbox.files_upload(**upload_kwargs)
except:
raise DropboxConnectionError(title)
return record_key
def load(self, record_key, secret_key=''):
'''
a method to retrieve byte data of appdata record
:param record_key: string with name of record
:param secret_key: [optional] string used to decrypt data
:return: byte data for record body
'''
title = '%s.load' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key,
'secret_key': secret_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# construct file path
file_path = '/%s' % record_key
# request file data
try:
metadata, response = self.dropbox.files_download(file_path)
except Exception as err:
if str(err).find("LookupError('not_found'") > -1:
raise Exception('%s(record_key=%s) does not exist.' % (title, record_key))
else:
raise DropboxConnectionError(title)
record_data = response.content
# decrypt (if necessary)
if secret_key:
from labpack.encryption import cryptolab
record_data = cryptolab.decrypt(record_data, secret_key)
return record_data
def conditional_filter(self, path_filters):
''' a method to construct a conditional filter function for list method
:param path_filters: dictionary or list of dictionaries with query criteria
:return: filter_function object
path_filters:
[ { 0: { conditional operators }, 1: { conditional_operators }, ... } ]
conditional operators:
"byte_data": false,
"discrete_values": [ "" ],
"excluded_values": [ "" ],
"greater_than": "",
"less_than": "",
"max_length": 0,
"max_value": "",
"min_length": 0,
"min_value": "",
"must_contain": [ "" ],
"must_not_contain": [ "" ],
"contains_either": [ "" ]
'''
title = '%s.conditional_filter' % self.__class__.__name__
from labpack.compilers.filters import positional_filter
filter_function = positional_filter(path_filters, title)
return filter_function
def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
'''
a method to list keys in the dropbox collection
:param prefix: string with prefix value to filter results
:param delimiter: string with value which results must not contain (after prefix)
:param filter_function: (positional arguments) function used to filter results
:param max_results: integer with maximum number of results to return
:param previous_key: string with key in collection to begin search after
:return: list of key strings
NOTE: each key string can be divided into one or more segments
based upon the / characters which occur in the key string as
well as its file extension type. if the key string represents
a file path, then each directory in the path, the file name
and the file extension are all separate indexed values.
eg. lab/unittests/1473719695.2165067.json is indexed:
[ 'lab', 'unittests', '1473719695.2165067', '.json' ]
it is possible to filter the records in the collection according
to one or more of these path segments using a filter_function.
NOTE: the filter_function must be able to accept an array of positional
arguments and return a value that can evaluate to true or false.
while searching the records, list produces an array of strings
which represent the directory structure in relative path of each
key string. if a filter_function is provided, this list of strings
is fed to the filter function. if the function evaluates this input
and returns a true value the file will be included in the list
results.
'''
title = '%s.list' % self.__class__.__name__
# validate input
input_fields = {
'prefix': prefix,
'delimiter': delimiter,
'max_results': max_results,
'previous_key': previous_key
}
for key, value in input_fields.items():
if value:
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate filter function
if filter_function:
try:
path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
filter_function(*path_segments)
except:
err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
raise TypeError('%s must accept positional arguments.' % err_msg)
# construct empty results list
results_list = []
check_key = True
if previous_key:
check_key = False
# determine root path
root_path = ''
if prefix:
from os import path
root_path, file_name = path.split(prefix)
# iterate over dropbox files
for file_path in self._walk(root_path):
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
if record_key == previous_key:
check_key = True
# find starting point
if not check_key:
continue
# apply prefix filter
partial_key = record_key
if prefix:
if record_key.find(prefix) == 0:
partial_key = record_key[len(prefix):]
else:
continue
# apply delimiter filter
if delimiter:
if partial_key.find(delimiter) > -1:
continue
# apply filter function
if filter_function:
if filter_function(*path_segments):
results_list.append(record_key)
else:
results_list.append(record_key)
# return results list
if len(results_list) == max_results:
return results_list
return results_list
def delete(self, record_key):
''' a method to delete a file
:param record_key: string with name of file
:return: string reporting outcome
'''
title = '%s.delete' % self.__class__.__name__
# validate inputs
input_fields = {
'record_key': record_key
}
for key, value in input_fields.items():
object_title = '%s(%s=%s)' % (title, key, str(value))
self.fields.validate(value, '.%s' % key, object_title)
# validate existence of file
if not self.exists(record_key):
exit_msg = '%s does not exist.' % record_key
return exit_msg
# remove file
current_dir = os.path.split(record_key)[0]
try:
file_path = '/%s' % record_key
self.dropbox.files_delete(file_path)
except:
raise DropboxConnectionError(title)
# remove empty directories in path to file
try:
while current_dir:
folder_path = '/%s' % current_dir
response = self.dropbox.files_list_folder(folder_path)
if not response.entries:
self.dropbox.files_delete(folder_path)
current_dir = os.path.split(current_dir)[0]
else:
break
except:
raise DropboxConnectionError(title)
exit_msg = '%s has been deleted.' % record_key
return exit_msg
def remove(self):
'''
a method to remove all records in the collection
NOTE: this method removes all the files in the collection, but the
collection folder itself created by oauth2 cannot be removed.
only the user can remove the app folder
:return: string with confirmation of deletion
'''
title = '%s.remove' % self.__class__.__name__
# get contents in root
try:
response = self.dropbox.files_list_folder(path='')
except:
raise DropboxConnectionError(title)
# populate delete list
delete_list = []
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
# continue retrieval if folder is large
if response.has_more:
try:
while response.has_more:
response = self.dropbox.files_list_folder_continue(response.cursor)
for file in response.entries:
delete_list.append(self.objects.DeleteArg(path=file.path_display))
except:
raise DropboxConnectionError(title)
# send batch delete request
try:
self.dropbox.files_delete_batch(delete_list)
except:
raise DropboxConnectionError(title)
# return outcome
insert = 'collection'
if self.collection_name:
insert = self.collection_name
exit_msg = 'Contents of %s will been removed from Dropbox.' % insert
return exit_msg
def export(self, storage_client, overwrite=True):
'''
a method to export all the records in collection to another platform
:param storage_client: class object with storage client methods
:return: string with exit message
'''
title = '%s.export' % self.__class__.__name__
# validate storage client
method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ]
for method in method_list:
if not getattr(storage_client, method, None):
from labpack.parsing.grammar import join_words
raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list)))
# walk collection folder to find files
import os
count = 0
skipped = 0
for file_path in self._walk():
path_segments = file_path.split(os.sep)
record_key = os.path.join(*path_segments)
record_key = record_key.replace('\\','/')
file_path = '/%s' % file_path
# retrieve data and metadata
try:
metadata, response = self.dropbox.files_download(file_path)
except:
raise DropboxConnectionError(title)
record_data = response.content
client_modified = metadata.client_modified
# import record into storage client
last_modified = 0.0
if client_modified:
from dateutil.tz import tzutc
from labpack.records.time import labDT
last_modified = labDT.fromPython(client_modified.replace(tzinfo=tzutc())).epoch()
outcome = storage_client._import(record_key, record_data, overwrite=overwrite, last_modified=last_modified)
if outcome:
count += 1
else:
skipped += 1
# report outcome
plural = ''
skip_insert = ''
new_folder = storage_client.collection_name
if count != 1:
plural = 's'
if skipped > 0:
skip_plural = ''
if skipped > 1:
skip_plural = 's'
skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural)
exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert)
return exit_msg
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN')
self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def _full_path(self, name):
if name == '/':
name = ''
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
metadata = self.client.files_get_metadata(full_path)
for entry in metadata['contents']:
entry['path'] = entry['path'].replace(full_path, '', 1)
entry['path'] = entry['path'].replace('/', '', 1)
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
self.client.files_upload(content, self._full_path(name))
return name
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def user_authenticated(self):
try:
account = self.api_client.users_get_current_account()
log.Debug("User authenticated as ,%s" % account)
return True
except:
log.Debug('User not authenticated')
return False
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException(
'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException(
'dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError(
'dpbx error: cannot interact, but need human attention',
log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
authresult = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' %
e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(authresult.access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info(
"dpbx: Authentication failed. Trying to obtain new access token"
)
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception.
# So this line should not be reached
raise BackendException(
"dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" %
self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
progress.report_transfer(0, file_size)
if file_size < DPBX_UPLOAD_CHUNK_SIZE:
# Upload whole file at once to avoid extra server request
res_metadata = self.put_file_small(source_path, remote_path)
else:
res_metadata = self.put_file_chunked(source_path, remote_path)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException(
'dpbx: result path mismatch: %s (expected: %s)' %
(res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException(
'dpbx: result size mismatch: %s (expected: %s)' %
(res_metadata.size, file_size))
def put_file_small(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
log.Debug('dpbx,files_upload(%s, [%d bytes])' %
(remote_path, file_size))
res_metadata = self.api_client.files_upload(
f.read(),
remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
log.Debug('dpbx,files_upload(): %s' % res_metadata)
progress.report_transfer(file_size, file_size)
return res_metadata
finally:
f.close()
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug(
'dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id,
f.tell())
commit_info = CommitInfo(remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug(
'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(
buf, upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug(
'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
% (len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(
buf, upload_cursor, commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' %
(upload_cursor.offset, file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError
) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset(
).correct_offset
log.Debug(
'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
% (upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException(
"dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' %
(remote_path, res_metadata, http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata
# again. Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" %
(local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
if not self.user_authenticated():
self.login()
remote_dir = '/' + urllib.unquote(
self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
res = []
try:
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
except ApiError as e:
if (isinstance(e.error, ListFolderError) and e.error.is_path()
and e.error.get_path().is_not_found()):
log.Debug('dpbx.list(%s): ignore missing folder (%s)' %
(remote_dir, e))
else:
raise
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [
x for x in file_list
if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn(
'Warning! It looks like there are automatically renamed files on backend'
)
log.Warn(
'They were probably created when using older version of duplicity.'
)
log.Warn('')
log.Warn(
'Please check your backup consistency. Most likely you will need to choose'
)
log.Warn(
'largest file from duplicity-* (number).gpg and remove brackets from its name.'
)
log.Warn('')
log.Warn(
'These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
CHUNK_SIZE = 4 * 1024 * 1024
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = oauth2_access_token or setting(
'DROPBOX_OAUTH2_TOKEN')
self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = Dropbox(oauth2_access_token)
def _full_path(self, path):
path = PurePosixPath(self.root_path) / path
path = str(path)
if path == '/':
path = ''
return path
def delete(self, name):
self.client.files_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.files_get_metadata(self._full_path(name)))
except ApiError:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
result = self.client.files_list_folder(full_path)
for entry in result.entries:
if isinstance(entry, FolderMetadata):
directories.append(entry.name)
else:
files.append(entry.name)
assert not result.has_more, "FIXME: Not implemented!"
return directories, files
def size(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.size
def modified_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
return metadata.server_modified
def accessed_time(self, name):
metadata = self.client.files_get_metadata(self._full_path(name))
# Note to the unwary, this is actually an mtime
return metadata.client_modified
def url(self, name):
try:
media = self.client.files_get_temporary_link(self._full_path(name))
return media.link
except ApiError:
raise ValueError("This file is not accessible via a URL.")
def _open(self, name, mode='rb'):
return DropBoxFile(self._full_path(name), self)
def _save(self, name, content):
try:
content.open()
if content.size <= self.CHUNK_SIZE:
self.client.files_upload(content.read(), self._full_path(name))
else:
self._chunked_upload(content, self._full_path(name))
finally:
content.close()
return name
def _chunked_upload(self, content, dest_path):
upload_session = self.client.files_upload_session_start(
content.read(self.CHUNK_SIZE))
cursor = UploadSessionCursor(session_id=upload_session.session_id,
offset=content.tell())
commit = CommitInfo(path=dest_path)
while content.tell() < content.size:
if (content.size - content.tell()) <= self.CHUNK_SIZE:
self.client.files_upload_session_finish(
content.read(self.CHUNK_SIZE), cursor, commit)
else:
self.client.files_upload_session_append_v2(
content.read(self.CHUNK_SIZE), cursor)
cursor.offset = content.tell()
class DropboxConnection(object):
class Parameters(object):
def __init__(self, auth_token, remote_dir_path):
self._auth_token = auth_token
self._remote_dir_path = remote_dir_path
@property
def auth_token(self):
return self._auth_token
@property
def remote_dir_path(self):
return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path
def __str__(self):
return """
remote_dir_path: {}
""".format(self.remote_dir_path)
@classmethod
def get_client_from_params(cls, dropbox_parameters):
#Dropbox connection placeholder
dropbox = None
if dropbox_parameters:
dropbox_params = DropboxConnection.Parameters(
dropbox_parameters[0], dropbox_parameters[1])
dropbox = DropboxConnection(dropbox_params)
return dropbox
@classmethod
def get_client(cls, auth_token, remote_dir_path):
#Initialize the paramters
params = DropboxConnection.Parameters(auth_token, remote_dir_path)
#Create dropbox client
client = DropboxConnection(params)
return client
def __init__(self, params):
#Required parameters
self._params = params
#Derived parameters
self._client = Dropbox(self._params.auth_token)
#Logging
self._logger = logging.get_logger(__name__)
def upload(self, source_file_path):
"""It upload the source files to the dropbox.
Arguments:
source_file_path {string} -- The source file path.
Raises:
ValueError -- It raise value error for invalid files.
"""
#Validate parameters
if not source_file_path:
raise ValueError("Invalid source file path")
with open(source_file_path, 'rb') as handle:
#Source file name
source_file_name = Path(source_file_path).name
#Remote path
remote_file_path = (self._params.remote_dir_path /
source_file_name).as_posix()
#File size
upload_size = path.getsize(source_file_path)
#Upload the files based on the upload size
if upload_size <= constants.DROPBOX_CHUNK_SIZE:
self._logger.info(
'Preparing to upload small file: %s with size: %d to: %s',
source_file_path, upload_size, remote_file_path)
self._upload_small_file(handle, remote_file_path)
else:
self._logger.info(
'Preparing to upload large file: %s with size: %d to: %s',
source_file_path, upload_size, remote_file_path)
self._upload_large_file(handle, upload_size, remote_file_path)
self._logger.info('Uploaded: %s', source_file_path)
def _upload_small_file(self, handle, remote_file_path):
"""It uploads a small source files to the dropbox.
Arguments:
handle {A File handle} -- The source file handle.
remote_file_path {string} -- The destination path of the file.
"""
self._client.files_upload(handle.read(),
remote_file_path,
mode=Dropbox_WriteMode.overwrite)
def _upload_large_file(self, handle, upload_size, remote_file_path):
"""It uploads a large source files to the dropbox.
Arguments:
handle {A File handle} -- The source file handle.
upload_size {int} -- The number of bytes to be uploaded.
remote_file_path {string} -- The destination path of the file.
"""
#Upload session
session = self._client.files_upload_session_start(
handle.read(constants.DROPBOX_CHUNK_SIZE))
cursor = Dropbox_UploadSessionCursor(session_id=session.session_id,
offset=handle.tell())
#Upload look
with tqdm(desc='Uploading: {}'.format(remote_file_path),
total=upload_size) as pbar:
#Update the progress bar for the session start reads
pbar.update(handle.tell())
while handle.tell() < upload_size:
#Calculate remaining bytes
remaining_bytes = upload_size - handle.tell()
#If it is the last chunk, finalize the upload
if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE:
#Commit info
commit = Dropbox_CommitInfo(
path=remote_file_path,
mode=Dropbox_WriteMode.overwrite)
#Finish upload
self._client.files_upload_session_finish(
handle.read(remaining_bytes), cursor, commit)
#Update progress
pbar.update(remaining_bytes)
#More than chunk size remaining to upload
else:
self._client.files_upload_session_append_v2(
handle.read(constants.DROPBOX_CHUNK_SIZE), cursor)
#Update the cursor
cursor.offset = handle.tell()
#Update the progress
pbar.update(constants.DROPBOX_CHUNK_SIZE)
#Refresh the progress bar
pbar.refresh()
def download(self, remote_file_path):
"""It downloads the remote files from the dropbox.
Arguments:
remote_file_path {Path} -- The path to the remote file.
Raises:
ValueError -- It raise value error for invalid file name.
"""
#Validate parameters
if not remote_file_path:
raise ValueError("Invalid remote file path")
#Destination file path
dest_file_path = remote_file_path
#Full remote file path
remote_file_path = self._params.remote_dir_path / remote_file_path
#Download file size placeholder
download_size = 0
try:
download_size = self._client.files_get_metadata(
remote_file_path.as_posix()).size
except ApiError as e:
raise FileNotFoundError(
'File: {} is not found'.format(remote_file_path.as_posix()), e)
self._logger.info('Preparing file download: %s with size: %d to: %s',
remote_file_path, download_size, dest_file_path)
#Download the file
self._download_file(dest_file_path, remote_file_path, download_size)
self._logger.info('Completed the file download: %s to: %s',
remote_file_path, dest_file_path)
def _download_file(self, dest_file_path, remote_file_path, download_size):
"""It downloads the remote files from the dropbox.
Arguments:
remote_file_path {A Path object} -- The path of the remote file.
dest_file_path {string} -- The destination file path.
download_size {int} -- The number of bytes to be downloaded.
"""
#Download
_, result = self._client.files_download(remote_file_path.as_posix())
#Temporary dest_file_name
tmp_dest_file_path = "{}.tmp".format(dest_file_path)
with open(tmp_dest_file_path, 'wb') as handle:
with tqdm(desc='Downloading: {}'.format(
remote_file_path.as_posix()),
total=download_size) as pbar:
for bytes_read in result.iter_content(
constants.DROPBOX_CHUNK_SIZE):
handle.write(bytes_read)
#Update the progress
pbar.update(len(bytes_read))
if Path(tmp_dest_file_path).exists():
rename(tmp_dest_file_path, dest_file_path)
def list(self, dir_path=Path(), file_name_prefix=''):
"""It lists the files in the dropbox folder that starts with the given prefix.
Arguments:
file_name_prefix {string} -- The prefix to filter the results.
"""
#Candidate directory whose contents are to be listed
candidate_dir_path = self._params.remote_dir_path / dir_path
self._logger.info('Enumerating: %s with file_name_prefix: %s',
candidate_dir_path, file_name_prefix)
#Call the downstream API
response = self._client.files_list_folder(
candidate_dir_path.as_posix())
#Output list placeholder
files = []
sizes = []
if response.entries:
#Log the response summary
self._logger.info('Got %d files in: %s', len(response.entries),
candidate_dir_path)
#Extract the name of files satisfying the input criteria from the response entries.
file_infos = [(dir_path / entry.name, entry.size)
for entry in response.entries
if entry.name.startswith(file_name_prefix)]
files, sizes = zip(*file_infos)
return files, sizes
def uploadFile(file, file_name):
dbx = Dropbox(token)
dbx.files_upload(file.read(), "/" + file_name, mode=WriteMode('overwrite'))
def process_user(account):
'''
Call /files/list_folder for the given user ID and process any changes.
'''
print("*" * 30, " - PROCESS_USER - ", "*" * 30)
# OAuth token for the user
token = redis_client.hget('tokens', account)
print("token: ", token)
# cursor for the user (None the first time)
cursor = redis_client.hget('cursors', account)
print("cursor: ", cursor)
output_file_extension = ".csv"
input_file_extensions = [
".textgrid",
".TextGrid",
]
processed_marker = "_cleaned"
dbx = Dropbox(token.decode())
has_more = True
while has_more:
print("there's more!")
if cursor is None:
print("cursor is 'None'!")
result = dbx.files_list_folder(path='')
print("result: ", result)
else:
print("entering files_list_folder_continue...")
result = dbx.files_list_folder_continue(cursor.decode())
print("result: ", result)
for entry in result.entries:
print("entry: ", entry)
# yapf: disable
if (
isinstance(entry, DeletedMetadata) or
isinstance(entry, FolderMetadata) or
not any(entry.path_lower.endswith(e)
for e in input_file_extensions)
):
# yapf: enable
print("skipping.")
continue
_, resp = dbx.files_download(entry.path_lower)
print("processing data...")
processed_data = clean_data(resp.content)
dbx.files_upload(processed_data,
entry.path_lower[:-4] + processed_marker +
output_file_extension,
mode=WriteMode('add'))
#mode=WriteMode('overwrite'))
# Update cursor
cursor = result.cursor
redis_client.hset('cursors', account, cursor)
# Repeat only if there's more to do
has_more = result.has_more
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox Public folder.
"""
def __init__(self, location='/Public'):
self.client = Dropbox(ACCESS_TOKEN)
self.account_info = self.client.users_get_current_account()
self.location = location
self.base_url = 'https://dl.dropboxusercontent.com/'
def _get_abs_path(self, name):
return os.path.realpath(os.path.join(self.location, name))
def _open(self, name, mode='rb'):
name = self._get_abs_path(name)
remote_file = DropboxFile(name, self, mode=mode)
return remote_file
def _save(self, name, content):
name = self._get_abs_path(name)
directory = os.path.dirname(name)
if not self.exists(directory) and directory:
self.client.files_create_folder(directory)
# response = self.client.files_get_metadata(directory)
# if not response['is_dir']:
# raise IOError("%s exists and is not a directory." % directory)
abs_name = os.path.realpath(os.path.join(self.location, name))
foo = self.client.files_upload(content.read(), abs_name)
return name
def delete(self, name):
name = self._get_abs_path(name)
self.client.files_delete(name)
def exists(self, name):
name = self._get_abs_path(name)
try:
self.client.files_get_metadata(name)
except ApiError as e:
if e.error.is_path() and e.error.get_path().is_not_found(): # not found
return False
raise e
return True
def listdir(self, path):
path = self._get_abs_path(path)
response = self.client.files_list_folder(path)
directories = []
files = []
for entry in response.entries:
if type(entry) == FolderMetadata:
directories.append(os.path.basename(entry.path_display))
elif type(entry) == FileMetadata:
files.append(os.path.basename(entry.path_display))
return directories, files
def size(self, name):
cache_key = 'django-dropbox-size:{}'.format(filepath_to_uri(name))
size = cache.get(cache_key)
if not size:
size = self.client.files_get_metadata(name).size
cache.set(cache_key, size, CACHE_TIMEOUT)
return size
def url(self, name):
if name.startswith(self.location):
name = name[len(self.location) + 1:]
name = os.path.basename(self.location) + "/" + name
if self.base_url is None:
raise ValueError("This file is not accessible via a URL.")
myurl = urlparse.urljoin(self.base_url, filepath_to_uri(name))
if "static" not in self.location:
# Use a dynamic URL for "non-static" files.
try:
new_name = os.path.dirname(self.location) + "/" + name
fp = filepath_to_uri(new_name)
cache_key = 'django-dropbox-size:{}'.format(fp)
myurl = cache.get(cache_key)
if not myurl:
try:
shared_link = self.client.sharing_create_shared_link(fp)
myurl = shared_link.url + '&raw=1'
logger.debug("shared link: {0}, myurl: {1}".format(shared_link, myurl))
except Exception,e:
logger.exception(e)
if myurl is None:
temp_link = self.client.files_get_temporary_link(fp)
myurl = temp_link.link
logger.debug("temp link: {0}, myurl: {1}".format(temp_link, myurl))
cache.set(cache_key, myurl, SHARE_LINK_CACHE_TIMEOUT)
except Exception,e:
logger.exception(e)
return myurl
"""
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {'mode': WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
'recursive': True,
'include_non_downloadable_files': False
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as err:
if _is_file_not_found_error(err):
raise KeyError(f"Key doesn't exist: {k}")
else:
raise ValueError(
"Some unknown error happened (sorry, the lazy dev didn't tell me more than that)."
)
if contents_response.status_code:
return contents_response.content
else:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
# def _entry_is_dir(entry):
# return not hasattr(entry, 'is_downloadable')
#
#
# def _entry_is_file(entry):
# return hasattr(entry, 'is_downloadable')
#
#
# def _extend_path(path, extension):
# extend_path = '/' + path + '/' + extension + '/'
# extend_path.replace('//', '/')
# return extend_path
#
#
# class DropboxLinkPersister(DropboxPersister):
# def __init__(self, url, oauth2_access_token):
# self._con = Dropbox(oauth2_access_token)
# self.url = url
# self.shared_link = SharedLink(url=url)
#
# def _yield_from_files_list_folder(self, path, path_gen):
# """
# yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
# in a depth search manner.
# """
# for x in path_gen.entries:
# try:
# if _entry_is_file(x):
# yield x.name
# else:
# folder_path = _extend_path(path, x.name)
# yield from self._get_path_gen_from_path(path=folder_path)
# except Exception as e:
# print(e)
# if path_gen.has_more:
# yield from self._get_path_gen_from_cursor(path_gen.cursor, path=path)
#
# def _get_path_gen_from_path(self, path):
# path_gen = self._con.files_list_folder(path=path, recursive=False, shared_link=self.shared_link)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def _get_path_gen_from_cursor(self, cursor, path):
# path_gen = self._con.files_list_folder_continue(cursor)
# yield from self._yield_from_files_list_folder(path, path_gen)
#
# def __iter__(self):
# yield from self._get_path_gen_from_path(path='')
class DropboxPersister(Persister):
"""
A persister for dropbox.
You need to have the python connector (if you don't: pip install dropbox)
You also need to have a token for your dropbox app. If you don't it's a google away.
Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.
>>> import json
>>> import os
>>>
>>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
>>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
>>> if '/py2store_data/test/_can_remove' in s:
... del s['/py2store_data/test/_can_remove']
...
>>>
>>> n = len(s)
>>> if n == 1:
... assert list(s) == ['/py2store_data/test/_can_remove']
...
>>> s['/py2store_data/test/_can_remove'] = b'this is a test'
>>> assert len(s) == n + 1
>>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
>>> '/py2store_data/test/_can_remove' in s
True
>>> del s['/py2store_data/test/_can_remove']
"""
def __init__(
self,
rootdir,
oauth2_access_token,
connection_kwargs=None,
files_upload_kwargs=None,
files_list_folder_kwargs=None,
rev=None,
):
if connection_kwargs is None:
connection_kwargs = {}
if files_upload_kwargs is None:
files_upload_kwargs = {"mode": WriteMode.overwrite}
if files_list_folder_kwargs is None:
files_list_folder_kwargs = {
"recursive": True,
"include_non_downloadable_files": False,
}
self._prefix = rootdir
self._con = Dropbox(oauth2_access_token, **connection_kwargs)
self._connection_kwargs = connection_kwargs
self._files_upload_kwargs = files_upload_kwargs
self._files_list_folder_kwargs = files_list_folder_kwargs
self._rev = rev
# TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!
def __iter__(self):
r = self._con.files_list_folder(self._prefix)
yield from (x.path_display for x in r.entries)
cursor = r.cursor
if r.has_more:
r = self._con.files_list_folder_continue(cursor)
yield from (x.path_display for x in r.entries)
def __getitem__(self, k):
try:
metadata, contents_response = self._con.files_download(k)
except ApiError as e:
if _is_file_not_found_error(e):
raise KeyError(f"Key doesn't exist: {k}")
raise
if not contents_response.status_code:
raise ValueError(
"Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
)
return contents_response.content
def __setitem__(self, k, v):
return self._con.files_upload(v, k, **self._files_upload_kwargs)
def __delitem__(self, k):
return self._con.files_delete_v2(k, self._rev)
class rpiImageDbxClass(rpiBaseClass):
"""
Implements the rpiImageDb class to manage images in a remote directory (dropbox).
"""
def __init__(self, name, rpi_apscheduler, rpi_events, rpi_config, cam_rpififo=None):
### Get the Dbx error event
#self._eventDbErr = rpi_events.eventErrList["DBXJob"]
### Get the custom config parameters
self._config = rpi_config
### Get FIFO buffer for images from the camera (deque)
self._imageFIFO = cam_rpififo
### The FIFO buffer for the uploaded images (deque)
self.imageUpldFIFO = rpififo.rpiFIFOClass([], 576)
self.imageUpldFIFO.crtSubDir = ''
### Init base class
super().__init__(name, rpi_apscheduler, rpi_events)
def __repr__(self):
return "<%s (name=%s, rpi_apscheduler=%s, rpi_events=dict(), rpi_config=%s, dbuff_rpififo=%s)>" % (self.__class__.__name__, self.name, self._sched, self._config, self._imageFIFO)
def __str__(self):
msg = super().__str__()
return "%s::: dbinfo: %s, config: %s\nimageUpldFIFO: %s\n%s" % \
(self.name, self.dbinfo, self._config, self.imageUpldFIFO, msg)
def __del__(self):
### Clean base class
super().__del__()
#
# Main interface methods
#
def jobRun(self):
try:
# Lock the buffer
self._imageFIFO.acquireSemaphore()
# Get the current images in the FIFO
# Refresh the last remote image when available
if len(self._imageFIFO):
# Update remote cam image with the current (last) image
if not (self._imageFIFO[-1] == self.crt_image_snap):
self._putImage(self._imageFIFO[-1], self._config['image_snap'], True)
self.crt_image_snap = self._imageFIFO[-1]
self.numImgUpdDb += 1
logging.info("Updated remote %s with %s" % (self._config['image_snap'], self._imageFIFO[-1]) )
# Lock the upload buffer
self.imageUpldFIFO.acquireSemaphore()
# Check if a new upload sub-folder has to be used
if not (self.imageUpldFIFO.crtSubDir == self._imageFIFO.crtSubDir):
self.imageUpldFIFO.crtSubDir = self._imageFIFO.crtSubDir
self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
self._mkdirImage(self.upldir)
# Upload only images in the FIFO which have not been uploaded yet
for img in self._imageFIFO:
if not img in self.imageUpldFIFO:
self._putImage(img, os.path.join(self.upldir, os.path.basename(img)))
logging.info("Uploaded %s" % img )
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
# Update status
self.statusUpdate = (self.name, self.numImgUpdDb)
else:
# Update status
self.statusUpdate = (self.name, ERRNONE)
logging.info('Nothing to upload')
# Handle exceptions, mostly HTTP/SSL related!
except exceptions.Timeout as e:
# Catching this error will catch both ReadTimeout and ConnectTimeout.
raise rpiBaseClassError("%s::: jobRun(): Connect/ReadTimeoutError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.ConnectionError as e:
# A Connection error occurred.
raise rpiBaseClassError("%s::: jobRun(): ConnectionError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.HTTPError as e:
# An HTTP error occurred.
raise rpiBaseClassError("%s::: jobRun(): HTTPError:\n%s" % (self.name, str(e)), ERRLEV2)
except exceptions.RequestException as e:
# There was an ambiguous exception that occurred while handling your request.
raise rpiBaseClassError("%s::: jobRun(): RequestException:\n%s" % (self.name, str(e)), ERRLEV2)
# except BadStatusLine as e:
# self.eventErr_set('run()')
# logging.debug("BadStatusLine:\n%s" % str(e))
# pass
except rpiBaseClassError as e:
if e.errval == ERRCRIT:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): %s" % (self.name, e.errmsg), e.errval)
except RuntimeError as e:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): RuntimeError:\n%s" % (self.name, str(e)), ERRCRIT)
except:
self.endDayOAM()
raise rpiBaseClassError("%s::: jobRun(): Unhandled Exception:\n%s" % (self.name, str(sys.exc_info())), ERRCRIT)
finally:
# Release the buffer
self._imageFIFO.releaseSemaphore()
def initClass(self):
""""
(re)Initialize the class.
"""
#self.imageDbHash = None
self._imageDbCursor = None
self.imageDbList = []
self.numImgUpdDb = 0
self.crt_image_snap = None
self.imgid = self._imageFIFO.camID + '.jpg'
self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
self.logfile = './upldlog.json'
### When there are already images listed in the upload log file, then
# make sure we don't upload them to the remote folder again
# Else, create the file with an empty list; to be updated in endDayOAM()
try:
self.imageUpldFIFO.acquireSemaphore()
self.imageUpldFIFO.clear()
if os.path.isfile(self.logfile):
with open(self.logfile,'r') as logf:
upldimg = json.load(logf)
for img in upldimg:
self.imageUpldFIFO.append(img)
del upldimg
logging.info("%s::: Local log file %s found and loaded." % (self.name, self.logfile))
else:
with open(self.logfile,'w') as logf:
json.dump([], logf)
logging.info("%s::: Local log file %s initialized." % (self.name, self.logfile))
except IOError:
raise rpiBaseClassError("%s::: initClass(): Local log file %s was not found or could not be created." % (self.name, self.logfile), ERRCRIT)
finally:
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
### Init Dropbox API client
self._token_file = self._config['token_file']
self._dbx = None
self.dbinfo = None
try:
with open(self._token_file, 'r') as token:
self._dbx = Dropbox(token.read())
info = self._dbx.users_get_current_account()
# info._all_field_names_ =
# {'account_id', 'is_paired', 'locale', 'email', 'name', 'team', 'country', 'account_type', 'referral_link'}
self.dbinfo ={'email': info.email, 'referral_link': info.referral_link}
logging.info("%s::: Loaded access token from ''%s''" % (self.name, self._token_file) )
### Create remote root folder (relative to app root) if it does not exist yet
self._mkdirImage(os.path.normpath(self._config['image_dir']))
except rpiBaseClassError as e:
if e.errval == ERRCRIT:
self.endDayOAM()
raise rpiBaseClassError("initClass(): %s" % e.errmsg, e.errval)
except IOError:
self.endDayOAM()
raise rpiBaseClassError("initClass(): Token file ''%s'' could not be read." % (self.name, self._token_file), ERRCRIT)
except AuthError as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): AuthError:\n%s" % e.error, ERRCRIT)
except DropboxException as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): DropboxException:\n%s" % str(e), ERRCRIT)
except InternalServerError as e:
self.endDayOAM()
raise rpiBaseClassError("initClass(): InternalServerError:\n%s" % str(e.status_code), ERRCRIT)
def endDayOAM(self):
"""
End-of-Day Operation and Maintenance sequence.
"""
self._lsImage(self.upldir)
logging.info("%s::: %d images in the remote folder %s" % (self.name, len(self.imageDbList), self.upldir))
# Lock the uplaod buffer
self.imageUpldFIFO.acquireSemaphore()
try:
upldimg=[]
for img in self.imageUpldFIFO:
upldimg.append(img)
with open(self.logfile,'w') as logf:
json.dump(upldimg, logf)
del upldimg
logging.info("%s::: Local log file %s updated." % (self.name, self.logfile))
except IOError:
raise rpiBaseClassError("endDayOAM(): Local log file %s was not found." % self.logfile, ERRCRIT)
finally:
# Release the upload buffer
self.imageUpldFIFO.releaseSemaphore()
# def endOAM(self):
# """
# End OAM procedure.
# """
@atexit.register
def atexitend():
self.endDayOAM()
def _lsImage(self,from_path):
"""
List the image/video files in the remote directory.
Stores the found file names in self.imageDbList.
"""
try:
if self._imageDbCursor is None:
self.ls_ref = self._dbx.files_list_folder('/' + os.path.normpath(from_path), recursive=False, include_media_info=True )
else:
new_ls = self._dbx.files_list_folder_continue(self._imageDbCursor)
if new_ls.entries == []:
logging.debug("%s::: _lsImage():: No changes on the server." % self.name)
else:
self.ls_ref = new_ls
# Select only images and only the ones for the current imgid (camid)
foundImg = False
for f in self.ls_ref.entries:
if 'media_info' in f._all_field_names_ and \
f.media_info is not None:
if self.imgid in f.path_lower:
img = '.%s' % f.path_lower
foundImg = True
if not img in self.imageDbList:
self.imageDbList.append(img)
if not foundImg:
self.imageDbList = []
### Store the hash of the folder
self._imageDbCursor = self.ls_ref.cursor
if len(self.imageDbList) > 0:
logging.debug("%s::: _lsImage():: imageDbList[0..%d]: %s .. %s" % (self.name, len(self.imageDbList)-1, self.imageDbList[0], self.imageDbList[-1]) )
else:
logging.debug("%s::: _lsImage():: imageDbList[]: empty" % self.name)
except ApiError as e:
raise rpiBaseClassError("_lsImage(): %s" % e.error, ERRLEV2)
def _putImage(self, from_path, to_path, overwrite=False):
"""
Copy local file to remote file.
Stores the uploaded files names in self.imageUpldFIFO.
Examples:
_putImage('./path/test.jpg', '/path/dropbox-upload-test.jpg')
"""
try:
mode = (WriteMode.overwrite if overwrite else WriteMode.add)
with open(from_path, "rb") as from_file:
self._dbx.files_upload( from_file, '/' + os.path.normpath(to_path), mode)
if not overwrite:
self.imageUpldFIFO.append(from_path)
logging.debug("%s::: _putImage(): Uploaded file from %s to remote %s" % (self.name, from_path, to_path))
except IOError:
raise rpiBaseClassError("_putImage(): Local img file %s could not be opened." % from_path, ERRCRIT)
except ApiError as e:
raise rpiBaseClassError("_putImage(): %s" % e.error, ERRLEV2)
def _mkdirImage(self, path):
"""
Create a new remote directory.
Examples:
_mkdirImage('/dropbox_dir_test')
"""
try:
self._dbx.files_create_folder('/' + os.path.normpath(path))
logging.debug("%s::: Remote output folder /%s created." % (self.name, path))
except ApiError as e:
noerr = False
# dropbox.files.CreateFolderError
if e.error.is_path():
# dropbox.files.WriteError
we = e.error.get_path()
if we.is_conflict():
# dropbox.files.WriteConflictError
wce = we.get_conflict()
# union tag is 'folder'
if wce.is_folder():
logging.info("%s::: Remote output folder /%s already exist!" % (self.name, path))
noerr = True
if not noerr:
raise rpiBaseClassError("_mkdirImage(): Remote output folder /%s was not created! %s" % (path, e.error), ERRCRIT)
else:
pass
def _mvImage(self, from_path, to_path):
"""
Move/rename a remote file or directory.
Examples:
_mvImage('./path1/dropbox-move-test.jpg', '/path2/dropbox-move-test.jpg')
"""
try:
self._dbx.files_move( '/' + os.path.normpath(from_path), '/' + os.path.normpath(to_path) )
logging.debug("%s::: _mvImage(): Moved file from %s to %s" % (self.name, from_path, to_path))
except ApiError as e:
raise rpiBaseClassError("_mvImage(): Image %s could not be moved to %s! %s" % (from_path, to_path, e.error), ERRLEV2)
from dropbox import Dropbox, files
from dotenv import load_dotenv
from glob import glob
from datetime import datetime
from os import path, environ, remove
load_dotenv()
token = environ['DROPBOX_TOKEN']
if __name__ == '__main__':
dbx = Dropbox(token)
dbx.users_get_current_account()
now = datetime.now()
csv_list = glob('csv/*.csv')
for file_path in csv_list:
with open(file_path, 'rb') as file:
dbx.files_upload(file.read(),
f'/{path.basename(file_path)}',
mode=files.WriteMode.overwrite)
delta = now - datetime.fromtimestamp(path.getmtime(file_path))
if delta.days > 0:
remove(file_path)
print(f'uploaded and remove: {file_path}')
else:
print(f'uploaded: {file_path}')
class Capture:
def __init__(self):
self.size = (1280, 720)
# create a display surface. standard pygame stuff
self.display = pygame.display.set_mode(self.size, 0)
# this is the same as what we saw before
self.clist = pygame.camera.list_cameras()
if not self.clist:
raise ValueError("Sorry, no cameras detected.")
self.cam = pygame.camera.Camera(self.clist[0], self.size)
self.cam.start()
# create a surface to capture to. for performance purposes
# bit depth is the same as that of the display surface.
self.snapshot = pygame.surface.Surface(self.size, 0, self.display)
self.text = 'Drop yer fone to take a sick insta!'
self.state = "waiting"
self.start_time = None
self.dropbox = Dropbox(os.environ['DROPBOX_TOKEN'])
def draw(self):
# if you don't want to tie the framerate to the camera, you can check
# if the camera has an image ready. note that while this works
# on most cameras, some will never return true.
if self.cam.query_image() and self.state != "show_photo":
self.snapshot = self.cam.get_image(self.snapshot)
# blit it to the display surface. simple!
self.display.blit(self.snapshot, (0, 0))
# Render Text
white = (255, 255, 255)
green = (0, 255, 0)
blue = (0, 0, 128)
font = pygame.font.Font('freesansbold.ttf', 32)
if self.state == "countdown" or self.state == "show_photo":
timer = 3.99 - (time.time() - self.start_time)
text = str(int(timer))
if -1 < timer < 1:
text = "Say cheese!"
if timer < -1:
self.state = "show_photo"
text = "Photo uploading..."
if timer < -4:
self.state = "waiting"
text = self.text
filename = f"{datetime.utcnow().strftime('%y_%m_%d_%H_%M_%S')}.jpg"
full_path = f"/home/pi/Dropbox/photobooth/{filename}"
pygame.image.save(self.snapshot, full_path)
with open(full_path, 'rb') as f:
self.dropbox.files_upload(
f.read(), path=f'/photobooth_cmci_studio/{filename}')
else:
text = self.text
text_obj = font.render(text, True, green, blue)
textRect = text_obj.get_rect()
textRect.center = (1280 // 2, int(720 * .90))
self.display.blit(text_obj, textRect)
pygame.display.flip()
def main(self):
going = True
while going:
events = pygame.event.get()
self.draw()
for e in events:
print(e)
if e.type == QUIT or (e.type == KEYDOWN and e.key == K_ESCAPE):
# close the camera safely
self.cam.stop()
going = False
if (e.type == KEYDOWN and e.key == K_SPACE):
self.trigger()
def trigger(self, charger_on_list=None):
print('Detected Charger On:', charger_on_list)
self.state = "countdown"
self.start_time = time.time()
def main() -> None:
logger.info("Loading parameters...")
if "TARGET_USERNAME" not in os.environ:
raise TweetBriefError("TARGET_USERNAME not found!")
if "CONSUMER_KEY" not in os.environ:
raise TweetBriefError("CONSUMER_KEY not found!")
if "CONSUMER_SECRET" not in os.environ:
raise TweetBriefError("CONSUMER_SECRET not found!")
if not any(storage in os.environ for storage in ["BRIEF_OUTPUT", "DROPBOX_ACCESS_TOKEN"]):
raise TweetBriefError("No storage provided!")
# Twitter API parameters
consumer_key = os.getenv("CONSUMER_KEY")
consumer_secret = os.getenv("CONSUMER_SECRET")
# bot parameters
target_username = os.getenv("TARGET_USERNAME")
single_author_max_tweets = os.getenv("SINGLE_AUTHOR_MAX_TWEETS", 3)
brief_period = os.getenv("BRIEF_PERIOD", 1)
brief_max_tweets = os.getenv("BRIEF_MAX_TWEETS", 30)
url2qrcode = str2bool(os.getenv("URL2QR", "True"))
# storage parameters
brief_output = os.getenv("BRIEF_OUTPUT", None)
dropbox_access_token = os.getenv("DROPBOX_ACCESS_TOKEN", None)
try:
single_author_max_tweets = int(single_author_max_tweets)
if single_author_max_tweets > 3:
logger.warning("SINGLE_AUTHOR_MAX_TWEETS is greater than 3! Setting to default (3) ...")
single_author_max_tweets = 3
except ValueError:
logger.warning("SINGLE_AUTHOR_MAX_TWEETS must be an integer! Setting to default (3) ...")
single_author_max_tweets = 3
try:
brief_period = int(brief_period)
except ValueError:
logger.warning("BRIEF_PERIOD must be an integer! Setting to default (1) ...")
brief_period = 1
try:
brief_max_tweets = int(brief_max_tweets)
except ValueError:
logger.warning("BRIEF_MAX_TWEETS must be an integer! Setting to default (30) ...")
brief_max_tweets = 30
if brief_output is not None:
try:
brief_output = Path(brief_output)
if not brief_output.is_dir():
brief_output.mkdir(parents=True)
if not os.access(brief_output, os.W_OK):
raise PermissionError(f"No write permissions on `{brief_output}`!")
except (FileExistsError, PermissionError):
logger.error(f"The path `{brief_output}` is broken!")
raise
if dropbox_access_token is not None:
try:
dbx = Dropbox(dropbox_access_token)
dbx.users_get_current_account()
except (AuthError, BadInputError):
logger.error("`DROPBOX_ACCESS_TOKEN` is invalid!")
raise
logger.info("Parameters loaded")
logger.info("Extracting tweets...")
extractor = TweetExtractor(consumer_key, consumer_secret, tweet_mode="extended")
tweets_in_brief = extractor.extract_top_tweets(
target_username, single_author_max_tweets, brief_max_tweets, brief_period
)
logger.info("Exporting brief...")
exporter = PDFExporter(url2qrcode)
date_str = datetime.now().strftime("%Y-%m-%d")
period_desc = (
"Daily" if brief_period == 1 else
"Weekly" if brief_period == 7 else
"Monthly" if 30 <= brief_period <= 31 else
f"Last {brief_period} days"
)
title = f"{period_desc} Twitter Brief for @{target_username} ({date_str})"
subtitle = f"Excluding RTs, top {single_author_max_tweets} tweets/author, {datetime.now().strftime('%H:%M:%S UTC')}"
pdf = exporter.export(tweets_in_brief, title=title, subtitle=subtitle)
filename = f"tweetbrief-{target_username}-{period_desc.lower()}-{date_str}.pdf"
if brief_output is not None:
logger.info("Saving locally...")
brief_path = brief_output / filename
with open(brief_path, "wb") as f:
f.write(pdf.getbuffer())
logger.info("Brief saved")
if dropbox_access_token is not None:
logger.info("Uploading to Dropbox...")
brief_path = Path("/") / filename
try:
dbx.files_upload(pdf.getvalue(), brief_path.as_posix())
except ApiError:
logger.error("Brief exists!")
raise
logger.info("Brief uploaded")
class DPBXBackend(duplicity.backend.Backend):
"""Connect to remote store using Dr*pB*x service"""
def __init__(self, parsed_url):
duplicity.backend.Backend.__init__(self, parsed_url)
self.api_account = None
self.api_client = None
self.auth_flow = None
self.login()
def user_authenticated(self):
try:
account = self.api_client.users_get_current_account()
log.Debug("User authenticated as ,%s" % account)
return True
except:
log.Debug('User not authenticated')
return False
def load_access_token(self):
return os.environ.get('DPBX_ACCESS_TOKEN', None)
def save_access_token(self, access_token):
raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
access_token)
def obtain_access_token(self):
log.Info("dpbx: trying to obtain access token")
for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
if env_var not in os.environ:
raise BackendException('dpbx: %s environment variable not set' % env_var)
app_key = os.environ['DPBX_APP_KEY']
app_secret = os.environ['DPBX_APP_SECRET']
if not sys.stdout.isatty() or not sys.stdin.isatty():
log.FatalError('dpbx error: cannot interact, but need human attention',
log.ErrorCode.backend_command_error)
auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
log.Debug('dpbx,auth_flow.start()')
authorize_url = auth_flow.start()
print
print '-' * 72
print "1. Go to: " + authorize_url
print "2. Click \"Allow\" (you might have to log in first)."
print "3. Copy the authorization code."
print '-' * 72
auth_code = raw_input("Enter the authorization code here: ").strip()
try:
log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
authresult = auth_flow.finish(auth_code)
except Exception as e:
raise BackendException('dpbx: Unable to obtain access token: %s' % e)
log.Info("dpbx: Authentication successfull")
self.save_access_token(authresult.access_token)
def login(self):
if self.load_access_token() is None:
self.obtain_access_token()
self.api_client = Dropbox(self.load_access_token())
self.api_account = None
try:
log.Debug('dpbx,users_get_current_account([token])')
self.api_account = self.api_client.users_get_current_account()
log.Debug("dpbx,%s" % self.api_account)
except (BadInputError, AuthError) as e:
log.Debug('dpbx,exception: %s' % e)
log.Info("dpbx: Authentication failed. Trying to obtain new access token")
self.obtain_access_token()
# We're assuming obtain_access_token will throw exception.
# So this line should not be reached
raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again")
log.Info("dpbx: Successfully authenticated as %s" %
self.api_account.name.display_name)
def _error_code(self, operation, e):
if isinstance(e, ApiError):
err = e.error
if isinstance(err, GetMetadataError) and err.is_path():
if err.get_path().is_not_found():
return log.ErrorCode.backend_not_found
elif isinstance(err, DeleteError) and err.is_path_lookup():
lookup = e.error.get_path_lookup()
if lookup.is_not_found():
return log.ErrorCode.backend_not_found
@command()
def _put(self, source_path, remote_filename):
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
file_size = os.path.getsize(source_path.name)
progress.report_transfer(0, file_size)
if file_size < DPBX_UPLOAD_CHUNK_SIZE:
# Upload whole file at once to avoid extra server request
res_metadata = self.put_file_small(source_path, remote_path)
else:
res_metadata = self.put_file_chunked(source_path, remote_path)
# A few sanity checks
if res_metadata.path_display != remote_path:
raise BackendException('dpbx: result path mismatch: %s (expected: %s)' %
(res_metadata.path_display, remote_path))
if res_metadata.size != file_size:
raise BackendException('dpbx: result size mismatch: %s (expected: %s)' %
(res_metadata.size, file_size))
def put_file_small(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
log.Debug('dpbx,files_upload(%s, [%d bytes])' % (remote_path, file_size))
res_metadata = self.api_client.files_upload(f.read(), remote_path,
mode=WriteMode.overwrite,
autorename=False,
client_modified=None,
mute=True)
log.Debug('dpbx,files_upload(): %s' % res_metadata)
progress.report_transfer(file_size, file_size)
return res_metadata
finally:
f.close()
def put_file_chunked(self, source_path, remote_path):
if not self.user_authenticated():
self.login()
file_size = os.path.getsize(source_path.name)
f = source_path.open('rb')
try:
buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' %
(len(buf), file_size))
upload_sid = self.api_client.files_upload_session_start(buf)
log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite,
autorename=False, client_modified=None,
mute=True)
res_metadata = None
progress.report_transfer(f.tell(), file_size)
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
is_eof = False
# We're doing our own error handling and retrying logic because
# we can benefit from Dpbx chunked upload and retry only failed
# chunk
while not is_eof or not res_metadata:
try:
if requested_offset is not None:
upload_cursor.offset = requested_offset
if f.tell() != upload_cursor.offset:
f.seek(upload_cursor.offset)
buf = f.read(current_chunk_size)
is_eof = f.tell() >= file_size
if not is_eof and len(buf) == 0:
continue
# reset temporary status variables
requested_offset = None
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
retry_number = globals.num_retries
if not is_eof:
assert len(buf) != 0
log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' %
(len(buf), upload_cursor.offset))
self.api_client.files_upload_session_append(buf,
upload_cursor.session_id,
upload_cursor.offset)
else:
log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' %
(len(buf), upload_cursor.offset))
res_metadata = self.api_client.files_upload_session_finish(buf,
upload_cursor,
commit_info)
upload_cursor.offset = f.tell()
log.Debug('progress: %d of %d' % (upload_cursor.offset,
file_size))
progress.report_transfer(upload_cursor.offset, file_size)
except ApiError as e:
error = e.error
if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
# Server reports that we should send another chunk.
# Most likely this is caused by network error during
# previous upload attempt. In such case we'll get
# expected offset from server and it's enough to just
# seek() and retry again
new_offset = error.get_incorrect_offset().correct_offset
log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' %
(upload_cursor.offset, new_offset))
if requested_offset is not None:
# chunk failed even after seek attempt. Something
# strange and no safe way to recover
raise BackendException("dpbx: unable to chunk upload")
else:
# will seek and retry
requested_offset = new_offset
continue
raise
except ConnectionError as e:
log.Debug('dpbx,files_upload_session_append: %s' % e)
retry_number -= 1
if not self.user_authenticated():
self.login()
if retry_number == 0:
raise
# We don't know for sure, was partial upload successful or
# not. So it's better to retry smaller amount to avoid extra
# reupload
log.Info('dpbx: sleeping a bit before chunk retry')
time.sleep(30)
current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
requested_offset = None
continue
if f.tell() != file_size:
raise BackendException('dpbx: something wrong')
log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
progress.report_transfer(f.tell(), file_size)
return res_metadata
finally:
f.close()
@command()
def _get(self, remote_filename, local_path):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()
log.Debug('dpbx,files_download(%s)' % remote_path)
res_metadata, http_fd = self.api_client.files_download(remote_path)
log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata,
http_fd))
file_size = res_metadata.size
to_fd = None
progress.report_transfer(0, file_size)
try:
to_fd = local_path.open('wb')
for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
to_fd.write(c)
progress.report_transfer(to_fd.tell(), file_size)
finally:
if to_fd:
to_fd.close()
http_fd.close()
# It's different from _query() check because we're not querying metadata
# again. Since this check is free, it's better to have it here
local_size = os.path.getsize(local_path.name)
if local_size != file_size:
raise BackendException("dpbx: wrong file size: %d (expected: %d)" %
(local_size, file_size))
local_path.setdata()
@command()
def _list(self):
# Do a long listing to avoid connection reset
if not self.user_authenticated():
self.login()
remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()
log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
res = []
try:
resp = self.api_client.files_list_folder(remote_dir)
log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))
while True:
res.extend([entry.name for entry in resp.entries])
if not resp.has_more:
break
resp = self.api_client.files_list_folder_continue(resp.cursor)
except ApiError as e:
if (isinstance(e.error, ListFolderError) and e.error.is_path() and
e.error.get_path().is_not_found()):
log.Debug('dpbx.list(%s): ignore missing folder (%s)' % (remote_dir, e))
else:
raise
# Warn users of old version dpbx about automatically renamed files
self.check_renamed_files(res)
return res
@command()
def _delete(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_delete(%s)' % remote_path)
self.api_client.files_delete(remote_path)
# files_permanently_delete seems to be better for backup purpose
# but it's only available for Business accounts
# self.api_client.files_permanently_delete(remote_path)
@command()
def _close(self):
"""close backend session? no! just "flush" the data"""
log.Debug('dpbx.close():')
@command()
def _query(self, filename):
if not self.user_authenticated():
self.login()
remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
remote_path = '/' + os.path.join(remote_dir, filename).rstrip()
log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
info = self.api_client.files_get_metadata(remote_path)
log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
return {'size': info.size}
def check_renamed_files(self, file_list):
if not self.user_authenticated():
self.login()
bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
if len(bad_list) == 0:
return
log.Warn('-' * 72)
log.Warn('Warning! It looks like there are automatically renamed files on backend')
log.Warn('They were probably created when using older version of duplicity.')
log.Warn('')
log.Warn('Please check your backup consistency. Most likely you will need to choose')
log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.')
log.Warn('')
log.Warn('These files are not managed by duplicity at all and will not be')
log.Warn('removed/rotated automatically.')
log.Warn('')
log.Warn('Affected files:')
for x in bad_list:
log.Warn('\t%s' % x)
log.Warn('')
log.Warn('In any case it\'s better to create full backup.')
log.Warn('-' * 72)
