def _upload(self, client: DropboxClient):
""" Upload new files from directory.
"""
now = time()
for filename in listdir(self.directory):
if fnmatch(filename, '*.upl'):
continue
local_name = '/' + filename
full_name = path.join(self.directory, filename)
upl_name = "{}.upl".format(full_name)
if not path.isfile(upl_name) and stat(full_name).st_mtime < now - 60:
with open(full_name, 'rb') as file_stream:
try:
client.put_file(local_name, file_stream)
share = client.share(local_name)
except (MaxRetryError, ErrorResponse):
continue
with DatabaseConnection() as db:
update = """
UPDATE events
SET url = '{}',
uploaded = current_timestamp
WHERE file = '{}'
""".format(share['url'], full_name)
db.dml(update)
try:
mknod(upl_name)
except FileExistsError:
pass
print("{} was uploaded to Dropbox.".format(filename))
def get_dropbox_share(request):
if request.method == 'GET':
user_profile = UserProfile.objects.get(user = request.user)
drop_client = DropboxClient(user_profile.dropbox_profile.access_token['key'])
res = drop_client.share(request.GET.get('path'))
return redirect(res[u'url'])
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or
metadata['is_dir'] or
not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
response, metadata = client.get_file_and_metadata(path)
md = response.read().decode()
html = markdown(md, extensions=['gfm'])
html_name = path[:-3] + '.html'
client.put_file(html_name, html, overwrite=True)
# Include URL to published file in HTML comment at top of Markdown
# file
if '<!-- Published file url:' != md.split('\n')[0]:
share_url = client.share(html_name, short_url=False).get('url')
file_key = share_url.split('/')[4]
url_name = urllib.parse.quote(html_name)
url_comment = ('<!-- Published file url:\n'
'https://dl.dropboxusercontent.com/s/'
'{}{}\n-->\n'.format(file_key, url_name))
md = url_comment + md
client.put_file(path, md, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or metadata['is_dir']
or not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
response, metadata = client.get_file_and_metadata(path)
md = response.read().decode()
html = markdown(md, extensions=['gfm'])
html_name = path[:-3] + '.html'
client.put_file(html_name, html, overwrite=True)
# Include URL to published file in HTML comment at top of Markdown
# file
if '<!-- Published file url:' != md.split('\n')[0]:
share_url = client.share(html_name, short_url=False).get('url')
file_key = share_url.split('/')[4]
url_name = urllib.parse.quote(html_name)
url_comment = ('<!-- Published file url:\n'
'https://dl.dropboxusercontent.com/s/'
'{}{}\n-->\n'.format(file_key, url_name))
md = url_comment + md
client.put_file(path, md, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
class DropboxSyncClient:
def __init__(self, oauth2_access_token):
self.access_token = oauth2_access_token
self._client = DropboxClient(oauth2_access_token)
def upload_file(self, dropbox_file_path, local_file_path, replace=False):
f = open(local_file_path, 'rb')
response = self._client.put_file(dropbox_file_path, f, replace)
return 1, response['path']
def generate_public_url(self, dropbox_file_path):
return self._client.share(dropbox_file_path)['url']
def delete_file(self, dropbox_file_path):
self._client.file_delete(dropbox_file_path)
return 1, None
def update_local_to_cloud(self, dropbox_file_path, local_file_path):
return 1, self.upload_file(dropbox_file_path, local_file_path, replace=True)
def update_cloud_to_local(self, dropbox_file_path, local_file_path):
try:
try:
os.makedirs(os.path.dirname(local_file_path))
except Exception as e:
pass
open(local_file_path, 'wb').write(self._client.get_file(dropbox_file_path).read())
return 1, None
except Exception as e:
print e
return 1, None
def get_file_list(self, dropbox_folder_path):
folder_metadata = self._client.metadata(dropbox_folder_path)
return [content['path'] for content in folder_metadata['contents']]
def set_access_token(self, access_token):
self._client = DropboxClient(access_token)
def get_remaining_space(self):
quota_info = self._client.account_info()['quota_info']
return quota_info['total'] - (quota_info['shared'] + quota_info['normal'])
class DropboxStorage(object):
calibre_db_path = '/%s/metadata.db' % settings.DROPBOX_CALIBRE_DIR
dropbox_cursor_key = 'dropbox_cursor'
def __init__(self):
session = DropboxSession(settings.DROPBOX_CONSUMER_KEY,
settings.DROPBOX_CONSUMER_SECRET,
settings.DROPBOX_ACCESS_TYPE,
locale=None)
session.set_token(settings.DROPBOX_ACCESS_TOKEN,
settings.DROPBOX_ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
def get_url(self, path, share=False):
try:
if share:
result = self.client.share(path, short_url=False)
return result['url'] + '?dl=1'
return self.client.media(path).get('url')
except ErrorResponse:
pass
def get_file(self, path):
try:
return self.client.get_file(path)
except ErrorResponse:
pass
def sync_db(self):
calibre_db = self.client.get_file(self.calibre_db_path)
with open(settings.DATABASES['calibre']['NAME'], 'wb') as f:
f.write(calibre_db.read())
def need_update(self):
delta = self.client.delta(cursor=cache.get(self.dropbox_cursor_key),
path_prefix=self.calibre_db_path)
cache.set(self.dropbox_cursor_key, delta['cursor'], timeout=None)
return len(delta['entries']) > 0
class DropboxStorage(object):
calibre_db_path = '/%s/metadata.db' % settings.DROPBOX_CALIBRE_DIR
dropbox_cursor_key = 'dropbox_cursor'
def __init__(self):
session = DropboxSession(settings.DROPBOX_CONSUMER_KEY,
settings.DROPBOX_CONSUMER_SECRET,
settings.DROPBOX_ACCESS_TYPE, locale=None)
session.set_token(settings.DROPBOX_ACCESS_TOKEN,
settings.DROPBOX_ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
def get_url(self, path, share=False):
try:
if share:
result = self.client.share(path, short_url=False)
return result['url'] + '?dl=1'
return self.client.media(path).get('url')
except ErrorResponse:
pass
def get_file(self, path):
try:
return self.client.get_file(path)
except ErrorResponse:
pass
def sync_db(self):
calibre_db = self.client.get_file(self.calibre_db_path)
with open(settings.DATABASES['calibre']['NAME'], 'wb') as f:
f.write(calibre_db.read())
def need_update(self):
delta = self.client.delta(cursor=cache.get(self.dropbox_cursor_key),
path_prefix=self.calibre_db_path)
cache.set(self.dropbox_cursor_key, delta['cursor'], timeout=None)
return len(delta['entries']) > 0
def cluster_all_files():
#sign in
dbx = dropbox.Dropbox(
'F2_PGWfw-GAAAAAAAAAACqNZoyJNzdLMd7x-BKLsGSE7hHM07KRMfT6jJgtWLgub')
client = DropboxClient(
'F2_PGWfw-GAAAAAAAAAACqNZoyJNzdLMd7x-BKLsGSE7hHM07KRMfT6jJgtWLgub')
#debug info
newFileMeta = None
cathegory_list = {}
for entry in dbx.files_list_folder('').entries:
if '.jpg' in entry.name:
print(entry.name)
newFileMeta = entry
print(newFileMeta.path_lower)
newFile = dbx.files_download_to_file('tmp/' + newFileMeta.name,
newFileMeta.path_lower)
cath = (DropboxUpdate.analyzeFile('tmp/' + newFileMeta.name)
['categories'][0]['name'])
link = client.share(newFileMeta.path_lower)['url']
cathegory_list[link] = cath
print cathegory_list
with open('img_dat.txt', 'w') as outfile:
json.dump(cathegory_list, outfile)
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox Public folder.
"""
def __init__(self, location='/Public'):
session = DropboxSession(CONSUMER_KEY,
CONSUMER_SECRET,
ACCESS_TYPE,
locale=None)
session.set_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
self.account_info = self.client.account_info()
self.location = location
self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format(
**self.account_info)
def _get_abs_path(self, name):
return os.path.realpath(os.path.join(self.location, name))
def _open(self, name, mode='rb'):
name = self._get_abs_path(name)
remote_file = DropboxFile(name, self, mode=mode)
return remote_file
def _save(self, name, content):
name = self._get_abs_path(name)
directory = os.path.dirname(name)
if not self.exists(directory) and directory:
self.client.file_create_folder(directory)
response = self.client.metadata(directory)
if not response['is_dir']:
raise IOError("%s exists and is not a directory." % directory)
abs_name = os.path.realpath(os.path.join(self.location, name))
self.client.put_file(abs_name, content)
return name
def delete(self, name):
name = self._get_abs_path(name)
self.client.file_delete(name)
def exists(self, name):
name = self._get_abs_path(name)
try:
metadata = self.client.metadata(name)
if metadata.get('is_deleted'):
return False
except ErrorResponse as e:
if e.status == 404: # not found
return False
raise e
return True
def listdir(self, path):
path = self._get_abs_path(path)
response = self.client.metadata(path)
directories = []
files = []
for entry in response.get('contents', []):
if entry['is_dir']:
directories.append(os.path.basename(entry['path']))
else:
files.append(os.path.basename(entry['path']))
return directories, files
def size(self, name):
cache_key = 'django-dropbox-size:%s' % filepath_to_uri(name)
size = cache.get(cache_key)
if not size:
size = self.client.metadata(filepath_to_uri(name))['bytes']
cache.set(cache_key, size, CACHE_TIMEOUT)
return size
def url(self, name):
cache_key = 'django-dropbox-url:%s' % filepath_to_uri(name)
url = cache.get(cache_key)
if not url:
url = self.client.share(filepath_to_uri(name),
short_url=False)['url'] + '?dl=1'
cache.set(cache_key, url, CACHE_TIMEOUT)
return url
def get_available_name(self, name):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
name = self._get_abs_path(name)
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a number (before
# the file extension, if one exists) to the filename until the generated
# filename doesn't exist.
count = itertools.count(1)
while self.exists(name):
# file_ext includes the dot.
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, count.next(), file_ext))
return name
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox Public folder.
"""
def __init__(self, location=None):
session = DropboxSession(DROPBOX.app_key, DROPBOX.app_secret, DROPBOX.access_type, locale=None)
session.set_token(DROPBOX.access_key, DROPBOX.access_secret)
self.client = DropboxClient(session)
self.overwrite_mode = DROPBOX.overwrite_mode
self._location = location
@prepend_path_with_attr("_location")
def delete(self, name):
self.client.file_delete(name)
@prepend_path_with_attr("_location")
def exists(self, name):
try:
metadata = self.client.metadata(name, list=False)
return not metadata.get('is_deleted')
except ErrorResponse as e:
if 404 == e.status: # not found
return False
raise e
return True
@prepend_path_with_attr("_location")
def listdir(self, name, query_filter=None):
if query_filter is None or len(query_filter) < 3:
metadata = self.client.metadata(name)
else:
metadata = self.client.search(name, query_filter, file_limit=25000)
directories = []
files = []
for entry in metadata.get('contents', []):
if entry['is_dir']:
directories.append(os.path.basename(entry['path']))
else:
files.append(os.path.basename(entry['path']))
return directories, files
@prepend_path_with_attr("_location")
def open(self, name, mode='rb'):
return DropboxFile(name, self, mode)
@prepend_path_with_attr("_location")
def save(self, name, content):
metadata = self.client.put_file(name, content)
return metadata['path']
@prepend_path_with_attr("_location")
def size(self, name):
return self.client.metadata(name, list=False)['bytes']
@prepend_path_with_attr("_location")
def url(self, name):
try:
return self.client.share(name)['url']
except ErrorResponse as e:
if 404 == e.status: # not found
return None
raise e
@prepend_path_with_attr("_location")
def get_available_name(self, name):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
if self.overwrite_mode:
return name
if self.exists(name):
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a number (before
# the file extension, if one exists) to the filename until the generated
# filename doesn't exist.
dir_contents = self.listdir(dir_name, file_root)
count = itertools.count(1)
while True:
# file_ext includes the dot.
name = "%s_%s%s" % (file_root, count.next(), file_ext)
if name not in dir_contents:
return os.path.join(dir_name, name)
else:
return name
@prepend_path_with_attr("_location")
def get_valid_name(self, name):
return name
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox Public folder.
"""
def __init__(self, location='/Public'):
session = DropboxSession(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TYPE, locale=None)
session.set_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
self.account_info = self.client.account_info()
self.location = location
self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format(**self.account_info)
def _get_abs_path(self, name):
return os.path.realpath(os.path.join(self.location, name))
def _open(self, name, mode='rb'):
name = self._get_abs_path(name)
remote_file = DropboxFile(name, self, mode=mode)
return remote_file
def _save(self, name, content):
name = self._get_abs_path(name)
directory = os.path.dirname(name)
if not self.exists(directory) and directory:
self.client.file_create_folder(directory)
response = self.client.metadata(directory)
if not response['is_dir']:
raise IOError("%s exists and is not a directory." % directory)
abs_name = os.path.realpath(os.path.join(self.location, name))
self.client.put_file(abs_name, content)
return name
def delete(self, name):
name = self._get_abs_path(name)
self.client.file_delete(name)
def exists(self, name):
name = self._get_abs_path(name)
try:
metadata = self.client.metadata(name)
if metadata.get('is_deleted'):
return False
except ErrorResponse as e:
if e.status == 404: # not found
return False
raise e
return True
def listdir(self, path):
path = self._get_abs_path(path)
response = self.client.metadata(path)
directories = []
files = []
for entry in response.get('contents', []):
if entry['is_dir']:
directories.append(os.path.basename(entry['path']))
else:
files.append(os.path.basename(entry['path']))
return directories, files
def size(self, name):
cache_key = 'django-dropbox-size:%s' % filepath_to_uri(name)
size = cache.get(cache_key)
if not size:
size = self.client.metadata(filepath_to_uri(name))['bytes']
cache.set(cache_key, size, CACHE_TIMEOUT)
return size
def url(self, name):
cache_key = 'django-dropbox-url:%s' % filepath_to_uri(name)
url = cache.get(cache_key)
if not url:
url = self.client.share(filepath_to_uri(name), short_url=False)['url'] + '?dl=1'
cache.set(cache_key, url, CACHE_TIMEOUT)
return url
def get_available_name(self, name):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
name = self._get_abs_path(name)
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a number (before
# the file extension, if one exists) to the filename until the generated
# filename doesn't exist.
count = itertools.count(1)
while self.exists(name):
# file_ext includes the dot.
name = os.path.join(dir_name, "%s_%s%s" % (file_root, count.next(), file_ext))
return name
def upload(dropbox_helper_id, access_token, size, max_retries):
from .models import DropboxUploadHelper
helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id)
client = DropboxClient(access_token)
retries = 0
try:
with open(helper.src, 'rb') as f:
uploader = client.get_chunked_uploader(f, size)
while uploader.offset < size:
helper.progress = uploader.offset / size
helper.save()
try:
uploader.upload_chunked()
except ErrorResponse as e:
if retries < max_retries:
retries += 1
else:
helper.failure_reason = str(e)
helper.save()
raise e
upload = uploader.finish(helper.dest)
except Exception as e:
helper.failure_reason = str(e)
helper.save()
couch_user = CouchUser.get_by_username(helper.user.username)
if helper.failure_reason is None:
share = client.share(upload['path'])
context = {
'share_url':
share.get('url', None),
'path':
u'Apps/{app}{dest}'.format(
app=settings.DROPBOX_APP_NAME,
dest=upload['path'],
)
}
with localize(couch_user.get_language_code()):
subject = _(u'{} has been uploaded to dropbox!'.format(
helper.dest))
html_content = render_to_string(
'dropbox/emails/upload_success.html', context)
text_content = render_to_string(
'dropbox/emails/upload_success.txt', context)
else:
context = {'reason': helper.failure_reason, 'path': helper.dest}
with localize(couch_user.get_language_code()):
subject = _(u'{} has failed to upload to dropbox'.format(
helper.dest))
html_content = render_to_string('dropbox/emails/upload_error.html',
context)
text_content = render_to_string('dropbox/emails/upload_error.txt',
context)
send_HTML_email(
subject,
helper.user.email,
html_content,
text_content=text_content,
)
