def _upload(self, client: DropboxClient): """ Upload new files from directory. """ now = time() for filename in listdir(self.directory): if fnmatch(filename, '*.upl'): continue local_name = '/' + filename full_name = path.join(self.directory, filename) upl_name = "{}.upl".format(full_name) if not path.isfile(upl_name) and stat(full_name).st_mtime < now - 60: with open(full_name, 'rb') as file_stream: try: client.put_file(local_name, file_stream) share = client.share(local_name) except (MaxRetryError, ErrorResponse): continue with DatabaseConnection() as db: update = """ UPDATE events SET url = '{}', uploaded = current_timestamp WHERE file = '{}' """.format(share['url'], full_name) db.dml(update) try: mknod(upl_name) except FileExistsError: pass print("{} was uploaded to Dropbox.".format(filename))
def get_dropbox_share(request): if request.method == 'GET': user_profile = UserProfile.objects.get(user = request.user) drop_client = DropboxClient(user_profile.dropbox_profile.access_token['key']) res = drop_client.share(request.GET.get('path')) return redirect(res[u'url'])
def process_user(uid): '''Call /delta for the given user ID and process any changes.''' # OAuth token for the user token = redis_client.hget('tokens', uid) # /delta cursor for the user (None the first time) cursor = redis_client.hget('cursors', uid) client = DropboxClient(token) has_more = True while has_more: result = client.delta(cursor) for path, metadata in result['entries']: # Ignore deleted files, folders, and non-markdown files if (metadata is None or metadata['is_dir'] or not path.endswith('.md')): continue # Convert to Markdown and store as <basename>.html response, metadata = client.get_file_and_metadata(path) md = response.read().decode() html = markdown(md, extensions=['gfm']) html_name = path[:-3] + '.html' client.put_file(html_name, html, overwrite=True) # Include URL to published file in HTML comment at top of Markdown # file if '<!-- Published file url:' != md.split('\n')[0]: share_url = client.share(html_name, short_url=False).get('url') file_key = share_url.split('/')[4] url_name = urllib.parse.quote(html_name) url_comment = ('<!-- Published file url:\n' 'https://dl.dropboxusercontent.com/s/' '{}{}\n-->\n'.format(file_key, url_name)) md = url_comment + md client.put_file(path, md, overwrite=True) # Update cursor cursor = result['cursor'] redis_client.hset('cursors', uid, cursor) # Repeat only if there's more to do has_more = result['has_more']
class DropboxSyncClient: def __init__(self, oauth2_access_token): self.access_token = oauth2_access_token self._client = DropboxClient(oauth2_access_token) def upload_file(self, dropbox_file_path, local_file_path, replace=False): f = open(local_file_path, 'rb') response = self._client.put_file(dropbox_file_path, f, replace) return 1, response['path'] def generate_public_url(self, dropbox_file_path): return self._client.share(dropbox_file_path)['url'] def delete_file(self, dropbox_file_path): self._client.file_delete(dropbox_file_path) return 1, None def update_local_to_cloud(self, dropbox_file_path, local_file_path): return 1, self.upload_file(dropbox_file_path, local_file_path, replace=True) def update_cloud_to_local(self, dropbox_file_path, local_file_path): try: try: os.makedirs(os.path.dirname(local_file_path)) except Exception as e: pass open(local_file_path, 'wb').write(self._client.get_file(dropbox_file_path).read()) return 1, None except Exception as e: print e return 1, None def get_file_list(self, dropbox_folder_path): folder_metadata = self._client.metadata(dropbox_folder_path) return [content['path'] for content in folder_metadata['contents']] def set_access_token(self, access_token): self._client = DropboxClient(access_token) def get_remaining_space(self): quota_info = self._client.account_info()['quota_info'] return quota_info['total'] - (quota_info['shared'] + quota_info['normal'])
class DropboxStorage(object): calibre_db_path = '/%s/metadata.db' % settings.DROPBOX_CALIBRE_DIR dropbox_cursor_key = 'dropbox_cursor' def __init__(self): session = DropboxSession(settings.DROPBOX_CONSUMER_KEY, settings.DROPBOX_CONSUMER_SECRET, settings.DROPBOX_ACCESS_TYPE, locale=None) session.set_token(settings.DROPBOX_ACCESS_TOKEN, settings.DROPBOX_ACCESS_TOKEN_SECRET) self.client = DropboxClient(session) def get_url(self, path, share=False): try: if share: result = self.client.share(path, short_url=False) return result['url'] + '?dl=1' return self.client.media(path).get('url') except ErrorResponse: pass def get_file(self, path): try: return self.client.get_file(path) except ErrorResponse: pass def sync_db(self): calibre_db = self.client.get_file(self.calibre_db_path) with open(settings.DATABASES['calibre']['NAME'], 'wb') as f: f.write(calibre_db.read()) def need_update(self): delta = self.client.delta(cursor=cache.get(self.dropbox_cursor_key), path_prefix=self.calibre_db_path) cache.set(self.dropbox_cursor_key, delta['cursor'], timeout=None) return len(delta['entries']) > 0
def cluster_all_files(): #sign in dbx = dropbox.Dropbox( 'F2_PGWfw-GAAAAAAAAAACqNZoyJNzdLMd7x-BKLsGSE7hHM07KRMfT6jJgtWLgub') client = DropboxClient( 'F2_PGWfw-GAAAAAAAAAACqNZoyJNzdLMd7x-BKLsGSE7hHM07KRMfT6jJgtWLgub') #debug info newFileMeta = None cathegory_list = {} for entry in dbx.files_list_folder('').entries: if '.jpg' in entry.name: print(entry.name) newFileMeta = entry print(newFileMeta.path_lower) newFile = dbx.files_download_to_file('tmp/' + newFileMeta.name, newFileMeta.path_lower) cath = (DropboxUpdate.analyzeFile('tmp/' + newFileMeta.name) ['categories'][0]['name']) link = client.share(newFileMeta.path_lower)['url'] cathegory_list[link] = cath print cathegory_list with open('img_dat.txt', 'w') as outfile: json.dump(cathegory_list, outfile)
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox Public folder. """ def __init__(self, location='/Public'): session = DropboxSession(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TYPE, locale=None) session.set_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) self.client = DropboxClient(session) self.account_info = self.client.account_info() self.location = location self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format( **self.account_info) def _get_abs_path(self, name): return os.path.realpath(os.path.join(self.location, name)) def _open(self, name, mode='rb'): name = self._get_abs_path(name) remote_file = DropboxFile(name, self, mode=mode) return remote_file def _save(self, name, content): name = self._get_abs_path(name) directory = os.path.dirname(name) if not self.exists(directory) and directory: self.client.file_create_folder(directory) response = self.client.metadata(directory) if not response['is_dir']: raise IOError("%s exists and is not a directory." % directory) abs_name = os.path.realpath(os.path.join(self.location, name)) self.client.put_file(abs_name, content) return name def delete(self, name): name = self._get_abs_path(name) self.client.file_delete(name) def exists(self, name): name = self._get_abs_path(name) try: metadata = self.client.metadata(name) if metadata.get('is_deleted'): return False except ErrorResponse as e: if e.status == 404: # not found return False raise e return True def listdir(self, path): path = self._get_abs_path(path) response = self.client.metadata(path) directories = [] files = [] for entry in response.get('contents', []): if entry['is_dir']: directories.append(os.path.basename(entry['path'])) else: files.append(os.path.basename(entry['path'])) return directories, files def size(self, name): cache_key = 'django-dropbox-size:%s' % filepath_to_uri(name) size = cache.get(cache_key) if not size: size = self.client.metadata(filepath_to_uri(name))['bytes'] cache.set(cache_key, size, CACHE_TIMEOUT) return size def url(self, name): cache_key = 'django-dropbox-url:%s' % filepath_to_uri(name) url = cache.get(cache_key) if not url: url = self.client.share(filepath_to_uri(name), short_url=False)['url'] + '?dl=1' cache.set(cache_key, url, CACHE_TIMEOUT) return url def get_available_name(self, name): """ Returns a filename that's free on the target storage system, and available for new content to be written to. """ name = self._get_abs_path(name) dir_name, file_name = os.path.split(name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a number (before # the file extension, if one exists) to the filename until the generated # filename doesn't exist. count = itertools.count(1) while self.exists(name): # file_ext includes the dot. name = os.path.join( dir_name, "%s_%s%s" % (file_root, count.next(), file_ext)) return name
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox Public folder. """ def __init__(self, location=None): session = DropboxSession(DROPBOX.app_key, DROPBOX.app_secret, DROPBOX.access_type, locale=None) session.set_token(DROPBOX.access_key, DROPBOX.access_secret) self.client = DropboxClient(session) self.overwrite_mode = DROPBOX.overwrite_mode self._location = location @prepend_path_with_attr("_location") def delete(self, name): self.client.file_delete(name) @prepend_path_with_attr("_location") def exists(self, name): try: metadata = self.client.metadata(name, list=False) return not metadata.get('is_deleted') except ErrorResponse as e: if 404 == e.status: # not found return False raise e return True @prepend_path_with_attr("_location") def listdir(self, name, query_filter=None): if query_filter is None or len(query_filter) < 3: metadata = self.client.metadata(name) else: metadata = self.client.search(name, query_filter, file_limit=25000) directories = [] files = [] for entry in metadata.get('contents', []): if entry['is_dir']: directories.append(os.path.basename(entry['path'])) else: files.append(os.path.basename(entry['path'])) return directories, files @prepend_path_with_attr("_location") def open(self, name, mode='rb'): return DropboxFile(name, self, mode) @prepend_path_with_attr("_location") def save(self, name, content): metadata = self.client.put_file(name, content) return metadata['path'] @prepend_path_with_attr("_location") def size(self, name): return self.client.metadata(name, list=False)['bytes'] @prepend_path_with_attr("_location") def url(self, name): try: return self.client.share(name)['url'] except ErrorResponse as e: if 404 == e.status: # not found return None raise e @prepend_path_with_attr("_location") def get_available_name(self, name): """ Returns a filename that's free on the target storage system, and available for new content to be written to. """ if self.overwrite_mode: return name if self.exists(name): dir_name, file_name = os.path.split(name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a number (before # the file extension, if one exists) to the filename until the generated # filename doesn't exist. dir_contents = self.listdir(dir_name, file_root) count = itertools.count(1) while True: # file_ext includes the dot. name = "%s_%s%s" % (file_root, count.next(), file_ext) if name not in dir_contents: return os.path.join(dir_name, name) else: return name @prepend_path_with_attr("_location") def get_valid_name(self, name): return name
class DropboxStorage(Storage): """ A storage class providing access to resources in a Dropbox Public folder. """ def __init__(self, location='/Public'): session = DropboxSession(CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TYPE, locale=None) session.set_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET) self.client = DropboxClient(session) self.account_info = self.client.account_info() self.location = location self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format(**self.account_info) def _get_abs_path(self, name): return os.path.realpath(os.path.join(self.location, name)) def _open(self, name, mode='rb'): name = self._get_abs_path(name) remote_file = DropboxFile(name, self, mode=mode) return remote_file def _save(self, name, content): name = self._get_abs_path(name) directory = os.path.dirname(name) if not self.exists(directory) and directory: self.client.file_create_folder(directory) response = self.client.metadata(directory) if not response['is_dir']: raise IOError("%s exists and is not a directory." % directory) abs_name = os.path.realpath(os.path.join(self.location, name)) self.client.put_file(abs_name, content) return name def delete(self, name): name = self._get_abs_path(name) self.client.file_delete(name) def exists(self, name): name = self._get_abs_path(name) try: metadata = self.client.metadata(name) if metadata.get('is_deleted'): return False except ErrorResponse as e: if e.status == 404: # not found return False raise e return True def listdir(self, path): path = self._get_abs_path(path) response = self.client.metadata(path) directories = [] files = [] for entry in response.get('contents', []): if entry['is_dir']: directories.append(os.path.basename(entry['path'])) else: files.append(os.path.basename(entry['path'])) return directories, files def size(self, name): cache_key = 'django-dropbox-size:%s' % filepath_to_uri(name) size = cache.get(cache_key) if not size: size = self.client.metadata(filepath_to_uri(name))['bytes'] cache.set(cache_key, size, CACHE_TIMEOUT) return size def url(self, name): cache_key = 'django-dropbox-url:%s' % filepath_to_uri(name) url = cache.get(cache_key) if not url: url = self.client.share(filepath_to_uri(name), short_url=False)['url'] + '?dl=1' cache.set(cache_key, url, CACHE_TIMEOUT) return url def get_available_name(self, name): """ Returns a filename that's free on the target storage system, and available for new content to be written to. """ name = self._get_abs_path(name) dir_name, file_name = os.path.split(name) file_root, file_ext = os.path.splitext(file_name) # If the filename already exists, add an underscore and a number (before # the file extension, if one exists) to the filename until the generated # filename doesn't exist. count = itertools.count(1) while self.exists(name): # file_ext includes the dot. name = os.path.join(dir_name, "%s_%s%s" % (file_root, count.next(), file_ext)) return name
def upload(dropbox_helper_id, access_token, size, max_retries): from .models import DropboxUploadHelper helper = DropboxUploadHelper.objects.get(id=dropbox_helper_id) client = DropboxClient(access_token) retries = 0 try: with open(helper.src, 'rb') as f: uploader = client.get_chunked_uploader(f, size) while uploader.offset < size: helper.progress = uploader.offset / size helper.save() try: uploader.upload_chunked() except ErrorResponse as e: if retries < max_retries: retries += 1 else: helper.failure_reason = str(e) helper.save() raise e upload = uploader.finish(helper.dest) except Exception as e: helper.failure_reason = str(e) helper.save() couch_user = CouchUser.get_by_username(helper.user.username) if helper.failure_reason is None: share = client.share(upload['path']) context = { 'share_url': share.get('url', None), 'path': u'Apps/{app}{dest}'.format( app=settings.DROPBOX_APP_NAME, dest=upload['path'], ) } with localize(couch_user.get_language_code()): subject = _(u'{} has been uploaded to dropbox!'.format( helper.dest)) html_content = render_to_string( 'dropbox/emails/upload_success.html', context) text_content = render_to_string( 'dropbox/emails/upload_success.txt', context) else: context = {'reason': helper.failure_reason, 'path': helper.dest} with localize(couch_user.get_language_code()): subject = _(u'{} has failed to upload to dropbox'.format( helper.dest)) html_content = render_to_string('dropbox/emails/upload_error.html', context) text_content = render_to_string('dropbox/emails/upload_error.txt', context) send_HTML_email( subject, helper.user.email, html_content, text_content=text_content, )