def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or metadata['is_dir']
or not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
html = markdown(client.get_file(path).read())
client.put_file(path[:-3] + '.html', html, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def decryptfile():
print "Decrypting file"
# To avoid writing plaintext to any files, copy the response
# data from Dropbox directly into a read/write in memory buffer
if request.method == 'POST':
infile = request.form['filename']
bfirmid = base64.b64encode(request.form.get('firmid'))
bclientid = base64.b64encode(request.form.get('clientid'))
try:
access_token = config.get('Credentials','access_token')
dclient = DropboxClient(access_token)
print "Requesting /%s" % infile
httpresp = dclient.get_file("/%s" % infile)
instream = io.BytesIO()
inbuf = io.BufferedRandom(instream)
inbuf.write(httpresp.read())
inbuf.flush()
inbuf.seek(0)
result = getfile(inbuf,bfirmid,bclientid)
print "done with getfile"
result.seek(0,os.SEEK_END)
print "Got %d bytes" % result.tell()
result.seek(0)
# Copy the decrypted data into a HTTP response object
# to be returned to the user
print "getting ready to return to response object"
return Response(chunkfd(result,blocksize=4096),
mimetype='application/octet-stream')
except Exception as e:
print e
class DropBox(BaseDrive):
def __init__(self, token, rootPath):
BaseDrive.__init__(self, token, rootPath)
APP_KEY = '5a91csqjtsujuw7'
APP_SECRET = 'x5wbkk2o273jqz7'
session = DropboxSession(APP_KEY, APP_SECRET)
print token
access_key, access_secret = token.split(',')
session.set_token(access_key, access_secret)
first_client = DropboxClient(session)
token1 = first_client.create_oauth2_access_token()
self.client = DropboxClient(token1)
def ls(self, path):
folder_metadata = self.client.metadata(path)
contents = folder_metadata['contents']
files = []
for content in contents:
if content['is_dir']:
files.append(MyFile(content['path'], True))
else:
files.append(MyFile(content['path'], False))
return files
def get(self, myfile, temp_filename):
out = open(temp_filename, 'wb')
f = self.client.get_file(myfile.path)
out.write(f.read())
out.close()
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or
metadata['is_dir'] or
not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
html = markdown(client.get_file(path).read())
client.put_file(path[:-3] + '.html', html, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def get(self):
client = DropboxClient(self.get_secure_cookie("access_token"))
try:
with client.get_file('/blog.db') as f:
out = open('blog.db', 'wb')
out.write(f.read())
out.close()
with client.get_file('/settings.py') as f:
out = open('settings.py', 'wb')
out.write(f.read())
out.close()
except:
print("Can't find any backup")
self.redirect('/admin/dropbox?message=Failed to load backup. Please make sure you have a backup on Dropbox')
finally:
self.redirect('/admin/dropbox?message=Load backup successfully')
def downloadFile(self, file, path):
from dropbox.client import DropboxClient
log = SetLog()
status = Status()
# Usado para comprimir el archivo
zip = Compress()
# Extrae el nombre, la extension del archivo
dir, name = os.path.split(file)
# Archivo local donde se almacenara
localFile = os.path.join(path, name)
cliente = DropboxClient(self.TOKEN)
thread.start_new_thread(status.setDownloadstatus, (name, path, 1,))
with self.stopwatch('download'):
try:
out = open(localFile, 'wb')
with self.stopwatch("download"):
with cliente.get_file(file) as f:
out.write(f.read())
except dropbox.exceptions.HttpError as err:
log.newLog("error_download", "T", file)
return False
out.close()
if os.path.exists(localFile):
thread.start_new_thread(status.setDownloadstatus, (name, path, 2,))
zip.uncompress(localFile)
log.newLog("success_download", "T", file)
thread.start_new_thread(status.setDownloadstatus, (name, path, 0,))
return True
else:
log.newLog("error_download", "T", file)
return False
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def generate_url(self, name):
url = self.client.share(name, name)
url = url['url']
f = request.urlopen(url)
path = f.geturl()
return re.sub('www.dropbox.com','dl.dropboxusercontent.com', path)
def url(self, name):
return name #
def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = DropboxClient(oauth2_access_token)
def delete(self, name):
self.client.file_delete(name)
def exists(self, name):
response = self.client.search('/', name, file_limit=1)
return bool(response)
def listdir(self, path):
directories, files = [], []
metadata = self.client.metadata(path)
for entry in metadata['contents']:
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.metadata(name)
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.metadata(name)
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.metadata(name)
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(name, self)
return remote_file
def _save(self, name, content):
self.client.put_file(name, content)
return name
def _read(self, name, num_bytes=None):
data = self.client.get_file(name)
return data.read(num_bytes)
class DropboxFiles():
"""Simple class for downloading/uploading files from/upto Dropbox
It requires an already generated access_token which can be provided
as a string or as a local filepath to the file containing one
"""
class TokenNotProvidedError():
pass
def __init__(self, filepath=None, token=None):
if not (filepath or token):
raise TokenNotProvidedError('Access token is not provided')
if filepath:
with open(filepath) as token_file:
token = token_file.read().strip()
self.client = DropboxClient(token)
def download(self, dropbox_filepath, local_filepath=None):
"""
If there is a file at Dropbox with the given `dropbox_filepath`
the function returns a filepath of downloaded file
otherwise it returns `None`
"""
filename = posixpath.basename(dropbox_filepath)
if not local_filepath:
local_filepath = filename
elif os.path.isdir(local_filepath):
local_filepath = os.path.join(local_filepath, filename)
try:
with self.client.get_file(dropbox_filepath) as dropbox_file, \
open(local_filepath, 'wb') as local_file:
local_file.write(dropbox_file.read())
except ErrorResponse as e:
if e.status == 404:
return None
else:
raise
return local_filepath
def upload(self, local_filepath, dropbox_filepath=None):
filename = os.path.basename(local_filepath)
if not dropbox_filepath:
dropbox_filepath = posixpath.join('/', filename)
elif dropbox_filepath.endswith('/'): # dropbox_filepath is a dir
dropbox_filepath = posixpath.join(dropbox_filepath, filename)
with open(local_filepath, 'rb') as local_file:
self.client.put_file(dropbox_filepath, local_file, overwrite=True)
def get(self):
client = DropboxClient(self.get_secure_cookie("access_token"))
try:
with client.get_file('/blog.db') as f:
out = open('blog.db', 'wb')
out.write(f.read())
out.close()
with client.get_file('/settings.py') as f:
out = open('settings.py', 'wb')
out.write(f.read())
out.close()
except:
print("Can't find any backup")
self.redirect(
'/admin/dropbox?message=Failed to load backup. Please make sure you have a backup on Dropbox'
)
finally:
self.redirect('/admin/dropbox?message=Load backup successfully')
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = DropboxClient(oauth2_access_token)
def delete(self, name):
self.client.file_delete(name)
def exists(self, name):
try:
return bool(self.client.metadata(name))
except ErrorResponse:
return False
def listdir(self, path):
directories, files = [], []
metadata = self.client.metadata(path)
for entry in metadata['contents']:
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.metadata(name)
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.metadata(name)
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.metadata(name)
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.media(name)
return media['url']
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(name, self)
return remote_file
def _save(self, name, content):
self.client.put_file(name, content)
return name
def _read(self, name, num_bytes=None):
data = self.client.get_file(name)
return data.read(num_bytes)
def main():
logging.info('BEGIN')
access_token = get_access_token()
if access_token is not None:
client = DropboxClient(access_token)
account_info = client.account_info()
if os.path.exists(SERIES_FILE):
os.remove(SERIES_FILE)
out = open(SERIES_FILE, 'a+')
out.write(client.get_file('/series.csv').read())
out.close()
reader = csv.reader(open(SERIES_FILE, 'r'), delimiter=',')
reader.next()
for row in reader:
sid = row[0]
pid = row[1]
active = row[4]
if(active == 0):
continue
if(sid == "" and pid != ""):
logging.info("pid %s (no sid)", pid)
response = client.search('', pid)
if len(response) == 0:
logging.info("pid %s not found in Dropbox", pid)
download(pid, client)
else:
logging.info("pid %s already in Dropbox", pid)
continue
logging.info("sid %s", sid)
r = requests.get("http://www.bbc.co.uk/programmes/" + sid + "/episodes/player.json")
if(r.status_code == 404):
logging.info("404 Not Found")
continue
for episode in r.json()["episodes"]:
tpid = episode["programme"]["pid"]
if(pid != "" and tpid != pid):
continue;
logging.info("pid %s START", tpid)
response = client.search('', tpid)
if len(response) == 0:
logging.info("pid %s not found in Dropbox", tpid)
download(tpid, client)
continue
else:
logging.info("pid %s already in Dropbox", tpid)
continue
logging.info("sid %s END", sid)
os.remove(SERIES_FILE)
logging.info('END')
class DropboxFiles():
"""Simple class for downloading/uploading files from/upto Dropbox
It requires an already generated access_token which can be provided
as a string or as a local filepath to the file containing one
"""
class TokenNotProvidedError():
pass
def __init__(self, filepath=None, token=None):
if not (filepath or token):
raise TokenNotProvidedError('Access token is not provided')
if filepath:
with open(filepath) as token_file:
token = token_file.read().strip()
self.client = DropboxClient(token)
def download(self, dropbox_filepath, local_filepath=None):
"""
If there is a file at Dropbox with the given `dropbox_filepath`
the function returns a filepath of downloaded file
otherwise it returns `None`
"""
filename = posixpath.basename(dropbox_filepath)
if not local_filepath:
local_filepath = filename
elif os.path.isdir(local_filepath):
local_filepath = os.path.join(local_filepath, filename)
try:
with self.client.get_file(dropbox_filepath) as dropbox_file, \
open(local_filepath, 'wb') as local_file:
local_file.write(dropbox_file.read())
except ErrorResponse as e:
if e.status == 404:
return None
else:
raise
return local_filepath
def upload(self, local_filepath, dropbox_filepath=None):
filename = os.path.basename(local_filepath)
if not dropbox_filepath:
dropbox_filepath = posixpath.join('/', filename)
elif dropbox_filepath.endswith('/'): # dropbox_filepath is a dir
dropbox_filepath = posixpath.join(dropbox_filepath, filename)
with open(local_filepath, 'rb') as local_file:
self.client.put_file(dropbox_filepath, local_file, overwrite=True)
class DropboxStorage(object):
calibre_db_path = '/%s/metadata.db' % settings.DROPBOX_CALIBRE_DIR
dropbox_cursor_key = 'dropbox_cursor'
def __init__(self):
session = DropboxSession(settings.DROPBOX_CONSUMER_KEY,
settings.DROPBOX_CONSUMER_SECRET,
settings.DROPBOX_ACCESS_TYPE,
locale=None)
session.set_token(settings.DROPBOX_ACCESS_TOKEN,
settings.DROPBOX_ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
def get_url(self, path, share=False):
try:
if share:
result = self.client.share(path, short_url=False)
return result['url'] + '?dl=1'
return self.client.media(path).get('url')
except ErrorResponse:
pass
def get_file(self, path):
try:
return self.client.get_file(path)
except ErrorResponse:
pass
def sync_db(self):
calibre_db = self.client.get_file(self.calibre_db_path)
with open(settings.DATABASES['calibre']['NAME'], 'wb') as f:
f.write(calibre_db.read())
def need_update(self):
delta = self.client.delta(cursor=cache.get(self.dropbox_cursor_key),
path_prefix=self.calibre_db_path)
cache.set(self.dropbox_cursor_key, delta['cursor'], timeout=None)
return len(delta['entries']) > 0
def init():
"""Generate the html and upload to S3
"""
client = DropboxClient(DROPBOX_TOKEN)
root = client.metadata(DROPBOX_ROOT)
for f in root["contents"]:
# Ignore deleted files, folders, and non-markdown files
if f["is_dir"] or not f["path"].endswith(".md"):
continue
# Convert to Markdown and store as <basename>.html
html = markdown(client.get_file(f["path"]).read())
s3_upload(html, f["path"])
def fetch_dropbox_file(request):
user = request.user
try:
access_token = user.profile.access_token
except Profile.DoesNotExist:
messages.add_message(request, messages.ERROR,
"Your dropbox account needs to be linked!")
return redirect('/dropbox/')
client = DropboxClient(access_token)
path = request.GET['mdFile']
resp = {'data': client.get_file(path).read()}
return HttpResponse(json.dumps(resp))
class DropboxStorage(object):
calibre_db_path = '/%s/metadata.db' % settings.DROPBOX_CALIBRE_DIR
dropbox_cursor_key = 'dropbox_cursor'
def __init__(self):
session = DropboxSession(settings.DROPBOX_CONSUMER_KEY,
settings.DROPBOX_CONSUMER_SECRET,
settings.DROPBOX_ACCESS_TYPE, locale=None)
session.set_token(settings.DROPBOX_ACCESS_TOKEN,
settings.DROPBOX_ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
def get_url(self, path, share=False):
try:
if share:
result = self.client.share(path, short_url=False)
return result['url'] + '?dl=1'
return self.client.media(path).get('url')
except ErrorResponse:
pass
def get_file(self, path):
try:
return self.client.get_file(path)
except ErrorResponse:
pass
def sync_db(self):
calibre_db = self.client.get_file(self.calibre_db_path)
with open(settings.DATABASES['calibre']['NAME'], 'wb') as f:
f.write(calibre_db.read())
def need_update(self):
delta = self.client.delta(cursor=cache.get(self.dropbox_cursor_key),
path_prefix=self.calibre_db_path)
cache.set(self.dropbox_cursor_key, delta['cursor'], timeout=None)
return len(delta['entries']) > 0
def init():
"""Generate the html and upload to S3
"""
client = DropboxClient(DROPBOX_TOKEN)
root = client.metadata(DROPBOX_ROOT)
for f in root['contents']:
# Ignore deleted files, folders, and non-markdown files
if f['is_dir'] or not f['path'].endswith('.md'):
continue
# Convert to Markdown and store as <basename>.html
html = markdown(client.get_file(f['path']).read())
s3_upload(html, f['path'])
def get(self, **kwargs):
path = kwargs.get('path')
siteID = path[:path.index('/')]
mapping_obj = database.Mapping.all().filter('SiteID =', siteID)
mapping_obj = mapping_obj.get()
if not mapping_obj:
self.redirect('/#banner')
return
client = DropboxClient(mapping_obj.user.access_token)
content = 'no'
f = client.get_file(path)
content = f.read()
f.close()
self.response.out.write(content)
class DropboxStorage(Storage):
def __init__(self, *args, **kwargs):
self.client = DropboxClient(settings.DROPBOX_ACCESS_TOKEN)
self.location = kwargs.get('location', settings.MEDIA_ROOT)
def path(self, name):
return safe_join(self.location, name)
def created_time(self, name):
raise NotImplementedError
def exists(self, name):
try:
return isinstance(self.client.metadata(self.path(name)), dict)
except:
return False
def get_available_name(self, name):
raise NotImplementedError
def get_valid_name(self, name):
raise NotImplementedError
def listdir(self, path):
meta = self.client.metadata(self.path(path))
directories, files = [], []
for entry in meta['contents']:
name = os.path.basename(entry['path'])
if entry['is_dir']:
directories.append(name)
else:
files.append(name)
return (directories, files)
def modified_time(self, name):
raise NotImplementedError
def open(self, name, mode='rb'):
return self.client.get_file(self.path(name))
def save(self, name, content, max_length=None):
raise NotImplementedError
def size(self, name):
return self.client.metadata(self.path(name)).bytes
def url(self, name):
return self.client.media(self.path(name))['url']
class DropboxDataProvider():
"""
read and write files in a remote dropbox uing the dropbox API
"""
def __init__(self, app_key, app_secret, access_type, access_token, access_token_secret,
location='',):
session = DropboxSession(app_key, app_secret, access_type)
session.set_token(access_token, access_token_secret)
self.client = DropboxClient(session)
self.account_info = self.client.account_info()
self.location = location
self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format(**self.account_info)
def read(self, filename):
return self.client.get_file(filename).read()
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
filename, fileext = os.path.splitext(path)
# Ignore deleted files, folders, and non-python files
if (metadata is None or metadata['is_dir'] or fileext != '.py'
or '-disappointed' in filename):
continue
with client.get_file(path) as fin:
original_code = fin.read()
try:
formatted_code = FormatCode(original_code)
suffix = "reformed"
# Only reform heretical code
if original_code != formatted_code:
formatted_code = credit(formatted_code)
client.put_file(filename + '-reformed.py',
formatted_code,
overwrite=True)
except:
# code itself was somehow invalid.
with open('facepalm.py', 'rb') as facepalm:
client.put_file(filename + '-disappointed.py',
facepalm,
overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
filename, fileext = os.path.splitext(path)
# Ignore deleted files, folders, and non-python files
if (metadata is None or metadata['is_dir'] or fileext != '.py' or
'-disappointed' in filename or '-reformed' in filename):
continue
with client.get_file(path) as fin:
original_code = fin.read()
try:
formatted_code = FormatCode(original_code)
suffix = "reformed"
# Only reform heretical code
if original_code != formatted_code:
formatted_code = credit(formatted_code)
client.put_file(filename + '-reformed.py',
formatted_code,
overwrite=True)
except:
# code itself was somehow invalid.
with open('facepalm.py', 'rb') as facepalm:
client.put_file(filename + '-disappointed.py', facepalm,
overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
class DropboxSyncClient:
def __init__(self, oauth2_access_token):
self.access_token = oauth2_access_token
self._client = DropboxClient(oauth2_access_token)
def upload_file(self, dropbox_file_path, local_file_path, replace=False):
f = open(local_file_path, 'rb')
response = self._client.put_file(dropbox_file_path, f, replace)
return 1, response['path']
def generate_public_url(self, dropbox_file_path):
return self._client.share(dropbox_file_path)['url']
def delete_file(self, dropbox_file_path):
self._client.file_delete(dropbox_file_path)
return 1, None
def update_local_to_cloud(self, dropbox_file_path, local_file_path):
return 1, self.upload_file(dropbox_file_path, local_file_path, replace=True)
def update_cloud_to_local(self, dropbox_file_path, local_file_path):
try:
try:
os.makedirs(os.path.dirname(local_file_path))
except Exception as e:
pass
open(local_file_path, 'wb').write(self._client.get_file(dropbox_file_path).read())
return 1, None
except Exception as e:
print e
return 1, None
def get_file_list(self, dropbox_folder_path):
folder_metadata = self._client.metadata(dropbox_folder_path)
return [content['path'] for content in folder_metadata['contents']]
def set_access_token(self, access_token):
self._client = DropboxClient(access_token)
def get_remaining_space(self):
quota_info = self._client.account_info()['quota_info']
return quota_info['total'] - (quota_info['shared'] + quota_info['normal'])
class DropboxFile(File):
def __init__(self, file, name=None):
self.dropbox_metadata = file
name = name or os.path.basename(self.dropbox_metadata["path"])
super(DropboxFile, self).__init__(file, name=name)
self._size = self.dropbox_metadata["size"]
access_type = getattr(settings, "DROPBOX_ACCESS_TYPE", "app_folder")
dropbox_session = DropboxSession(settings.DROPBOX_APP_KEY,
settings.DROPBOX_APP_SECRET_KEY,
access_type)
dropbox_session.set_token(settings.DROPBOX_APP_ACCESS_TOKEN,
settings.DROPBOX_APP_ACCESS_TOKEN_SECRET)
self.dropbox_client = DropboxClient(dropbox_session)
def open(self, mode=None):
self.file = self.dropbox_client.get_file(self.dropbox_metadata["path"])
def close(self):
pass
class PiBox(object):
def __init__(self, dropbox_token, pi_box_root, delta_cursor_file=''):
self.token = dropbox_token
self.pi_box_root = pi_box_root
self.client = DropboxClient(dropbox_token)
self.delta_cursor_file = delta_cursor_file
self.delta_cursor = self._read_cursor()
def _full_local_path(self, path):
return self.pi_box_root + path
def _read_cursor(self):
try:
with open(self.delta_cursor_file, 'r') as f:
return f.read()
except IOError:
return None
def _save_cursor(self):
try:
with open(self.delta_cursor_file, 'w+') as f:
f.write(self.delta_cursor)
except IOError:
pass
def get_delta(self):
response = self.client.delta(cursor=self.delta_cursor)
self.delta_cursor = response['cursor']
self._save_cursor()
return response
def get_file(self, from_path):
output_file = open(self._full_local_path(from_path), 'w+b')
with self.client.get_file(from_path) as f:
output_file.write(f.read())
def make_local_directory(self, path):
local_path = self._full_local_path(path)
if not os.path.exists(local_path):
os.makedirs(local_path)
def update(init=False):
"""Generate the html and upload to S3 only for the files that have
changed
"""
if init:
cursor = None
else:
cursor = redis_client.get('cursor')
client = DropboxClient(DROPBOX_TOKEN)
has_more = True
while has_more:
result = client.delta(cursor=cursor, path_prefix=DROPBOX_ROOT)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or metadata['is_dir']
or not path.endswith('.md')):
continue
# Extract file name from full path
filename = parse_name(path)
# Convert to Markdown
html_raw = markdown(client.get_file(path).read())
html = add_template(html_raw)
# Upload the file to S3
s3_upload(html, filename)
# Update cursor
cursor = result['cursor']
redis_client.set('cursor', cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def update(init=False):
"""Generate the html and upload to S3 only for the files that have
changed
"""
if init:
cursor = None
else:
cursor = redis_client.get("cursor")
client = DropboxClient(DROPBOX_TOKEN)
has_more = True
while has_more:
result = client.delta(cursor=cursor, path_prefix=DROPBOX_ROOT)
for path, metadata in result["entries"]:
# Ignore deleted files, folders, and non-markdown files
if metadata is None or metadata["is_dir"] or not path.endswith(".md"):
continue
# Extract file name from full path
filename = parse_name(path)
# Convert to Markdown
html_raw = markdown(client.get_file(path).read())
html = add_template(html_raw)
# Upload the file to S3
s3_upload(html, filename)
# Update cursor
cursor = result["cursor"]
redis_client.set("cursor", cursor)
# Repeat only if there's more to do
has_more = result["has_more"]
def to_be_synced(file_path):
dc = DropboxClient(access_token)
# check if file exists on Dropbox
file_name = os.path.basename(file_path)
tmp_file = open(file_path + '.poom', 'wb+')
try:
with dc.get_file('/' + file_name) as f:
tmp_file.write(f.read())
f.close()
tmp_file.close()
logger.debug("Comparing files...")
# get dropbox file info
dr_file_data = dc.metadata('/' + file_name)
# get dropbox file last modified time in UTC
dr_time = parse(dr_file_data['modified'])
# get local file last modified time
file_localtime = parse(time.ctime(os.path.getmtime(file_path)))
# convert local time to utc
local_dt = get_localzone().localize(file_localtime, is_dst=None)
loc_time = local_dt.astimezone(pytz.utc)
# compare the last modified times
if dr_time > loc_time:
logger.debug("Newer version on dropbox")
os.remove(file_path)
logger.debug("Removed file from disk")
os.rename(file_path + '.poom', file_path)
else:
logger.debug("Newer version on local disk")
return False
return True
except dbrest.ErrorResponse as e:
if e.status == 404:
return False
def get_program(self):
# Must use Dropbox to get program files.
from dropbox.client import DropboxClient
from posixpath import join
import os
# Set up the Dropbox connection. Not sure how access_tokens will work
access_token = os.environ.get('access_token')
dropbox_dir = os.environ.get('dropbox_dir')
client = DropboxClient(access_token)
# If this is our first time with this file, set the program name and
# location.
self.program_location = join(
dropbox_dir,
'programs',
self.program_name
)
# Retrieve the REST object from Dropbox
prog_obj = client.get_file(self.program_location)
# Put the program file contents into an array for parsing
program_content = prog_obj.readlines()
# Send that stuff back.
return program_content
class DBoxClient(object):
def __init__(self):
self._logger = logging.getLogger(config.dpbox['logger']['name'])
self.cache_file = config.dpbox['cachefile']
self._token = None
self._load()
key, secret = decode_dropbox_key(config.dpbox['app']['encoded'])
self.session = DBoxSession(config.dpbox['app']['key'],
config.dpbox['app']['secret'],
access_type=config.dpbox['app']['access'])
if (self._token):
self.session.set_token(self._token[0], self._token[1])
else:
self._token = self.session.link()
self._save()
self.client = DropboxClient(self.session)
def reset(self):
self._logger.debug('[dpbox v%s] resetting local state' % (VERSION))
self._save()
def download(self, source, directory=''):
if len(directory) > 0 and directory[len(directory) - 1] != '/':
directory += '/'
self._logger.info(u'[dpbox v%s] FETCH %s -> %s' %
(VERSION, unicode(source), unicode(directory)))
self._download(source, directory)
def _download(self, source, directory):
try:
metadata = self.client.metadata(source)
self._logger.debug(u'metadata for %s' % source)
self._logger.debug(metadata)
except Exception as e:
self._logger.error('[dpbox v%s] error fetching file' % (VERSION))
self._logger.exception(e)
return # Will check later if we've got everything.
segs = metadata['path'].split('/')
directory += segs[len(segs) - 1]
if metadata['is_dir']:
try:
os.stat(directory)
except:
os.mkdir(directory)
for item in metadata['contents']:
self._download(item['path'], directory + '/')
else:
f = self.client.get_file(source)
print 'writing file to disc...'
destination = open(os.path.expanduser(directory.encode('utf-8')),
'wb')
destination.write(f.read())
destination.close()
print u"[rev %s] %s - '%s' downloaded" % (
metadata['revision'], metadata['size'], directory)
def upload(self, source, directory):
if len(directory) > 0 and directory[len(directory) - 1] != '/':
directory += '/'
segs = source.split('/')
directory += segs[len(segs) - 1]
f = open(source, 'rb')
print u'uploading file %s -> %s' % (source, directory)
response = self.client.put_file(directory.encode('utf-8'), f)
print "[rev %s] %s - '%s' uploaded" % (
response['revision'], response['size'], response['path'])
self._logger.debug('[dpbox v%s] upload response: %s' %
(VERSION, response))
def list(self, directory):
path = unicode(directory).encode('utf-8')
metadata = self.client.metadata(path)
self._logger.debug('[dpbox v%s] metadata: %s' % (VERSION, metadata))
print 'display content of ', metadata['path'], ', ', metadata['size']
for item in metadata['contents']:
if item['is_dir']:
print 'd ', item['path']
else:
print 'f ', item['path']
def infos(self, item, filename=''):
path = unicode(item).encode('utf-8')
metadata = self.client.metadata(path)
if len(filename) > 0:
with open(filename, 'w') as outfile:
json.dump(metadata, outfile)
else:
print metadata
def user(self, item=''):
infos = self.client.account_info()
self._logger.debug(u'[dpbox v%s] %s' % (VERSION, infos))
if len(item) > 0:
print item, ' = ', infos[item]
return
for name in infos:
space = ' ' * (20 - len(name))
if isinstance(infos[name], types.DictType):
print name, ':'
for key in infos[name]:
print ' -> ', key, ': ', infos[name][key]
else:
print name, space, infos[name]
def disconnect(self):
cachefile = os.path.expanduser(self.cache_file)
if os.path.exists(cachefile):
os.unlink(cachefile)
self.session.unlink()
print 'disconnected from service'
def _save(self):
with open(os.path.expanduser(self.cache_file), 'w') as f:
f.write(''.join([json.dumps(self._token), '\n']))
# f.write(''.join([json.dumps(self.remote_dir), '\n']))
def _load(self):
cachefile = os.path.expanduser(self.cache_file)
if not os.path.exists(cachefile):
self._logger.warn('[dpbox v%s] Cache file not found: %s' %
(VERSION, cachefile))
self.reset()
return
try:
with open(cachefile, 'r') as f:
dir_changed = False
try:
line = f.readline() # Token.
self._token = json.loads(line)
self._logger.debug('[dpbox v%s] loaded token' % (VERSION))
except Exception as e:
self._logger.warn('[dpbox v%s] can\'t load cache state' %
(VERSION))
self._logger.exception(e)
# try:
# line = f.readline() # Dropbox directory.
# directory = json.loads(line)
# if directory != self.remote_dir: # Don't use state.
# self._logger.info(u'remote dir changed "%s" -> "%s"' %
# (directory, self.remote_dir))
# dir_changed = True
# except Exception as e:
# self._logger.warn('can\'t load cache state')
# self._logger.exception(e)
if dir_changed:
return
except Exception as e:
self._logger.error('[dpbox v%s] error opening cache file' %
(VERSION))
self._logger.exception(e)
class DropPy:
def __init__(self, directory="/", key=None, secret=None, key_save="./", cursor=None):
"""Intialize a Dropbox connection, at directory specified or root.
string directory. Location to consider root, relative to Dropbox root.
string keysLoc. Location to store authentication json
Any exceptions during the authorization process are not caught.
See https://www.dropbox.com/developers/core/docs/python
"""
self.key = key
self.secret = secret
self.key_save = key_save
self._dropbox_date = "%a, %d %b %Y %H:%M:%S %z"
access_token = self._auth()
self.cursor = cursor
if directory[-1] != "/":
directory += "/"
self.directory = directory
self.client = DropboxClient(access_token)
def _auth(self):
"""Attempts to load an access token from key_save
If unavailable, will guide the user through authentication
"""
pathname = _path(self.key_save, ".droppy")
if path.exists(pathname):
try:
with open(pathname) as token:
access_token = token.read()
return access_token
except:
# If this fails for any reason, just have them reauth
pass
client = DropboxOAuth2FlowNoRedirect(self.key, self.secret)
auth_url = client.start()
print("Visit for authorization:\n{}".format(auth_url))
auth_code = input("Enter the authorization key: ")
access_token, user_id = client.finish(auth_code)
self._writeToken(access_token)
return access_token
def _writeToken(self, access_token=""):
"""Writes the access token to specified key location"""
pathname = _path(self.key_save, ".droppy")
with open(pathname, "w+") as token_file:
token_file.write(access_token)
def _get_dropbox_datetime(self, date_string):
return datetime.strptime(date_string, self._dropbox_date)
def _set_dropbox_datetime(self, datetime_obj):
return datetime_obj.strftime(self._dropbox_date)
def logout(self):
"""Destroys the current access token. The user will have to reauth."""
self.client.disable_access_token()
def account_info(self):
"""Returns account info such as quota, email and display name."""
return self.client.account_info()
def download(self, target, to="./", rev=None, start=None, length=None):
"""Downloads the current file to the specified, or local, directory
target
The path to the file that will be downloaded.
If the first character is the forward slash ("/"), it will ignore
the relative path of the DropPy instance, and instead begin from
the Dropbox root
to
The local directory to download the file to.
Defaults to current directory
rev
Optional previous rev value of the file to be downloaded.
start
Optional byte value from which to start downloading.
length
Optional length in bytes for partially downloading the file.
If length is specified but start is not, then the last length bytes
will be downloaded.
Raises
400: Bad request (may be due to many things; check e.error for details).
404: No file was found at the given path, or the file that was there was deleted.
200: Request was okay but response was malformed in some way.
"""
filename = target.split("/").pop()
target = _path(self.directory, target)
with open(_path(to, filename), "wb") as out:
with self.client.get_file(target, rev, start, length) as download:
out.write(download.read())
def upload_chunked(self, fd, to=None, length=None):
"""Creates a chunked uploader
If the file exists on the server, another is uploaded with (#) as a suffix
fd
File object from which the data will be sourced from
to
Optional path to upload to. Defaults to initialized directory
length
The number of bytes to upload. Defaults to full file.
"""
if length is None:
length = path.getsize(fd.name)
if to is None:
to = self.directory
to = _path(self.directory, to)
filename = path.split(fd.name)[1]
if length < 1:
self.client.put_file(_path(to, filename), fd)
else:
uploader = self.client.get_chunked_uploader(fd, length)
while uploader.offset < length:
uploader.upload_chunked()
uploader.finish(_path(to, filename))
def delta(self):
"""Retreive delta information from Dropbox.
Allows you to monitor for changes. First change, cursor of None, returns all
files.
Subsequent calls, with cursor provided by previous calls, will provide changed files
Returns all entries
"""
result = self.client.delta(self.cursor, _trim_d(self.directory))
self.cursor = result["cursor"]
entries = result["entries"]
while result["has_more"]:
result = self.client.delta(self.cursor, _trim_d(self.directory))
self.cursor = result["cursor"]
entries = entries + result["entries"]
return entries
def longpoll(self):
pass
def move(self, source, destination):
"""Moves a file from one place to another. Both source and destination are
relative to initalized folder, unless preceded by directory altering prefix
(eg. "/", "../", "../newFolder")
source
Origin of the file to move
destination
Place to move the file to
Raises
400: Bad request (may be due to many things; check e.error for details).
403: An invalid move operation was attempted (e.g. there is already a file
at the given destination, or moving a shared folder into a shared folder).
404: No file was found at given from_path.
503: User over storage quota.
"""
source = _path(self.directory, source)
destination = _path(self.directory, destination)
self.client.file_move(source, destination)
def get_remote_files(self, directory="", deleted=False):
remote_path = _path(self.directory, directory)
metadata = self.client.metadata(remote_path, include_deleted=deleted)
remote_files = metadata["contents"]
for item in remote_files:
if item["is_dir"]:
remote_files = remote_files + self.get_remote_files(item["path"], deleted)
return remote_files
def sync(self, local=getcwd(), deleted=False, hidden=False):
"""Syncs the local file system to the remote file system.
By default, will not delete any files that differ, only add new files
local
The local file directory to recusively sync with the remote.
Default ./
delete
Delete local files to keep in sync
Does not delete remote files as it may not be running 24/7, tracking deletions
Default False
hidden
Include hidden files in sync
Default False
"""
local = path.abspath(local)
if not path.isdir(local):
raise ValueError("sync requires local to be a directory.")
if local[-1] != "/":
local += "/"
local_files = {}
for item in list(Path(local).glob("**/*")):
if not hidden and ( search("/\.\w+", str(item)) \
or match("\.\w+", str(item)) ):
continue
local_files[str(item)[len(local):]] = {
"mtime": int(item.stat().st_mtime)
}
remote_files_meta = self.get_remote_files(deleted=deleted)
remote_files = {}
remote_dirs = []
for item in remote_files_meta:
isDeleted = "is_deleted" in item and item["is_deleted"]
i = item["path"]
mtime = self._get_dropbox_datetime(item["modified"])
if not hidden and (search("/\.\w+", str(i)) or match("\.\w+", str(i))):
continue
# Dropbox is not case sensitive, so make sure we preserve for local
if i.startswith(self.directory) or i.startswith(self.directory.lower()):
i = i[len(self.directory):]
if item["is_dir"]:
remote_dirs.append(i)
remote_files[i] = {
"mtime": int(mtime.strftime("%s")),
"deleted": isDeleted
}
download = sorted([item for item in remote_files if item not in \
local_files and not remote_files[item]["deleted"]])
delete = sorted([item for item in remote_files if item in \
local_files and remote_files[item]["deleted"]])
#upload = sorted([item for item in local_files if item not in remote_files])
#for item in upload:
# item_path = local / Path(item)
#
# if item_path.is_dir():
# self.client.file_create_folder(_path(self.directory, item))
# else:
# parts = [part for part in item_path.parts if part not in Path(local).parts]
# to = "/".join(parts[:-1])
# with open(str(item_path), "rb") as f:
# self.upload_chunked(f,to=to)
for item in download:
item_path = Path(item)
if item in remote_dirs:
mkdir(_path(local, item))
else:
parts = item_path.parts
to = _path(local, "/".join(parts[:-1]))
self.download(_path(self.directory + item), to=to)
for item in delete:
p = Path(local) / item
if p.is_dir():
rmtree( str(p) )
elif p.is_file():
remove( str(p) )
class DropboxStorage(Storage):
def __init__(self, config_section):
logging.info('init dropbox storage')
self.base_dir = config_section['base_dir']
self.app_key = config_section['app_key']
self.app_secret = config_section['app_secret']
self.user_token_fn = expanduser(config_section['user_token_file'])
token = self._read_token()
if token is None:
token = self._ask_user_to_auth()
self._write_token(token)
self.client = DropboxClient(token)
def get_file(self, source_fn, dest_fn):
logging.info('get file ' + source_fn + ' from dropbox to ' + dest_fn)
with open(dest_fn, 'wb') as dest_file:
with self.client.get_file(
os.path.join(self.base_dir, source_fn)) as source_file:
dest_file.write(source_file.read())
def put_file(self, source_fn, dest_fn):
logging.info('put file ' + source_fn + ' to dropbox ' + dest_fn)
with open(source_fn, 'rb') as source_file:
self.client.put_file(
os.path.join(self.base_dir, dest_fn), source_file)
def _init_dropbox_client(self, token):
logging.info('init dropbox client')
self.client = DropboxClient(token)
def _ask_user_to_auth(self):
logging.info('asking user for token...')
flow = DropboxOAuth2FlowNoRedirect(self.app_key, self.app_secret)
authorize_url = flow.start()
print(authorize_url)
code = input('> ').strip()
token, user = flow.finish(code)
return token
def _read_token(self):
logging.info('load user token')
if os.path.exists(self.user_token_fn):
with open(self.user_token_fn, 'r') as token_file:
return token_file.readline()
logging.warn('token not found')
return None
def _write_token(self, token):
logging.info('save user token')
with open(self.user_token_fn, 'w') as token_file:
token_file.write(token)
def get_fn(self, fn):
# very bad, very ugly
return self.base_dir + fn
class DropboxAPI(StorageAPI, AppendOnlyLog):
"dropbox@auth : dropbox.com account with auth info"
def __init__(self):
from params import AUTH_DIR
authdir = AUTH_DIR
self.auth_file = os.path.join(authdir, 'dropbox.auth')
try:
with open(self.auth_file, 'r') as file:
ACCESS_TOKEN = file.readline().rstrip()
USER_ID = file.readline().rstrip()
except IOError:
ACCESS_TOKEN, USER_ID = self._authorize()
self.client = DropboxClient(ACCESS_TOKEN)
def sid(self):
return util.md5("dropbox") % 10000
def copy(self):
return DropboxAPI()
def _authorize(self):
dbg.info('Request access token from Dropbox')
flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
# print 'Open auth url:', authorize_url
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
# Change to rely on browser
print(
"We need to authorize access to Dropbox. Please visit the following URL and authorize the access:"
)
print(authorize_url)
print("")
code = raw_input("Input the code you got: ").strip()
#code = #raw_input("Enter the authorization code here: ").strip()
access_token, user_id = flow.finish(code)
with open(self.auth_file, 'w') as file:
file.write(access_token + "\n")
file.write(user_id + "\n")
dbg.info('Authentication successful')
return (access_token, user_id)
# return: list of file paths
def listdir(self, path):
dic = self.client.metadata(path)
lst = map(lambda x: x["path"], dic["contents"])
lst = map(lambda x: x.split("/")[-1], lst)
return lst
def exists(self, path):
try:
dic = self.client.metadata(path)
if (dic.has_key("is_deleted") and dic["is_deleted"]): return False
return True
except:
return False
def get(self, path):
"""Get the file content
Args:
path: string
Returns:
content: string
"""
conn = self.client.get_file(path)
content = conn.read()
conn.close()
return content
def get_file_rev(self, path, rev):
# get file of a previous version with rev hash_id
content = None
try:
conn = self.client.get_file(path, rev=rev)
content = conn.read()
conn.close()
except ErrorResponse as detail:
#print "[get_file_rev] File doesn't exist", detail
return None
return content
def put(self, path, content):
"""Upload the file
Args:
path: string
content: string, size <= 4MB
Returns: None
"""
from dropbox.rest import ErrorResponse
strobj = StringIO(content)
try:
metadata = self.client.put_file(path,
strobj,
overwrite=False,
autorename=False)
except ErrorResponse as e:
if e.status == 409:
raise ItemAlreadyExists(e.status, e.reason)
else:
raise APIError(e.status, e.reason)
return True
def putdir(self, path):
self.client.file_create_folder(path)
def update(self, path, content):
"""Update the file
Args and returns same as put
"""
strobj = StringIO(content)
metadata = self.client.put_file(path, strobj, overwrite=True)
return True
def rm(self, path):
"""Delete the file
Args:
path: string
"""
self.client.file_delete(path)
def rmdir(self, path):
self.client.file_delete(path)
def metadata(self, path):
# only for file, not dir
_md = self.client.metadata(path)
md = {}
md['size'] = _md['bytes']
md['mtime'] = util.convert_time(_md['modified'])
return md
def delta(self, path=None, cursor=None):
resp = self.client.delta(cursor=cursor, path_prefix=path)
cursor = resp['cursor']
changes = []
for entry in resp['entries']:
event = {}
if entry[1]:
# we don't care about delete event
event['path'] = entry[0]
if entry[1]['is_dir']:
event['type'] = 'folder'
else:
event['type'] = 'file'
changes.append(event)
return cursor, changes
def poll(self, path=None, cursor=None, timeout=30):
# timeout max 480
import requests
import time
from error import PollError
beg_time = time.time()
end_time = beg_time + timeout
curr_time = beg_time
url = 'https://api-notify.dropbox.com/1/longpoll_delta'
params = {}
changes = []
if path:
path = util.format_path(path)
if not cursor:
cursor, _ = self.delta(path)
curr_time = time.time()
while True:
params['cursor'] = cursor
params['timeout'] = max(30, int(end_time -
curr_time)) # minimum 30 second
resp = requests.request('GET', url, params=params)
obj = resp.json()
if 'error' in obj:
raise PollError(resp.status_code, resp.text)
if obj['changes']:
cursor, _delta = self.delta(path, cursor)
changes.extend(_delta)
if changes:
break
curr_time = time.time()
if curr_time > end_time:
break
return cursor, changes
def init_log(self, path):
if not self.exists(path):
self.put(path, '')
def reset_log(self, path):
if self.exists(path):
self.rm(path)
def append(self, path, msg):
self.update(path, msg)
def get_logs(self, path, last_clock):
length = 5
# latest revision comes first
revisions = self.client.revisions(path, rev_limit=length)
if not revisions:
return [], None
new_logs = []
new_clock = revisions[0]['rev']
end = False # if reach to end
while True:
for metadata in revisions:
if last_clock and metadata['rev'] == last_clock:
end = True
break
if end: break
if len(revisions) < length: break
# still have logs unread, double the length
length *= 2
revisions = self.client.revisions(path, rev_limit=length)
# download the content of unseen rev
for metadata in revisions:
if last_clock and metadata['rev'] == last_clock:
break
if 'is_deleted' in metadata and metadata['is_deleted']: continue
msg = self.get_file_rev(path, metadata['rev'])
if len(msg) > 0:
new_logs.insert(0, msg)
return new_logs, new_clock
def __msg_index(self, fn):
return eval(fn[3:])
def init_log2(self, path):
if not self.exists(path):
self.putdir(path)
def append2(self, path, msg):
path = util.format_path(path)
lst = sorted(self.listdir(path))
if lst:
index = self.__msg_index(lst[-1]) + 1
else:
index = 0
while True:
fn = 'msg%d' % index
fpath = path + '/' + fn
try:
self.put(fpath, msg)
except ItemAlreadyExists:
index += 1
else:
break
def get_logs2(self, path, last_clock):
path = util.format_path(path)
lst = self.listdir(path)
if not lst:
return [], None
srt = {}
for fn in lst:
srt[self.__msg_index(fn)] = fn
lst = [srt[i] for i in sorted(srt.keys(), reverse=True)]
new_logs = []
new_clock = self.__msg_index(lst[0])
for fn in lst:
if last_clock == None and self.__msg_index(fn) == last_clock: break
msg = self.get(path + '/' + fn)
new_logs.insert(0, msg)
return new_logs, new_clock
def share(self, path, target_email):
url = "https://www.dropbox.com/"
print 'Get access token from Dropbox'
print 'Open auth url:', url
browser = webdriver.PhantomJS(
service_log_path=os.path.join(tempfile.gettempdir(),
'ghostdriver.log'),
service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
browser.get(url)
try:
wait = WebDriverWait(browser, 30)
btn = wait.until(
EC.element_to_be_clickable(
(By.XPATH, "//div[@id='sign-in']/a")))
btn.click()
email = wait.until(
EC.element_to_be_clickable(
(By.XPATH, "//input[@id='login_email']")))
email.send_keys(raw_input("Enter your Dropbox email:"))
pwd = browser.find_element_by_xpath(
"//input[@id='login_password']")
pwd.send_keys(getpass.getpass("Enter your Dropbox password:"******"//a[text()='%s']" % path)))
target_folder.click()
wait.until(EC.title_contains("%s" % path))
share_btn = browser.find_element_by_xpath(
"//a[@id='global_share_button']")
share_btn.click()
target = wait.until(
EC.element_to_be_clickable((
By.XPATH,
"//form[@class='invite-more-form']//input[@spellcheck][@type='text']"
)))
target.send_keys(target_email)
confirm_btn = browser.find_element_by_xpath(
"//form[@class='invite-more-form']//input[@type='button'][1]")
confirm_btn.click()
except:
print(browser.title)
assert False
# print(browser.current_url)
# print(browser.page_source)
pass
class DropboxAPI(StorageAPI, AppendOnlyLog):
"dropbox@auth : dropbox.com account with auth info"
def __init__(self):
from params import AUTH_DIR
authdir = AUTH_DIR
self.auth_file = os.path.join(authdir, 'dropbox.auth')
try:
with open(self.auth_file, 'r') as file:
ACCESS_TOKEN = file.readline().rstrip()
USER_ID = file.readline().rstrip()
except IOError:
ACCESS_TOKEN, USER_ID = self._authorize()
self.client = DropboxClient(ACCESS_TOKEN)
def sid(self):
return util.md5("dropbox") % 10000
def copy(self):
return DropboxAPI()
def _authorize(self):
dbg.info('Request access token from Dropbox')
flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
authorize_url = flow.start()
# print 'Open auth url:', authorize_url
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'))
#browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
# Change to rely on browser
print("We need to authorize access to Dropbox. Please visit the following URL and authorize the access:")
print(authorize_url)
print("")
code = raw_input("Input the code you got: ").strip()
#code = #raw_input("Enter the authorization code here: ").strip()
access_token, user_id = flow.finish(code)
with open(self.auth_file, 'w') as file:
file.write(access_token + "\n")
file.write(user_id + "\n")
dbg.info('Authentication successful')
return (access_token, user_id)
# return: list of file paths
def listdir(self, path):
dic = self.client.metadata(path)
lst = map(lambda x:x["path"], dic["contents"])
lst = map(lambda x:x.split("/")[-1], lst)
return lst
def exists(self, path):
try:
dic = self.client.metadata(path)
if(dic.has_key("is_deleted") and dic["is_deleted"]): return False
return True
except:
return False
def get(self, path):
"""Get the file content
Args:
path: string
Returns:
content: string
"""
conn = self.client.get_file(path)
content = conn.read()
conn.close()
return content
def get_file_rev(self, path, rev):
# get file of a previous version with rev hash_id
content = None
try:
conn = self.client.get_file(path, rev=rev)
content = conn.read()
conn.close()
except ErrorResponse as detail:
#print "[get_file_rev] File doesn't exist", detail
return None
return content
def put(self, path, content):
"""Upload the file
Args:
path: string
content: string, size <= 4MB
Returns: None
"""
from dropbox.rest import ErrorResponse
strobj = StringIO(content)
try:
metadata = self.client.put_file(path, strobj, overwrite=False, autorename=False)
except ErrorResponse as e:
if e.status == 409:
raise ItemAlreadyExists(e.status, e.reason)
else:
raise APIError(e.status, e.reason)
return True
def putdir(self, path):
self.client.file_create_folder(path)
def update(self, path, content):
"""Update the file
Args and returns same as put
"""
strobj = StringIO(content)
metadata = self.client.put_file(path, strobj, overwrite=True)
return True
def rm(self, path):
"""Delete the file
Args:
path: string
"""
self.client.file_delete(path)
def rmdir(self, path):
self.client.file_delete(path)
def metadata(self, path):
# only for file, not dir
_md = self.client.metadata(path)
md = {}
md['size'] = _md['bytes']
md['mtime'] = util.convert_time(_md['modified'])
return md
def delta(self, path=None, cursor=None):
resp = self.client.delta(cursor=cursor, path_prefix=path)
cursor = resp['cursor']
changes = []
for entry in resp['entries']:
event = {}
if entry[1]:
# we don't care about delete event
event['path'] = entry[0]
if entry[1]['is_dir']:
event['type'] = 'folder'
else:
event['type'] = 'file'
changes.append(event)
return cursor, changes
def poll(self, path=None, cursor=None, timeout=30):
# timeout max 480
import requests
import time
from error import PollError
beg_time = time.time()
end_time = beg_time + timeout
curr_time = beg_time
url = 'https://api-notify.dropbox.com/1/longpoll_delta'
params = {}
changes = []
if path:
path = util.format_path(path)
if not cursor:
cursor, _ = self.delta(path)
curr_time = time.time()
while True:
params['cursor'] = cursor
params['timeout'] = max(30, int(end_time - curr_time)) # minimum 30 second
resp = requests.request('GET', url, params=params)
obj = resp.json()
if 'error' in obj:
raise PollError(resp.status_code, resp.text)
if obj['changes']:
cursor, _delta = self.delta(path, cursor)
changes.extend(_delta)
if changes:
break
curr_time = time.time()
if curr_time > end_time:
break
return cursor, changes
def init_log(self, path):
if not self.exists(path):
self.put(path, '')
def reset_log(self, path):
if self.exists(path):
self.rm(path)
def append(self, path, msg):
self.update(path, msg)
def get_logs(self, path, last_clock):
length = 5
# latest revision comes first
revisions = self.client.revisions(path, rev_limit=length)
if not revisions:
return [], None
new_logs = []
new_clock = revisions[0]['rev']
end = False # if reach to end
while True:
for metadata in revisions:
if last_clock and metadata['rev'] == last_clock:
end = True
break
if end: break
if len(revisions) < length: break
# still have logs unread, double the length
length *= 2
revisions = self.client.revisions(path, rev_limit=length)
# download the content of unseen rev
for metadata in revisions:
if last_clock and metadata['rev'] == last_clock:
break
if 'is_deleted' in metadata and metadata['is_deleted']: continue
msg = self.get_file_rev(path, metadata['rev'])
if len(msg) > 0:
new_logs.insert(0, msg)
return new_logs, new_clock
def __msg_index(self, fn):
return eval(fn[3:])
def init_log2(self, path):
if not self.exists(path):
self.putdir(path)
def append2(self, path, msg):
path = util.format_path(path)
lst = sorted(self.listdir(path))
if lst:
index = self.__msg_index(lst[-1]) + 1
else:
index = 0
while True:
fn = 'msg%d' % index
fpath = path + '/' + fn
try:
self.put(fpath, msg)
except ItemAlreadyExists:
index += 1
else:
break
def get_logs2(self, path, last_clock):
path = util.format_path(path)
lst = self.listdir(path)
if not lst:
return [], None
srt = {}
for fn in lst:
srt[self.__msg_index(fn)] = fn
lst = [srt[i] for i in sorted(srt.keys(), reverse=True)]
new_logs = []
new_clock = self.__msg_index(lst[0])
for fn in lst:
if last_clock == None and self.__msg_index(fn) == last_clock: break
msg = self.get(path + '/' + fn)
new_logs.insert(0, msg)
return new_logs, new_clock
def share(self, path, target_email):
url = "https://www.dropbox.com/"
print 'Get access token from Dropbox'
print 'Open auth url:', url
browser = webdriver.PhantomJS(service_log_path=os.path.join(tempfile.gettempdir(), 'ghostdriver.log'), service_args=['--ignore-ssl-errors=true', '--ssl-protocol=tlsv1'])
browser.get(url)
try:
wait = WebDriverWait(browser, 30)
btn = wait.until(EC.element_to_be_clickable((By.XPATH, "//div[@id='sign-in']/a")))
btn.click()
email = wait.until(EC.element_to_be_clickable((By.XPATH, "//input[@id='login_email']")))
email.send_keys(raw_input("Enter your Dropbox email:"))
pwd = browser.find_element_by_xpath("//input[@id='login_password']")
pwd.send_keys(getpass.getpass("Enter your Dropbox password:"******"//a[text()='%s']" % path)))
target_folder.click()
wait.until(EC.title_contains("%s" % path))
share_btn = browser.find_element_by_xpath("//a[@id='global_share_button']")
share_btn.click()
target = wait.until(EC.element_to_be_clickable((By.XPATH, "//form[@class='invite-more-form']//input[@spellcheck][@type='text']")))
target.send_keys(target_email)
confirm_btn = browser.find_element_by_xpath("//form[@class='invite-more-form']//input[@type='button'][1]")
confirm_btn.click()
except:
print(browser.title)
assert False
# print(browser.current_url)
# print(browser.page_source)
pass
class DropBox(clouddrive.CloudDrive):
name = "DropBox"
def __init__(self, access_token):
self.client = DropboxClient(access_token)
self.access_token = access_token
clouddrive.CloudDrive.__init__(self, access_token)
@staticmethod
def auth():
#app_key = 'knbyx2adg14kkn5'
#app_secret = 'kh3ulgqry8jffqp'
app_key = 'eif0l7bgnpb06di'
app_secret = 'qa02jhdo4jrwaid'
global Auth_DROPMytoken
global Auth_Drop_Running_true
def keep_running():
global Auth_Drop_Running_true
return Auth_Drop_Running_true
class AuthHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_HEAD(s):
s.send_response(200)
s.send_header("Content-type", "text/html")
s.end_headers()
def do_GET(s):
s.send_response(200)
s.send_header("Content-type", "text/html")
s.end_headers()
if s.path.find("code=") != -1:
global Auth_DROPMytoken
global Auth_Drop_Running_true
Auth_DROPMytoken = s.path
Auth_Drop_Running_true = False
s.wfile.write("ok see command line")
return
class MYrequest:
def __init__(self, url):
self.url = url
def get(self, mystring):
if self.url.find(mystring) == -1:
return
item = self.url.split(mystring + "=")[1]
if item.find("&") != -1:
item = item.split("&")[0]
return urllib.unquote(item).decode()
redirect_url = "http://localhost:8787"
my_session = {}
Auth_Drop_Running_true = True
flow = DropboxOAuth2Flow(app_key, app_secret, redirect_url, my_session,
"dropbox-auth-csrf-token")
authorize_url = flow.start()
webbrowser.open(authorize_url)
try:
httpd = BaseHTTPServer.HTTPServer(("", 8787), AuthHandler)
while keep_running():
httpd.handle_request()
except KeyboardInterrupt:
pass
httpd.server_close()
token, user_id, url_state = flow.finish(MYrequest(Auth_DROPMytoken))
clouddrive.CloudDrive.access_token = token
return token
def read(self, filename):
try:
data = []
with self.client.get_file(filename) as f:
data += f.read()
except:
return -1
return ''.join(data)
def write(self, filename, data):
try:
response = self.client.put_file(filename, data, True)
except:
return -1
return 1
def mkdir(self, dirname):
try:
self.client.file_create_folder(dirname)
except:
return -1
return 1
def delete(self, filename):
try:
self.client.file_delete(filename)
except:
return -1
return 1
def capacity(self):
try:
a = self.client.account_info()["quota_info"]
except:
return -1
return a["quota"] - a["shared"] - a["normal"]
def listing(self, path="/"):
lists = self.client.metadata(path)
filelist = []
for x in lists["contents"]:
filelist.append((x['path'], x['is_dir']))
return filelist
class DropboxNoteProvider(RemoteNoteProvider):
__DAY_ONE_EXTENSION= ".doentry"
def __init__(self, accessToken, folder, proxyHost=None, proxyPort=None, proxyUser=None, proxyPassword=None):
self.__token= accessToken
self.__notesPath= folder + "/entries"
self.__removedNotesPath= self.__notesPath + "/deleted"
self.__photosPath= folder + "/photos"
self.__removedPhotosPath= self.__photosPath + "/deleted"
self.__client= DropboxClient(self.__token, rest_client=_restClient(proxyHost, proxyPort, proxyUser, proxyPassword))
self.__notesCache= {}
self.__dayOneFlavor= folder == SyncFolder.DayOne
def requiresReverseUpdate(self):
return True
@online
@expires
def sync(self):
notes= self.__list(self.__notesPath, False)
removedNotes= self.__list(self.__removedNotesPath, True)
#Clean inconsistent files
for uuid in list(notes.keys()):
if uuid in removedNotes:
if removedNotes[uuid].lastModified >= notes[uuid].lastModified:
self.__client.file_delete(self.__notePath(uuid))
if notes[uuid].hasPhoto:
self.__client.file_delete(self.__photoPath(uuid))
del notes[uuid]
else:
self.__client.file_delete(self.__removedNotePath(uuid))
del removedNotes[uuid]
notes.update(removedNotes)
self.__notesCache= notes
return notes
@online
@expires
def get(self, uuid):
response, metadata= self.__client.get_file_and_metadata(self.__notePath(uuid))
with response:
note= unmarshalNote(response.read(), getLastModified(metadata))
if uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
try:
with self.__client.get_file(self.__photoPath(uuid)) as response:
note.photo= response.read()
except ErrorResponse as e:
if e.status == 404: #Photo does not exist
pass
else:
raise e
return renderHtml(note)
@online
@expires
def add(self, note):
"""
The note lastModified time is updated from Dropbox when the operations succeeds (unfortunately there's no way to
set the modified time in Dropbox).
"""
uuid= note.uuid
result= self.__client.put_file(self.__notePath(uuid), marshalNote(note))
if len(result["path"]) != len(self.__notePath(uuid)):
try:
self.__client.file_delete(result["path"])
except ErrorResponse:
pass
raise RuntimeError("Note[uuid=%s] already exists" % uuid)
note.lastModified= getLastModified(result)
if note.photo:
self.__client.put_file(self.__photoPath(uuid), note.photo, overwrite=True)
elif uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
try:
self.__client.file_delete(self.__photoPath(uuid))
except ErrorResponse:
pass
renderHtml(note)
#Clean removed note if exists
if uuid in self.__notesCache and self.__notesCache[uuid].removed:
try:
self.__client.file_delete(self.__removedNotePath(uuid))
except ErrorResponse:
pass
@online
@expires
def update(self, note):
"""
The note lastModified time is updated from Dropbox when the operations succeeds (unfortunately there's no way to
set the modified time in Dropbox).
"""
uuid= note.uuid
#Check if note exists
if self.__notesCache and (uuid not in self.__notesCache or self.__notesCache[uuid].removed):
raise RuntimeError("Note[uuid=%s] does not exist" % uuid)
result= self.__client.put_file(self.__notePath(uuid), marshalNote(note), overwrite=True)
note.lastModified= getLastModified(result)
if note.photo:
self.__client.put_file(self.__photoPath(uuid), note.photo, overwrite=True)
elif uuid not in self.__notesCache or self.__notesCache[uuid].hasPhoto:
try:
self.__client.file_delete(self.__photoPath(uuid))
except ErrorResponse:
pass
renderHtml(note)
@online
@expires
def remove(self, note):
"""
The note lastModified time is updated from Dropbox when the operations succeeds (unfortunately there's no way to
set the modified time in Dropbox).
"""
uuid= note.uuid
#Remove note if exists
if uuid in self.__notesCache and not self.__notesCache[uuid].removed:
try:
self.__client.file_delete(self.__notePath(uuid))
except ErrorResponse:
pass
#Remove photo if exists
if uuid in self.__notesCache and self.__notesCache[uuid].hasPhoto:
try:
self.__client.file_delete(self.__photoPath(uuid))
except ErrorResponse:
pass
result= self.__client.put_file(self.__removedNotePath(uuid), b"", overwrite=True)
note.lastModified= getLastModified(result)
def __list(self, path, removed):
folder= self.__client.metadata(path)
if not folder["is_dir"]:
raise RuntimeError("Path is not a folder")
notes= {}
pos= len(path) + 1
for file in folder["contents"]:
if not file["is_dir"]:
name= self.__fileUuid(file["path"][pos:])
if isUuid(name):
notes[name]= NoteStatus(name, getLastModified(file), removed)
if not removed:
folder= self.__client.metadata(self.__photosPath)
if not folder["is_dir"]:
raise RuntimeError("Path is not a folder")
pos= len(self.__photosPath) + 1
for file in folder["contents"]:
name= file["path"][pos:]
if not file["is_dir"] and name.endswith(".jpg"):
name= name[:-4]
if name in notes:
notes[name].hasPhoto= True
return notes
def __fileUuid(self, filename):
if self.__dayOneFlavor and filename.endswith(self.__DAY_ONE_EXTENSION):
return filename[:-(len(self.__DAY_ONE_EXTENSION))]
return filename
def __buildNotePath(self, parentPath, uuid):
path= parentPath + "/" + uuid
if self.__dayOneFlavor:
path+= self.__DAY_ONE_EXTENSION
return path
def __notePath(self, uuid):
return self.__buildNotePath(self.__notesPath, uuid)
def __removedNotePath(self, uuid):
return self.__buildNotePath(self.__removedNotesPath, uuid)
def __photoPath(self, uuid):
return self.__photosPath + "/" + uuid + ".jpg"
class Dropbox(object):
""" Gather photos into a gallery from a Dropbox folder
"""
APP_KEY = 'dummyvalue'
APP_SECRET = 'dummyvalue'
ACCESS_TOKEN = 'dummyvalue'
USER_ID = 'dummyvalue'
def __init__(self):
self.client = DropboxClient(self.ACCESS_TOKEN)
def save_gallery(self, path):
""" Save a folder path as a gallery
:param path: Dropbox path containing the images
:type path: str
"""
if not isinstance(path, str):
raise TypeError('Path must be a string')
gal = models.Gallery.objects.get(title=path)
if gal:
raise KeyError('That gallery already exists: %s' % gal)
images = simplejson.loads(self.client.search('/%s' % path, '.jpg'))
if not images:
raise ValueError('Folder is empty')
gal = models.Gallery(title=path)
gal.save()
for image in images:
ntf = tempfile.NamedTemporaryFile(suffix='.%s' % models.get_extension(image['path']))
with self.client.get_file(image['path']) as dbf:
ntf.write(dbf.read())
img = models.Image(gallery_id=gal, path=files.File(ntf))
img.save()
def test_method(self):
lf = tempfile.NamedTemporaryFile(suffix='.%s' % models.get_extension('/08-12-2012_1432.jpg'))
with self.client.get_file('/08-12-2012_1432.jpg') as f:
lf.write(f.read())
im = models.Image(gallery_id=models.Gallery.objects.get(pk=1), path=files.File(lf))
im.save()
@staticmethod
def get_token():
""" Go through a no-redirect flow for an OAuth token
:returns: str
"""
from dropbox.client import DropboxOAuth2FlowNoRedirect
from dropbox import rest as dbrest
auth_flow = DropboxOAuth2FlowNoRedirect(Dropbox.APP_KEY, Dropbox.APP_SECRET)
authorize_url = auth_flow.start()
print('1. Go to: ' + authorize_url)
print('2. Click "Allow" (you might have to log in first).')
print('3. Copy the authorization code.')
auth_code = input('Enter the authorization code here: ').strip()
try:
access_token, user_id = auth_flow.finish(auth_code)
except dbrest.ErrorResponse as e:
print('Error: %s' % (e,))
return
print(access_token, user_id)
return access_token
class DBoxClient(object):
def __init__(self):
self._logger = logging.getLogger(config.dpbox['logger']['name'])
self.cache_file = config.dpbox['cachefile']
self._token = None
self._load()
key, secret = decode_dropbox_key(config.dpbox['app']['encoded'])
self.session = DBoxSession(config.dpbox['app']['key'], config.dpbox['app']['secret'], access_type=config.dpbox['app']['access'])
if(self._token):
self.session.set_token(self._token[0], self._token[1])
else:
self._token = self.session.link()
self._save()
self.client = DropboxClient(self.session)
def reset(self):
self._logger.debug('[dpbox v%s] resetting local state' % (VERSION))
self._save()
def download(self, source, directory=''):
if len(directory) > 0 and directory[len(directory)-1] != '/':
directory += '/'
self._logger.info(u'[dpbox v%s] FETCH %s -> %s' % (VERSION, unicode(source), unicode(directory)))
self._download(source, directory)
def _download(self, source, directory):
try:
metadata = self.client.metadata(source)
self._logger.debug(u'metadata for %s' % source)
self._logger.debug(metadata)
except Exception as e:
self._logger.error('[dpbox v%s] error fetching file' % (VERSION))
self._logger.exception(e)
return # Will check later if we've got everything.
segs = metadata['path'].split('/')
directory += segs[len(segs)-1]
if metadata['is_dir']:
try:
os.stat(directory)
except:
os.mkdir(directory)
for item in metadata['contents']:
self._download(item['path'], directory + '/')
else:
f = self.client.get_file(source)
print 'writing file to disc...'
destination = open(os.path.expanduser(directory.encode('utf-8')), 'wb')
destination.write(f.read())
destination.close()
print u"[rev %s] %s - '%s' downloaded" % (metadata['revision'], metadata['size'], directory)
def upload(self, source, directory):
if len(directory) > 0 and directory[len(directory)-1] != '/':
directory += '/'
segs = source.split('/')
directory += segs[len(segs)-1]
f = open(source, 'rb')
print u'uploading file %s -> %s' % (source, directory)
response = self.client.put_file(directory.encode('utf-8'), f)
print "[rev %s] %s - '%s' uploaded" % (response['revision'], response['size'], response['path'])
self._logger.debug('[dpbox v%s] upload response: %s' % (VERSION, response))
def list(self, directory):
path = unicode(directory).encode('utf-8')
metadata = self.client.metadata(path)
self._logger.debug('[dpbox v%s] metadata: %s' % (VERSION, metadata))
print 'display content of ', metadata['path'], ', ', metadata['size']
for item in metadata['contents']:
if item['is_dir']:
print 'd ', item['path']
else:
print 'f ', item['path']
def infos(self, item, filename=''):
path = unicode(item).encode('utf-8')
metadata = self.client.metadata(path)
if len(filename) > 0:
with open(filename, 'w') as outfile:
json.dump(metadata, outfile)
else:
print metadata
def user(self, item=''):
infos = self.client.account_info()
self._logger.debug(u'[dpbox v%s] %s' % (VERSION, infos))
if len(item) > 0:
print item, ' = ', infos[item]
return
for name in infos:
space = ' '*(20-len(name))
if isinstance(infos[name], types.DictType):
print name, ':'
for key in infos[name]:
print ' -> ', key, ': ', infos[name][key]
else:
print name, space, infos[name]
def disconnect(self):
cachefile = os.path.expanduser(self.cache_file)
if os.path.exists(cachefile):
os.unlink(cachefile)
self.session.unlink()
print 'disconnected from service'
def _save(self):
with open(os.path.expanduser(self.cache_file), 'w') as f:
f.write(''.join([json.dumps(self._token), '\n']))
# f.write(''.join([json.dumps(self.remote_dir), '\n']))
def _load(self):
cachefile = os.path.expanduser(self.cache_file)
if not os.path.exists(cachefile):
self._logger.warn('[dpbox v%s] Cache file not found: %s' % (VERSION, cachefile))
self.reset()
return
try:
with open(cachefile, 'r') as f:
dir_changed = False
try:
line = f.readline() # Token.
self._token = json.loads(line)
self._logger.debug('[dpbox v%s] loaded token' % (VERSION))
except Exception as e:
self._logger.warn('[dpbox v%s] can\'t load cache state' % (VERSION))
self._logger.exception(e)
# try:
# line = f.readline() # Dropbox directory.
# directory = json.loads(line)
# if directory != self.remote_dir: # Don't use state.
# self._logger.info(u'remote dir changed "%s" -> "%s"' %
# (directory, self.remote_dir))
# dir_changed = True
# except Exception as e:
# self._logger.warn('can\'t load cache state')
# self._logger.exception(e)
if dir_changed:
return
except Exception as e:
self._logger.error('[dpbox v%s] error opening cache file' % (VERSION))
self._logger.exception(e)
class DropboxDownloader:
def __init__(self, output_folder):
self.output_folder = output_folder
self.c = None
self.connect_to_dropbox()
def connect_to_dropbox(self):
"""
Connect to Dropbox, allowing us to use their API.
"""
auth_flow = DropboxOAuth2FlowNoRedirect("cmru2e8mi7ikbbf", "21417x86w06gpdh")
authorize_url = auth_flow.start()
print("1. Go to: " + authorize_url)
print("2. Click \"Allow\" (you might have to log in first).")
print("3. Copy the authorization code.")
auth_code = input("Enter the authorization code here: ").strip()
try:
access_token, user_id = auth_flow.finish(auth_code)
except dbrest.ErrorResponse as e:
print(('Error: %s' % (e,)))
return None
self.c = DropboxClient(access_token)
def _recursive_file_search(self, path, pattern):
"""
Searches recursively for files.
:param path: Path to search
:param pattern: Glob-style pattern (eg. *.tex)
:return:
"""
matches = []
for root, dirnames, filenames in os.walk(path):
for filename in fnmatch.filter(filenames, pattern):
matches.append(os.path.join(root, filename))
return matches
def download_revisions(self, filename, output_folder=None):
"""
Download all available revisions of the given filename (must be relative to the Dropbox root),
storing them in output_folder.
:param filename: Relative path to file inside the Dropbox folder
:param output_folder: Folder to download to - defaults to None, meaning it uses the class attribute
output_folder
"""
revs = self.c.revisions(filename)
for rev in revs:
print(rev)
revision_id = rev['rev']
mod_time = rev['client_mtime'].replace(" ", "_").replace(":", "").replace("+", "").replace(',', '')
if output_folder is None:
output_folder = self.output_folder
if not os.path.exists(output_folder):
os.mkdir(output_folder)
folder = os.path.join(output_folder, os.path.splitext(os.path.basename(filename))[0])
if not os.path.exists(folder):
os.mkdir(folder)
out_filename = os.path.join(folder, '%s.tex' % (mod_time))
if not os.path.exists(out_filename):
outfile = open(out_filename, 'wb')
with self.c.get_file(filename, rev=revision_id) as f:
outfile.write(f.read())
outfile.close()
else:
print("Already done, skipping")
def download_history_for_files(self, folder, globstring, dropbox_location, recursive=True):
"""
Download all available revisions for a given set of files.
:param folder: The full path to the Dropbox folder which contains the files you're interested in
:param globstring: The globstring (eg. *.txt) to use to select files
:param dropbox_location: The full path to the root of your Dropbox folder
:param recursive: Whether to search recursively (default) or not
"""
if recursive:
files = self._recursive_file_search(folder, globstring)
else:
files = glob(folder + globstring)
print(files)
for f in files:
print(f)
dropboxpath = os.path.relpath(f, dropbox_location).replace("\\", "/")
self.download_revisions(dropboxpath)
class WebSite(form_class, base_class):
def __init__(self, parent=None):
super(WebSite, self).__init__(parent)
self.setupUi(self)
self.addADSButton.setIcon(QtGui.QIcon(os.path.join(scriptdir, "images/plus_32.ico")))
self.removeADSButton.setIcon(QtGui.QIcon(os.path.join(scriptdir, "images/delete_32.ico")))
self.addAnimationsButton.setIcon(QtGui.QIcon(os.path.join(scriptdir, "images/plus_32.ico")))
self.removeAnimationsButton.setIcon(QtGui.QIcon(os.path.join(scriptdir, "images/delete_32.ico")))
self.transferButton.setIcon(QtGui.QIcon(os.path.join(scriptdir, "images/ftp_64.png")))
self.localDir = "F:/Documents/Enseignement/Corot/"
self.remoteDir = "F:/Documents/Enseignement/Test/"
os.chdir(self.localDir)
self.transferInfo = []
self.getLocalInfo()
self.client = DropboxClient('gS7qI3Fbn2AAAAAAAAAAFqW6G3O73LJxk9yZutwCO5Q2RT-bbYVdbbz-EskRLiQj')
self.getRemoteInfo()
self.fillForms()
def getRemoteInfo(self):
try:
with self.client.get_file('data.json') as f:
self.remoteInfo = json.loads(f.read().decode('utf-8'))
except:
self.remoteInfo = []
def getLocalInfo(self):
self.localInfo = []
for name in sorted(os.listdir('DS')):
if 'DS' in name and os.path.isdir('DS/' + name):
self.localInfo.append({'nom': name, 'type': 'DS', 'enoncepath': 'DS/' + name + '/' + name + '.pdf',
'corrigepath': 'DS/' + name + '/' + name + '_corrige.pdf'})
for name in sorted(os.listdir('DM')):
if 'DM' in name and os.path.isdir('DM/' + name):
self.localInfo.append({'nom': name, 'type': 'DM', 'enoncepath': 'DM/' + name + '/' + name + '.pdf',
'corrigepath': 'DM/' + name + '/' + name + '_corrige.pdf'})
for name in sorted(os.listdir('Cours')):
if os.path.isdir('Cours/' + name):
s = read(self.localDir + "Cours/" + name + "/" + name + ".tex")
titre = re.search(r"\\titrecours{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append({'nom': titre, 'type': 'cours', 'path': 'Cours/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('Formulaires')):
if os.path.isdir('Formulaires/' + name):
s = read(self.localDir + "Formulaires/" + name + "/" + name + ".tex")
titre = re.search(r"\\titreformulaire{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append(
{'nom': titre, 'type': 'formulaire', 'path': 'Formulaires/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('Colles')):
if 'ProgColles' in name and os.path.isdir('Colles/' + name):
self.localInfo.append({'nom': name, 'type': 'colle', 'path': 'Colles/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('Interros')):
if 'Interro' in name and os.path.isdir('Interros/' + name):
self.localInfo.append(
{'nom': name, 'type': 'interro', 'path': 'Interros/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('TD')):
if os.path.isdir('TD/' + name):
s = read(self.localDir + "TD/" + name + "/" + name + ".tex")
titre = re.search(r"\\titretd{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append({'nom': titre, 'type': 'TD', 'enoncepath': 'TD/' + name + '/' + name + '.pdf',
'corrigepath': 'TD/' + name + '/' + name + '_corrige.pdf'})
return
for name in sorted(os.listdir('Info')):
if os.path.isdir('Info/' + name):
s = read(self.localDir + "Info/" + name + "/" + name + ".tex");
titre = re.search(r"\\titrecours{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append({'nom': titre, 'type': 'info', 'path': 'Info/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('SlidesInfo')):
if os.path.isdir('SlidesInfo/' + name):
s = read(self.localDir + "SlidesInfo/" + name + "/" + name + ".tex");
titre = re.search(r"\\title{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append(
{'nom': titre, 'type': 'slidesinfo', 'path': 'SlidesInfo/' + name + '/' + name + '.pdf'})
for name in sorted(os.listdir('TDInfo')):
if os.path.isdir('TDInfo/' + name):
s = read(self.localDir + "TDInfo/" + name + "/" + name + ".tex")
titre = re.search(r"\\titretd{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append(
{'nom': titre, 'type': 'TDinfo', 'enoncepath': 'TDInfo/' + name + '/' + name + '.pdf',
'corrigepath': 'TDInfo/' + name + '/' + name + '_corrige.pdf'})
for name in sorted(os.listdir('TPInfo')):
if os.path.isdir('TPInfo/' + name):
s = read(self.localDir + "TPInfo/" + name + "/" + name + ".tex")
titre = re.search(r"\\titretd{(.*?)}", s, re.DOTALL).group(1).replace('\\\\', ' ')
self.localInfo.append(
{'nom': titre, 'type': 'TPinfo', 'enoncepath': 'TPInfo/' + name + '/' + name + '.pdf',
'corrigepath': 'TPInfo/' + name + '/' + name + '_corrige.pdf'})
for name in sorted(os.listdir('DSInfo')):
if 'DSInfo' in name and os.path.isdir('DSInfo/' + name):
self.localInfo.append(
{'nom': name, 'type': 'DSinfo', 'enoncepath': 'DSInfo/' + name + '/' + name + '.pdf',
'corrigepath': 'DSInfo/' + name + '/' + name + '_corrige.pdf'})
def clearForms(self):
for form in (
self.formDS, self.formDM, self.formColles, self.formCours, self.formInterros, self.formTD,
self.formInfo,
self.formTDInfo, self.formTPInfo, self.formADS, self.formFormulaires):
if form is not None:
while form.count():
item = form.takeAt(0)
widget = item.widget()
if widget is not None:
widget.deleteLater()
def fillForms(self):
n = next((item for item in self.remoteInfo if item['type'] == 'Colloscope'), {'type': 'Colloscope'})
self.formVieclasse.addWidget(FileWidget(n.copy(), self))
n = next((item for item in self.remoteInfo if item['type'] == 'Emploi du temps'), {'type': 'Emploi du temps'})
self.formVieclasse.addWidget(FileWidget(n.copy(), self))
n = next((item for item in self.remoteInfo if item['type'] == 'Planning des DS'), {'type': 'Planning des DS'})
self.formVieclasse.addWidget(FileWidget(n.copy(), self))
n = next((item for item in self.remoteInfo if item['type'] == 'Notes premier semestre'),
{'type': 'Notes premier semestre'})
self.formVieclasse.addWidget(FileWidget(n.copy(), self))
n = next((item for item in self.remoteInfo if item['type'] == 'Notes second semestre'),
{'type': 'Notes second semestre'})
self.formVieclasse.addWidget(FileWidget(n.copy(), self))
for ds in [e for e in self.localInfo if e['type'] == 'DS']:
self.formDS.addWidget(EnonceCorrigeWidget(ds, self))
for dm in [e for e in self.localInfo if e['type'] == 'DM']:
self.formDM.addWidget(EnonceCorrigeWidget(dm, self))
for i, cours in enumerate([e for e in self.localInfo if e['type'] == 'cours']):
self.formCours.addWidget(CoursWidget(cours, self), i / 4, i % 4)
for i, formulaire in enumerate([e for e in self.localInfo if e['type'] == 'formulaire']):
self.formFormulaires.addWidget(CoursWidget(formulaire, self), i / 4, i % 4)
for i, colle in enumerate([e for e in self.localInfo if e['type'] == 'colle']):
self.formColles.addWidget(CoursWidget(colle, self), i / 4, i % 4)
for i, interro in enumerate([e for e in self.localInfo if e['type'] == 'interro']):
self.formInterros.addWidget(CoursWidget(interro, self), i / 4, i % 4)
for td in [e for e in self.localInfo if e['type'] == 'TD']:
self.formTD.addWidget(EnonceCorrigeWidget(td, self))
for i, info in enumerate([e for e in self.localInfo if e['type'] == 'info']):
self.formInfo.addWidget(CoursWidget(info, self), i / 4, i % 4)
for i, slidesinfo in enumerate([e for e in self.localInfo if e['type'] == 'slidesinfo']):
self.formSlidesInfo.addWidget(CoursWidget(slidesinfo), i / 4, i % 4)
for tdinfo in [e for e in self.localInfo if e['type'] == 'TDinfo']:
self.formTDInfo.addWidget(EnonceCorrigeWidget(tdinfo, self))
for tpinfo in [e for e in self.localInfo if e['type'] == 'TPinfo']:
self.formTPInfo.addWidget(EnonceCorrigeWidget(tpinfo, self))
for dsinfo in [e for e in self.localInfo if e['type'] == 'DSinfo']:
self.formDSInfo.addWidget(EnonceCorrigeWidget(dsinfo, self))
for animation in [e for e in self.remoteInfo if e['type'] == 'animation']:
self.formAnimations.addWidget(AnimationWidget(animation, self))
def updateMessage(self, mess):
self.detailedMessage += mess + '\n'
self.message.setDetailedText(self.detailedMessage)
@QtCore.pyqtSlot()
def on_addADSButton_clicked(self):
ads = {'nom': '', 'eleve': '', 'type': 'ads', 'date': '', 'path': ''}
self.formADS.addWidget(ADSWidget(ads, self))
@QtCore.pyqtSlot()
def on_removeADSButton_clicked(self):
for widget in self.findChildren(ADSWidget):
if widget.check.isChecked():
self.formADS.removeWidget(widget)
widget.deleteLater()
self.formADS.update()
@QtCore.pyqtSlot()
def on_addAnimationsButton_clicked(self):
animation = {'nom': '', 'lien': '', 'type': 'animation', 'icon': ''}
self.formAnimations.addWidget(AnimationWidget(animation, self))
@QtCore.pyqtSlot()
def on_removeAnimationsButton_clicked(self):
for widget in self.findChildren(AnimationWidget):
if widget.check.isChecked():
self.formAnimations.removeWidget(widget)
widget.deleteLater()
self.formAnimations.update()
@QtCore.pyqtSlot()
def on_transferButton_clicked(self):
self.message = QtWidgets.QMessageBox(QtWidgets.QMessageBox.Information, "Transfert", "Transfert",
QtWidgets.QMessageBox.Cancel)
spacer = QtWidgets.QSpacerItem(500, 0, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding)
l = self.message.layout()
l.addItem(spacer, l.rowCount(), 0, 1, l.columnCount())
self.detailedMessage = 'Début du transfert\n'
self.message.setDetailedText(self.detailedMessage)
self.message.show()
transferFiles = [e['path'] for e in self.transferInfo if 'path' in e.keys()]
transferFiles += [e['enoncepath'] for e in self.transferInfo if 'enoncepath' in e.keys()]
transferFiles += [e['corrigepath'] for e in self.transferInfo if 'corrigepath' in e.keys()]
remoteFiles = [e['path'] for e in self.remoteInfo if 'path' in e.keys()]
remoteFiles += [e['enoncepath'] for e in self.remoteInfo if 'enoncepath' in e.keys()]
remoteFiles += [e['corrigepath'] for e in self.remoteInfo if 'corrigepath' in e.keys()]
copyFiles = [f for f in transferFiles if f not in remoteFiles]
updateFiles = [f for f in transferFiles if f in remoteFiles]
deleteFiles = [f for f in remoteFiles if f not in transferFiles]
self.thread = TransferThread(self.client, self.localDir, copyFiles, updateFiles, deleteFiles, self.transferInfo)
self.thread.start()
self.thread.message.connect(self.updateMessage)
class DropboxDownloader:
def __init__(self, output_folder):
self.output_folder = output_folder
self.c = None
self.connect_to_dropbox()
def connect_to_dropbox(self):
"""
Connect to Dropbox, allowing us to use their API.
"""
auth_flow = DropboxOAuth2FlowNoRedirect("cmru2e8mi7ikbbf",
"21417x86w06gpdh")
authorize_url = auth_flow.start()
print("1. Go to: " + authorize_url)
print("2. Click \"Allow\" (you might have to log in first).")
print("3. Copy the authorization code.")
auth_code = input("Enter the authorization code here: ").strip()
try:
access_token, user_id = auth_flow.finish(auth_code)
except dbrest.ErrorResponse as e:
print(('Error: %s' % (e, )))
return None
self.c = DropboxClient(access_token)
def _recursive_file_search(self, path, pattern):
"""
Searches recursively for files.
:param path: Path to search
:param pattern: Glob-style pattern (eg. *.tex)
:return:
"""
matches = []
for root, dirnames, filenames in os.walk(path):
for filename in fnmatch.filter(filenames, pattern):
matches.append(os.path.join(root, filename))
return matches
def download_revisions(self, filename, output_folder=None):
"""
Download all available revisions of the given filename (must be relative to the Dropbox root),
storing them in output_folder.
:param filename: Relative path to file inside the Dropbox folder
:param output_folder: Folder to download to - defaults to None, meaning it uses the class attribute
output_folder
"""
revs = self.c.revisions(filename)
for rev in revs:
print(rev)
revision_id = rev['rev']
mod_time = rev['client_mtime'].replace(" ", "_").replace(
":", "").replace("+", "").replace(',', '')
if output_folder is None:
output_folder = self.output_folder
if not os.path.exists(output_folder):
os.mkdir(output_folder)
folder = os.path.join(
output_folder,
os.path.splitext(os.path.basename(filename))[0])
if not os.path.exists(folder):
os.mkdir(folder)
out_filename = os.path.join(folder, '%s.tex' % (mod_time))
if not os.path.exists(out_filename):
outfile = open(out_filename, 'wb')
with self.c.get_file(filename, rev=revision_id) as f:
outfile.write(f.read())
outfile.close()
else:
print("Already done, skipping")
def download_history_for_files(self,
folder,
globstring,
dropbox_location,
recursive=True):
"""
Download all available revisions for a given set of files.
:param folder: The full path to the Dropbox folder which contains the files you're interested in
:param globstring: The globstring (eg. *.txt) to use to select files
:param dropbox_location: The full path to the root of your Dropbox folder
:param recursive: Whether to search recursively (default) or not
"""
if recursive:
files = self._recursive_file_search(folder, globstring)
else:
files = glob(folder + globstring)
print(files)
for f in files:
print(f)
dropboxpath = os.path.relpath(f,
dropbox_location).replace("\\", "/")
self.download_revisions(dropboxpath)
def sync_posts(author_id):
from posts.models import Author, Post
author = Author.objects.get(pk=author_id)
if (
cache.get(author.sync_cache_key) is None or
cache.get(author.sync_start_time_cache_key) is None or
cache.get(author.sync_start_time_cache_key) + MAX_SYNC_TIMEOUT < datetime.datetime.now()
):
try:
cache.set(author.sync_cache_key, True)
cache.set(author.sync_start_time_cache_key, datetime.datetime.now())
cache.set(author.sync_total_key, "~%s" % cache.get(author.sync_total_key, "0"))
cache.set(author.sync_current_key, 0)
if author.dayone_valid:
client = DropboxClient(author.dropbox_access_token)
full_dayone_entry_path = "%s/entries" % author.dropbox_dayone_folder_path
full_dayone_image_path = "%s/photos" % author.dropbox_dayone_folder_path
file_list = client.metadata(full_dayone_entry_path)
image_list = client.metadata(full_dayone_image_path)
cache.set(author.sync_total_key, len(file_list["contents"]))
count = 0
for f in file_list["contents"]:
do_update = False
dayone_id = f["path"].split("/")[-1]
cache.set(author.sync_current_key, count)
exists = False
if Post.objects.filter(dayone_id=dayone_id).count() > 0:
exists = True
p = Post.objects.get(dayone_id=dayone_id)
if p.dayone_last_rev != f["revision"]:
dayone_update_time = datetime_from_utc_to_local(datetime.datetime(*time.strptime(f["modified"], '%a, %d %b %Y %H:%M:%S +0000')[:6]))
if dayone_update_time > p.updated_at:
do_update = True
if not do_update:
image = get_matching_image_meta_if_exists(dayone_id, image_list)
if image:
image_update_time = datetime_from_utc_to_local(datetime.datetime(*time.strptime(image["modified"], '%a, %d %b %Y %H:%M:%S +0000')[:6]))
if image_update_time > p.updated_at:
do_update = True
else:
do_update = True
if do_update:
if not cache.get(author.sync_cache_key):
print "Interrupted."
break;
plist = None
with client.get_file(f["path"]) as fh:
try:
plist = plistlib.readPlist(fh)
except:
print "failed to parse: %s" % f["path"]
print fh.read()
if plist:
content = u"%s" % plist["Entry Text"]
split = content.split("\n")
title = split[0]
body = "\n".join(split[1:])
draft = "Publish URL" not in plist
image = get_matching_image_meta_if_exists(dayone_id, image_list)
if image:
print "getting image"
m = hashlib.sha1()
m.update("%s %s" % (dayone_id, datetime.datetime.now()))
image_name = "%s%s" % (dayone_id.split(".")[0], m.hexdigest())
kwargs = {
"author": author,
"title": title,
"body": body,
"dayone_post": True,
"dayone_id": dayone_id,
"dayone_posted": datetime_from_utc_to_local(get_from_plist_if_exists("Creation Date", plist)),
"dayone_last_modified": datetime_from_utc_to_local(datetime.datetime(*time.strptime(f["modified"], '%a, %d %b %Y %H:%M:%S +0000')[:6])),
"dayone_last_rev": f["revision"],
"is_draft": draft,
"location_area": get_from_plist_if_exists("Location.Administrative Area", plist),
"location_country": get_from_plist_if_exists("Location.Country", plist),
"latitude": get_from_plist_if_exists("Location.Latitude", plist),
"longitude": get_from_plist_if_exists("Location.Longitude", plist),
"location_name": get_from_plist_if_exists("Location.Place Name", plist),
"time_zone_string": get_from_plist_if_exists("Location.Time Zone", plist),
"weather_temp_f": get_from_plist_if_exists("Weather.Fahrenheit", plist),
"weather_temp_c": get_from_plist_if_exists("Weather.Celsius", plist),
"weather_description": get_from_plist_if_exists("Weather.Description", plist),
"weather_icon": get_from_plist_if_exists("Weather.IconName", plist),
"weather_pressure": get_from_plist_if_exists("Weather.Pressure MB", plist),
"weather_relative_humidity": get_from_plist_if_exists("Weather.Relative Humidity", plist),
"weather_wind_bearing": get_from_plist_if_exists("Weather.Wind Bearing", plist),
"weather_wind_chill_c": get_from_plist_if_exists("Weather.Wind Chill Celsius", plist),
"weather_wind_speed_kph": get_from_plist_if_exists("Weather.Wind Speed KPH", plist),
}
if exists:
if not p.is_draft:
kwargs["is_draft"] = False
for (key, value) in kwargs.items():
setattr(p, key, value)
if not p.written_on:
kwargs["written_on"] = get_from_plist_if_exists("Creation Date", plist),
if image:
image_file = client.get_file(image["path"])
p.dayone_image.save(
"%s.jpg" % image_name,
ContentFile(StringIO(image_file.read()).getvalue())
)
image_file.close()
p.save()
else:
p = Post.objects.create(**kwargs)
if image:
image_file = client.get_file(image["path"])
p.dayone_image.save(
"%s.jpg" % image_name,
ContentFile(StringIO(image_file.read()).getvalue())
)
p.save()
image_file.close()
# print p.slug
count += 1
author.last_dropbox_sync = datetime.datetime.now()
author.save()
except:
import traceback; traceback.print_exc();
pass
cache.delete(author.sync_cache_key)
cache.delete(author.sync_start_time_cache_key)
else:
print "Sync for %s already running." % author
print "Done"
