class DBUpload:
def __init__(self, key, secret):
#See if I already have a stored access_token
try:
with open('.dbtoken.json', 'r') as json_data_file:
data = json.load(json_data_file)
accessToken = data['accessToken']
except:
accessToken = None
if accessToken is None:
# connect to dropbox and start the session authorization process
flow = DropboxOAuth2FlowNoRedirect(key, secret)
logging.info("Authorize this application: {}".format(flow.start()))
authCode = input("Enter auth code here: ").strip()
# finish the authorization and grab the Dropbox client
(accessToken, userID) = flow.finish(authCode)
data = {'accessToken' : accessToken}
with open('.dbtoken.json', 'w') as outfile:
json.dump(data, outfile)
self.client = DropboxClient(accessToken)
logging.info("dropbox account linked")
self.Q = Queue()
self.thread = Thread(target=self.pull_from_queue, args=())
self.thread.daemon = True
self.thread.start()
## # setup the queue
## q = Queue()
## number_workers = 4
##
## for i in range(number_workers):
## t = Thread(target=pull_from_queue, q)
## t.daemon = True
## t.start()
##
## while True:
## time.sleep(1)
def upload_file(self, source, target, timestamp):
logging.info(" {}".format(timestamp))
self.client.put_file(target, open(source, "rb"))
# remove the file
os.remove(source)
def queue_file(self, source, target, timestamp):
self.Q.put((source, target, timestamp))
def pull_from_queue(self):
while True:
if not self.Q.empty():
(source, target, timestamp) = self.Q.get()
logging.info ("found item in queue" )
self.upload_file(source,target, timestamp)
else:
time.sleep(1.0)
def _upload(self, client: DropboxClient):
""" Upload new files from directory.
"""
now = time()
for filename in listdir(self.directory):
if fnmatch(filename, '*.upl'):
continue
local_name = '/' + filename
full_name = path.join(self.directory, filename)
upl_name = "{}.upl".format(full_name)
if not path.isfile(upl_name) and stat(full_name).st_mtime < now - 60:
with open(full_name, 'rb') as file_stream:
try:
client.put_file(local_name, file_stream)
share = client.share(local_name)
except (MaxRetryError, ErrorResponse):
continue
with DatabaseConnection() as db:
update = """
UPDATE events
SET url = '{}',
uploaded = current_timestamp
WHERE file = '{}'
""".format(share['url'], full_name)
db.dml(update)
try:
mknod(upl_name)
except FileExistsError:
pass
print("{} was uploaded to Dropbox.".format(filename))
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def generate_url(self, name):
url = self.client.share(name, name)
url = url['url']
f = request.urlopen(url)
path = f.geturl()
return re.sub('www.dropbox.com','dl.dropboxusercontent.com', path)
def url(self, name):
return name #
def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = DropboxClient(oauth2_access_token)
def delete(self, name):
self.client.file_delete(name)
def exists(self, name):
response = self.client.search('/', name, file_limit=1)
return bool(response)
def listdir(self, path):
directories, files = [], []
metadata = self.client.metadata(path)
for entry in metadata['contents']:
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.metadata(name)
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.metadata(name)
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.metadata(name)
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(name, self)
return remote_file
def _save(self, name, content):
self.client.put_file(name, content)
return name
def _read(self, name, num_bytes=None):
data = self.client.get_file(name)
return data.read(num_bytes)
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or
metadata['is_dir'] or
not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
html = markdown(client.get_file(path).read())
client.put_file(path[:-3] + '.html', html, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
class DropboxFiles():
"""Simple class for downloading/uploading files from/upto Dropbox
It requires an already generated access_token which can be provided
as a string or as a local filepath to the file containing one
"""
class TokenNotProvidedError():
pass
def __init__(self, filepath=None, token=None):
if not (filepath or token):
raise TokenNotProvidedError('Access token is not provided')
if filepath:
with open(filepath) as token_file:
token = token_file.read().strip()
self.client = DropboxClient(token)
def download(self, dropbox_filepath, local_filepath=None):
"""
If there is a file at Dropbox with the given `dropbox_filepath`
the function returns a filepath of downloaded file
otherwise it returns `None`
"""
filename = posixpath.basename(dropbox_filepath)
if not local_filepath:
local_filepath = filename
elif os.path.isdir(local_filepath):
local_filepath = os.path.join(local_filepath, filename)
try:
with self.client.get_file(dropbox_filepath) as dropbox_file, \
open(local_filepath, 'wb') as local_file:
local_file.write(dropbox_file.read())
except ErrorResponse as e:
if e.status == 404:
return None
else:
raise
return local_filepath
def upload(self, local_filepath, dropbox_filepath=None):
filename = os.path.basename(local_filepath)
if not dropbox_filepath:
dropbox_filepath = posixpath.join('/', filename)
elif dropbox_filepath.endswith('/'): # dropbox_filepath is a dir
dropbox_filepath = posixpath.join(dropbox_filepath, filename)
with open(local_filepath, 'rb') as local_file:
self.client.put_file(dropbox_filepath, local_file, overwrite=True)
def send_file_to_dropbox(data, task_pair_id, access_token,
filename, path, url):
client = DropboxClient(access_token)
file_path = posixpath.join(path, filename)
file_path = file_path.replace('\\', '|')
file_path = file_path.encode('ascii', errors='replace').decode()
resp = requests.get(url)
resp.raise_for_status()
client.put_file(file_path, resp.content, overwrite=True)
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def __init__(self, oauth2_access_token=setting('DROPBOX_OAUTH2_TOKEN')):
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = DropboxClient(oauth2_access_token)
def delete(self, name):
self.client.file_delete(name)
def exists(self, name):
try:
return bool(self.client.metadata(name))
except ErrorResponse:
return False
def listdir(self, path):
directories, files = [], []
metadata = self.client.metadata(path)
for entry in metadata['contents']:
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.metadata(name)
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.metadata(name)
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.metadata(name)
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.media(name)
return media['url']
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(name, self)
return remote_file
def _save(self, name, content):
self.client.put_file(name, content)
return name
def _read(self, name, num_bytes=None):
data = self.client.get_file(name)
return data.read(num_bytes)
def upload_file(file_path):
dc = DropboxClient(access_token)
try:
f = open(file_path, 'rb')
file_name = os.path.basename(file_path)
dc.put_file(file_name, f, overwrite=True)
f.close()
except IOError:
logger.error('Error: can\'t find file or read data')
except dbrest.ErrorResponse as e:
logger.error(e)
def upload_file(path):
access_token = load_dropbox_token()
client = DropboxClient(access_token)
name = path.split("/")[-1]
with open(path, 'rb') as data:
try:
client.put_file(name, data)
except Exception as e:
logging.exception(e)
else:
os.remove(path)
def get(self):
client = DropboxClient(self.get_secure_cookie("access_token"))
try:
client.file_delete('/blog.db')
client.file_delete('/settings.py')
except:
print("Can't find any backup")
finally:
with open(os.path.join(os.path.abspath(os.path.dirname("__file__")), 'blog.db'), 'rb') as f:
response = client.put_file('/blog.db', f)
with open(os.path.join(os.path.abspath(os.path.dirname("__file__")), 'settings.py'), 'rb') as f:
response = client.put_file('/settings.py', f)
self.redirect("/admin/dropbox?message=Backup successfully")
def put_chunk_data(self, chunk):
access_token = "JkT9Fsx17gQAAAAAAAAAs1DAbvAU3sQrP1pJ8WiTdnG7HzQHE70idZ9Ph-GBuA3s"
hash = hashlib.md5(chunk.data)
path = '/' + hash.hexdigest()
client = DropboxClient(access_token)
try:
response = client.metadata(path)
if 'is_deleted' in response and response['is_deleted']:
response = client.put_file('/' + path, base64.b64encode(chunk.data))
except rest.ErrorResponse as e:
if e.status == 404:
response = client.put_file('/' + path, base64.b64encode(chunk.data))
chunk.update_info('dropbox', response['path'] )
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, and non-markdown files
if (metadata is None or
metadata['is_dir'] or
not path.endswith('.md')):
continue
# Convert to Markdown and store as <basename>.html
response, metadata = client.get_file_and_metadata(path)
md = response.read().decode()
html = markdown(md, extensions=['gfm'])
html_name = path[:-3] + '.html'
client.put_file(html_name, html, overwrite=True)
# Include URL to published file in HTML comment at top of Markdown
# file
if '<!-- Published file url:' != md.split('\n')[0]:
share_url = client.share(html_name, short_url=False).get('url')
file_key = share_url.split('/')[4]
url_name = urllib.parse.quote(html_name)
url_comment = ('<!-- Published file url:\n'
'https://dl.dropboxusercontent.com/s/'
'{}{}\n-->\n'.format(file_key, url_name))
md = url_comment + md
client.put_file(path, md, overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
class DropboxStorage(Storage):
def __init__(self):
flow = DropboxOAuth2FlowNoRedirect(settings.APP_KEY,
settings.APP_SECRET)
#authorize_url = flow.start()
#access_token, user_id = flow.finish(settings.ACCESS_TOKEN)
self.client = DropboxClient(settings.ACCESS_TOKEN)
def _open(self, name, mode='rb'):
return File(name, mode=mode)
def _save(self, name, content):
response = self.client.put_file(name, content, overwrite=True)
return response['path']
def delete(self, name):
self.client.file_delete(name)
def exists(self, name):
try:
self.client.metadata(name)
except ErrorResponse as e:
if e.status == 404:
return False
raise e
return True
def size(self, name):
return self.client.metadata(name)['bytes']
def url(self, name):
return self.client.metadata(name)
def upload_welcome_pdf(dropbox_id):
user = User.query.filter_by(dropbox_id=dropbox_id,
active=True).first()
if user is None:
return False
# If we've already sent the welcome PDF before, Dropbox webhook went
# trigger, so do it here.
if user.uploaded_welcome_pdf:
return kindlebox(dropbox_id)
analytics.track(str(user.id), 'Sent welcome pdf')
client = DropboxClient(user.access_token)
try:
with open('app/static/bookdrop_welcome.pdf', 'rb') as f:
response = client.put_file('Welcome to BookDrop.pdf', f, overwrite=True)
if response:
log.info(u"Welcome PDF sent to user ID {0}.".format(user.id))
else:
raise Exception("No response received after sending welcome PDF")
user.set_uploaded_welcome_pdf()
db.session.commit()
except:
log.error((u"Welcome PDF failed for user ID "
"{0}.").format(user.id), exc_info=True)
return False
return True
def dropbox_upload(request):
if request.method == 'POST':
# POST-ed file, file name, path to upload to
fupload = request.FILES['file']
fname = fupload.name
fsize = fupload.size
path = '/' + fname
# Make connection to dropbox API
token = UserProfile.objects.get(user=request.user).dropbox_token
client = DropboxClient(token)
if fupload.multiple_chunks(): # If file is large enough to require chunked uploading
uploader = client.get_chunked_uploader(fupload, fname)
print 'uploading: ', fsize
while uploader.offset < fsize:
try:
upload = uploader.upload_chunked()
except rest.ErrorResponse, e:
print 'ERROR WHILE UPLOADING CHUNKED: ', str(e)
uploader.finish(path)
else:
response = client.put_file(path, fupload)
print 'uploaded: ', response
# print request.FILES
return render(request, 'website/dropbox/fileview.html')
def multiuploader(request):
if request.method == 'POST':
user_profile = UserProfile.objects.get(user = request.user)
drop_client = DropboxClient(user_profile.dropbox_profile.access_token['key'])
files = request.FILES.getlist(u'files[]')
for file in files:
folder = Project.objects.get(id=request.POST['project_id']).title
result_db = drop_client.put_file(settings.APP_NAME + '/' + folder + '/' + file.name, file.file)
#generating json response array
result = []
result.append({"files": [
{
"name": result_db['path'][result_db['path'].rfind('/') + 1:],
"size": result_db['bytes'],
"mime": result_db['mime_type']
}
]})
response_data = simplejson.dumps(result)
#checking for json data type
if "application/json" in request.META['HTTP_ACCEPT_ENCODING']:
mimetype = 'application/json'
else:
mimetype = 'text/plain'
return HttpResponse(response_data, mimetype=mimetype)
else: #GET
return HttpResponse('Only POST accepted')
def post(self):
user_email = self.session.get('user')
user_obj = getUser(user_email)
errors = []
success = []
user_sitename = self.request.get('Sitename', '')
user_siteID = self.request.get('SiteID', '')
if( user_siteID == "" ):
errors.append("Don't forget to give siteID!")
template_values = {'errors': '<br/>'.join(errors),'user': user_obj, 'addsite' : True}
showIndex(self, template_values)
return
idobj = database.Mapping.all()
idobj.filter("SiteID =", user_siteID)
counter = idobj.count(limit=1)
if (counter == 0):
success.append("URL registered!")
database.Mapping(Sitename = user_sitename, SiteID = user_siteID, user=user_obj).put()
client = DropboxClient(user_obj.access_token)
response = client.put_file('%s/index.html' % user_siteID, '<h1>Hello World</h1>')
template_values = {'success': '<br/>'.join(success), 'user' : True}
self.redirect('/home#banner')
else:
errors.append("ID already exist!! Try another :)")
template_values = {'success': '<br/>'.join(success), 'errors': '<br/>'.join(errors), 'user' : True,'addsite' : True}
self.redirect('/addsite#banner')
showIndex(self, template_values)
class DropboxBackupClient(object):
def __init__(self, key,secret,type, token_path):
self.TOKENS_FILEPATH = token_path
self.session = session.DropboxSession(key, secret, type)
def check_for_authorization(self):
try:
if os.path.exists(self.TOKENS_FILEPATH):
with open(self.TOKENS_FILEPATH, 'rb') as tokenhandle:
tokendata = pickle.load(tokenhandle)
self.request_token = tokendata.get('request_token')
self.access_token = tokendata.get('access_token')
self.session.set_token(self.access_token.key, self.access_token.secret)
self.client = DropboxClient(self.session)
self.client.account_info()
return True
else:
self.get_authorization()
except Exception:
return False
return True
def prompt_for_authorization(self):
print "Dropbox not authorized, visit the following URL to authorize %s and press Enter to continue" % self.session.build_authorize_url(self.request_token)
raw_input()
self.access_token = self.session.obtain_access_token(self.request_token)
self.tokendata = dict(request_token=self.request_token, access_token=self.access_token)
with open(self.TOKENS_FILEPATH, 'wb') as tokenhandle:
pickle.dump(self.tokendata, tokenhandle, -1)
def get_authorization(self):
self.request_token = self.session.obtain_request_token()
self.authorize_url = self.session.build_authorize_url(self.request_token)
self.prompt_for_authorization()
self.client = DropboxClient(self.session)
self.client.account_info()
def upload_file(self, file_object, upload_target):
if self.check_for_authorization():
self.client.put_file(upload_target, file_object)
def Upload2Dropbox():
binaryPath = os.path.join(RESULT_PATH, OUTPUT_BINARY_NAME)
try:
client = DropboxClient(access_token)
with open(binaryPath, 'rb') as f:
client.put_file('/Internal Releases/CommonAPI/v0.3.0/%s' % OUTPUT_BINARY_NAME, f, overwrite= True)
for p, s, export in PROJECTS:
srcPath = os.path.join(RESULT_PATH, export)
with open(srcPath, 'rb') as f:
client.put_file('/Internal Releases/CommonAPI/v0.3.0/%s' % export, f, overwrite= True)
except Exception as e:
print(e)
return False
return True
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
filename, fileext = os.path.splitext(path)
# Ignore deleted files, folders, and non-python files
if (metadata is None or metadata['is_dir'] or fileext != '.py' or
'-disappointed' in filename or '-reformed' in filename):
continue
with client.get_file(path) as fin:
original_code = fin.read()
try:
formatted_code = FormatCode(original_code)
suffix = "reformed"
# Only reform heretical code
if original_code != formatted_code:
formatted_code = credit(formatted_code)
client.put_file(filename + '-reformed.py',
formatted_code,
overwrite=True)
except:
# code itself was somehow invalid.
with open('facepalm.py', 'rb') as facepalm:
client.put_file(filename + '-disappointed.py', facepalm,
overwrite=True)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def process_user(uid):
'''Call /delta for the given user ID and process any changes.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
# Ignore deleted files, folders, non-jpegs, and our output
if (metadata is None or
metadata['is_dir'] or
not metadata.get('thumb_exists') or
path.endswith('-corrected.jpg')):
continue
temp_path = tempfile.mkdtemp()
with open(temp_path + '/input.jpg', 'wb') as fout:
with client.thumbnail(path, size='l', format='JPEG') as fin:
fout.write(fin.read())
shell.convert(temp_path + '/input.jpg', '-modulate', 250, '-fill', 'gold', '-tint', 100, temp_path + '/corrected.jpg')
with open(temp_path + '/corrected.jpg', 'rb') as f:
client.put_file(os.path.splitext(path)[0]+'-corrected.jpg', f, overwrite=True)
shutil.rmtree(temp_path)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def backup(filename):
"""
Backs up a specified file
"""
token = DevelopmentConfig.dropbox_app_token
client = DropboxClient(token)
f = open(filename, 'rb')
dropbox_filename = '/' + filename
response = client.put_file(dropbox_filename, f, overwrite=True)
return "uploaded:", response
def haunt_user(uid):
'''Call /delta for the given user ID and haunt any new images.'''
# OAuth token for the user
token = redis_client.hget('tokens', uid)
# /delta cursor for the user (None the first time)
cursor = redis_client.hget('cursors', uid)
# These are all v1-style API calls.
client = DropboxClient(token)
has_more = True
while has_more:
result = client.delta(cursor)
for path, metadata in result['entries']:
if (metadata is None or metadata['is_dir'] or not metadata.get('thumb_exists') or path.endswith('-spookified.gif')):
continue
temp_path = tempfile.mkdtemp()
with open(temp_path + '/alive.jpg', 'wb') as fout:
with client.thumbnail(path, size='l', format='JPEG') as fin:
fout.write(fin.read())
width = int(shell.identify('-format', '%[fx:w]', temp_path + '/alive.jpg').stdout)
shell.composite('-watermark', '50%', '-gravity', 'center', '-channel', 'RGBA', '(', 'halloweenghost.png', '-resize', '{}x'.format(width), ')', temp_path + '/alive.jpg', temp_path + '/second_frame.png')
shell.convert('-delay', '4000x1000', '-loop', 0, temp_path + '/alive.jpg', '-delay', '200x1000', temp_path + '/second_frame.png', temp_path + '/ghost.gif')
with open(temp_path + '/ghost.gif', 'rb') as f:
client.put_file(os.path.splitext(path)[0]+'-spookified.gif', f, overwrite=True)
shutil.rmtree(temp_path)
# Update cursor
cursor = result['cursor']
redis_client.hset('cursors', uid, cursor)
# Repeat only if there's more to do
has_more = result['has_more']
def dropbox_upload(timef):
"""uploads plots to dropbox"""
client = DropboxClient(ACCESS_TOKEN)
for root, dirs, files in os.walk(plots_dir):
includes = ['*_'+timef+'.*']
includes = r'|'.join([fnmatch.translate(x) for x in includes])
files = [f for f in files if re.match(includes, f)]
for filename in files:
# construct the full local path
local_path = os.path.join(root, filename)
# construct the full Dropbox path
relative_path = os.path.relpath(local_path, plots_dir)
dropbox_path = os.path.join('/plots/'+SID, relative_path)
# upload the file
try:
with open(local_path, 'rb') as f:
client.put_file(dropbox_path, f, overwrite=True)
except:
pass
def upload_dropbox_file(request):
if request.method == 'POST':
user_profile = UserProfile.objects.get(user = request.user)
#sess = session.DropboxSession(settings.DROPBOX_AUTH_KEY, settings.DROPBOX_AUTH_SECRET, access_type=settings.DROPBOX_ACCESS_TYPE)
#sess.set_token(user_profile.dropbox_profile.access_token['key'], user_profile.dropbox_profile.access_token['secret'])
drop_client = DropboxClient(user_profile.dropbox_profile.access_token['key'])
file = request.FILES.getlist(u'files[]')
folder = Project.objects.get(id=request.POST['project_id']).title
result = drop_client.put_file(settings.APP_NAME + '/' + folder + '/' + file.name, file.file)
dest_path = result['path']
return redirect("/project/{0}/files/".format(request.POST.get('project_id')))
class Webcam:
def __init__(self, plantId):
self.plantId = plantId
def takePicture(self):
self.client = DropboxClient(ACCES_TOKEN)
print self.client.account_info()
name = datetime.datetime.now().strftime("%y-%m-%d-%H-%M") + '.jpg'
command = PICTURE_CMD + name
process = subprocess.Popen(command.split(), stdout=subprocess.PIPE)
process.wait()
response = self.client.put_file(self.plantId + '/' + name, open(name, 'r'))
os.remove(name)
def save_note(access_token, data):
if not data['note']['title']:
return json_response(False, 'The title is missing.')
if not TITLE_RE.match(data['note']['title']):
return json_response(False, 'Allowed characters in the title: A-Z, a-z, 0-9, -, _')
if 'overwrite' not in data:
return json_response(False, 'An error occurred while processing the note.')
try:
client = DropboxClient(access_token)
#TODO: determine if the title was changed, if so, remove old note
response = client.put_file('/' + data['note']['title'], json.dumps(data['note']), overwrite=data['overwrite'])
except dropbox.rest.ErrorResponse as e:
app.logger.exception(e)
return json_response(False, e.user_error_msg)
return json_response(True, 'Your note was saved successfully.')
def main(session):
dropbox = DropboxClient(settings.DROPBOX_OAUTH_TOKEN)
dburi = settings.SQLALCHEMY_DATABASE_URI
dbpath = dburi.replace('sqlite:////', '')
env = 'production' if not settings.DEBUG else 'dev'
filename = 'backup.{}.{}.db'.format(now().format('YYYYMMDDHHmm'), env)
logger.info('Backup started for {} with filename {}.'.format(dbpath, filename))
with open(dbpath) as db:
try:
response = dropbox.put_file(filename, db)
rev = response['rev']
logger.info("Backup complete with revision id {}.".format(rev))
except ErrorResponse as err:
code = err.status
msg = err.error_msg
logger.error("Backup failed with code {} message '{}'.".format(code, msg))
class DropboxSyncClient:
def __init__(self, oauth2_access_token):
self.access_token = oauth2_access_token
self._client = DropboxClient(oauth2_access_token)
def upload_file(self, dropbox_file_path, local_file_path, replace=False):
f = open(local_file_path, 'rb')
response = self._client.put_file(dropbox_file_path, f, replace)
return 1, response['path']
def generate_public_url(self, dropbox_file_path):
return self._client.share(dropbox_file_path)['url']
def delete_file(self, dropbox_file_path):
self._client.file_delete(dropbox_file_path)
return 1, None
def update_local_to_cloud(self, dropbox_file_path, local_file_path):
return 1, self.upload_file(dropbox_file_path, local_file_path, replace=True)
def update_cloud_to_local(self, dropbox_file_path, local_file_path):
try:
try:
os.makedirs(os.path.dirname(local_file_path))
except Exception as e:
pass
open(local_file_path, 'wb').write(self._client.get_file(dropbox_file_path).read())
return 1, None
except Exception as e:
print e
return 1, None
def get_file_list(self, dropbox_folder_path):
folder_metadata = self._client.metadata(dropbox_folder_path)
return [content['path'] for content in folder_metadata['contents']]
def set_access_token(self, access_token):
self._client = DropboxClient(access_token)
def get_remaining_space(self):
quota_info = self._client.account_info()['quota_info']
return quota_info['total'] - (quota_info['shared'] + quota_info['normal'])
class DropboxDataStorage(FileSystemStorage):
def __init__(self):
self.client = DropboxClient(settings.DROPBOX_ACCESS_TOKEN)
def _open(self, name, mode='rb'):
f, metadata = self.client.get_file_and_metadata(name)
tmp_file = open('test','wb')
tmp_file.write(f.read())
return File(tmp_file, mode)
def _save(self,name,content):
f = content
name = str(self.get_available_name(name)).replace('\\', '/')
try:
response = self.client.put_file(name, f)
if 'rev' in response:
return name
else:
raise DropboxDataStorageException("Incorrect Dropbox Response")
except dbrest.ErrorResponse,e:
raise DropboxDataStorageException("Dropbox Response Error: %s"%str(e))
class DropboxStorage(Storage):
"""
A storage class providing access to resources in a Dropbox Public folder.
"""
def __init__(self, location='/Public'):
session = DropboxSession(CONSUMER_KEY,
CONSUMER_SECRET,
ACCESS_TYPE,
locale=None)
session.set_token(ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
self.client = DropboxClient(session)
self.account_info = self.client.account_info()
self.location = location
self.base_url = 'http://dl.dropbox.com/u/{uid}/'.format(
**self.account_info)
def _get_abs_path(self, name):
return os.path.realpath(os.path.join(self.location, name))
def _open(self, name, mode='rb'):
name = self._get_abs_path(name)
remote_file = DropboxFile(name, self, mode=mode)
return remote_file
def _save(self, name, content):
name = self._get_abs_path(name)
directory = os.path.dirname(name)
if not self.exists(directory) and directory:
self.client.file_create_folder(directory)
response = self.client.metadata(directory)
if not response['is_dir']:
raise IOError("%s exists and is not a directory." % directory)
abs_name = os.path.realpath(os.path.join(self.location, name))
self.client.put_file(abs_name, content)
return name
def delete(self, name):
name = self._get_abs_path(name)
self.client.file_delete(name)
def exists(self, name):
name = self._get_abs_path(name)
try:
metadata = self.client.metadata(name)
if metadata.get('is_deleted'):
return False
except ErrorResponse as e:
if e.status == 404: # not found
return False
raise e
return True
def listdir(self, path):
path = self._get_abs_path(path)
response = self.client.metadata(path)
directories = []
files = []
for entry in response.get('contents', []):
if entry['is_dir']:
directories.append(os.path.basename(entry['path']))
else:
files.append(os.path.basename(entry['path']))
return directories, files
def size(self, name):
cache_key = 'django-dropbox-size:%s' % filepath_to_uri(name)
size = cache.get(cache_key)
if not size:
size = self.client.metadata(filepath_to_uri(name))['bytes']
cache.set(cache_key, size, CACHE_TIMEOUT)
return size
def url(self, name):
cache_key = 'django-dropbox-url:%s' % filepath_to_uri(name)
url = cache.get(cache_key)
if not url:
url = self.client.share(filepath_to_uri(name),
short_url=False)['url'] + '?dl=1'
cache.set(cache_key, url, CACHE_TIMEOUT)
return url
def get_available_name(self, name):
"""
Returns a filename that's free on the target storage system, and
available for new content to be written to.
"""
name = self._get_abs_path(name)
dir_name, file_name = os.path.split(name)
file_root, file_ext = os.path.splitext(file_name)
# If the filename already exists, add an underscore and a number (before
# the file extension, if one exists) to the filename until the generated
# filename doesn't exist.
count = itertools.count(1)
while self.exists(name):
# file_ext includes the dot.
name = os.path.join(
dir_name, "%s_%s%s" % (file_root, count.next(), file_ext))
return name
motionCounter += 1
# check to see if the number of frames with consistent motion is
# high enough
if motionCounter >= conf["min_motion_frames"]:
# check to see if dropbox sohuld be used
if conf["use_dropbox"]:
# write the image to temporary file
t = TempImage()
cv2.imwrite(t.path, frame)
# upload the image to Dropbox and cleanup the tempory image
print("[UPLOAD] {}".format(ts))
path = "{base_path}/{timestamp}.jpg".format(
base_path=conf["dropbox_base_path"], timestamp=ts)
client.put_file(path, open(t.path, "rb"))
t.cleanup()
# update the last uploaded timestamp and reset the motion
# counter
lastUploaded = timestamp
motionCounter = 0
# otherwise, the room is not occupied
else:
motionCounter = 0
# check to see if the frames should be displayed to screen
if conf["show_video"]:
# display the security feed
cv2.imshow("Security Feed", frame)
class DropBox(clouddrive.CloudDrive):
name = "DropBox"
def __init__(self, access_token):
self.client = DropboxClient(access_token)
self.access_token = access_token
clouddrive.CloudDrive.__init__(self, access_token)
@staticmethod
def auth():
#app_key = 'knbyx2adg14kkn5'
#app_secret = 'kh3ulgqry8jffqp'
app_key = 'eif0l7bgnpb06di'
app_secret = 'qa02jhdo4jrwaid'
global Auth_DROPMytoken
global Auth_Drop_Running_true
def keep_running():
global Auth_Drop_Running_true
return Auth_Drop_Running_true
class AuthHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_HEAD(s):
s.send_response(200)
s.send_header("Content-type", "text/html")
s.end_headers()
def do_GET(s):
s.send_response(200)
s.send_header("Content-type", "text/html")
s.end_headers()
if s.path.find("code=") != -1:
global Auth_DROPMytoken
global Auth_Drop_Running_true
Auth_DROPMytoken = s.path
Auth_Drop_Running_true = False
s.wfile.write("ok see command line")
return
class MYrequest:
def __init__(self, url):
self.url = url
def get(self, mystring):
if self.url.find(mystring) == -1:
return
item = self.url.split(mystring + "=")[1]
if item.find("&") != -1:
item = item.split("&")[0]
return urllib.unquote(item).decode()
redirect_url = "http://localhost:8787"
my_session = {}
Auth_Drop_Running_true = True
flow = DropboxOAuth2Flow(app_key, app_secret, redirect_url, my_session,
"dropbox-auth-csrf-token")
authorize_url = flow.start()
webbrowser.open(authorize_url)
try:
httpd = BaseHTTPServer.HTTPServer(("", 8787), AuthHandler)
while keep_running():
httpd.handle_request()
except KeyboardInterrupt:
pass
httpd.server_close()
token, user_id, url_state = flow.finish(MYrequest(Auth_DROPMytoken))
clouddrive.CloudDrive.access_token = token
return token
def read(self, filename):
try:
data = []
with self.client.get_file(filename) as f:
data += f.read()
except:
return -1
return ''.join(data)
def write(self, filename, data):
try:
response = self.client.put_file(filename, data, True)
except:
return -1
return 1
def mkdir(self, dirname):
try:
self.client.file_create_folder(dirname)
except:
return -1
return 1
def delete(self, filename):
try:
self.client.file_delete(filename)
except:
return -1
return 1
def capacity(self):
try:
a = self.client.account_info()["quota_info"]
except:
return -1
return a["quota"] - a["shared"] - a["normal"]
def listing(self, path="/"):
lists = self.client.metadata(path)
filelist = []
for x in lists["contents"]:
filelist.append((x['path'], x['is_dir']))
return filelist
print " WELCOME :", known_people[prediction]
cv2.putText(image, "Verified User : {}".format(known_people[prediction]), (10, image.shape[0] - 30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2)
cv2.imwrite("Captured_Img.jpg", image)
else:
print "ALERT! NOT VERIFIED! CALL THE POLICE!"
cv2.putText(image, "ALERT! NOT VERIFIED! CALL THE POLICE!", (10, image.shape[0] -30), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 2)
cv2.imwrite("Captured_Img.jpg", image)
if conf["use_dropbox"]:
# check if image file was already created due to presence of faces
# else create image file due to motion detection only
if not(os.path.isfile("Captured_Img.jpg")):
cv2.imwrite("Captured_Img.jpg", image)
image_path = "Captured_Img.jpg"
path = "{base_path}/{timestamp}.jpg".format(base_path = conf["dropbox_base_path"], timestamp = ts)
client.put_file(path, open("Captured_Img.jpg", "rb"))
os.remove(image_path)
lastUploaded = timestamp
motionCounter = 0
else:
motionCounter = 0
cv2.imshow("Security Camera", image)
rawCapture.truncate(0)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cv2.destroyAllWindows()
with open("blocklist.txt", "a+") as f:
for line in contents:
f.write(line)
os.remove('ultBlockList.tmp.gz')
if __name__ == "__main__":
print("Getting list page")
soup = mksoup(requests.get("https://www.iblocklist.com/lists.php").text)
links = {} #dict of name of list -> its url
for row in soup.find_all("tr")[1:]: #for each table row
section = str(list(row.children)[0])
pieces = section.split("\"")
links[pieces[4].split("<")[0][1:]] = pieces[3]
for link in links: #download and combine files
print "Downloading " + link + " blocklist."
value = get_value_from(links[link])
if value == "subscription":
print "Blocklist is not available for free download D:"
elif value == "unavailable":
print "URL is unavailable"
else: #download and add this sucker
process(value)
if token:
file = open('blocklist.txt', 'rb')
response = db_client.put_file('/blocklist.txt', file, overwrite=True)
print 'Uploaded to Dropbox!'
class DropboxService():
@property
def DESCRIPTION():
return "dropbox"
def __init__(self):
self.token = None
self.dropbox_instance = None
self.__init()
"""
initializes the service state for use.
If necessary starts a setup process to obtain the necessary information
to execute the operations (e.g. service access)
"""
def __init(self):
self.__check_configuration()
## TODO what if it fails?
self.dropbox_instance = DropboxClient(self.token)
def __check_configuration(self):
self.token = utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.AUTH_TOKEN)
if self.token != None:
## TODO check if it is sstill valid... user may have unistalled the app
return
self.__fill_app_keys()
access_token = None
app_key = utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_KEY)
app_secret = utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_SECRET)
req_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
authorize_url = req_flow.start()
print ("1. Go to: " + authorize_url)
print ("2. Click \"Allow\" (you might have to log in first).")
print ("3. Copy the authorization code.")
auth_code = input("Enter the authorization code here: ").strip()
## TODO function that uses this, should be the one to caugh the exception
try:
access_token, user_id = req_flow.finish(auth_code)
except dbrest.ErrorResponse as e:
print('Error: %s' % (e,))
utils.write_config_section(DropboxService.DESCRIPTION.fget(), constants.AUTH_TOKEN, access_token)
self.token = access_token
if not self.__has_backup_data():
print ("You do not seem to have the backup file/folders configured. Pybackdata won't backup anything " + \
"without that setting.")
if not self.__has_backup_folder():
print ("You do not seem to have a target location for the backups. Pybackdata won't backup anything " + \
"without that setting.")
def __fill_app_keys(self):
to_update = []
if utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_KEY) == None:
app_key = input("Enter your development app key: ").strip()
utils.write_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_KEY, app_key)
if utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_SECRET) == None:
app_secret = input("Enter your development app secret: ").strip()
utils.write_config_section(DropboxService.DESCRIPTION.fget(), constants.APP_SECRET, app_secret)
def backup(self):
if not self.__has_backup_data():
print ("No data to backup. Terminated!")
return
if not self.__has_backup_folder():
print ("No destination location for backups. Terminated!")
return
file_list = self.__get_backup_data()
self.__backup(file_list, constants.BACKUP_FOLDER)
def __has_backup_data(self):
return utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.BACKUP_DATA) != None
def __has_backup_folder(self):
return utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.BACKUP_FOLDER) != None
## The path for each file should be separated by a ;
def __get_backup_data(self):
raw_list = utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.BACKUP_DATA)
split_list = raw_list.split(";")
for idx in range(len(split_list)):
split_list[idx] = split_list[idx].strip()
return split_list
"""
Backups the list of files given by <files>
"""
def __backup(self, files, dropbox_folder=None):
for data in files:
self._put_file(data)
def _put_file(self, data):
try:
with open(data, 'rb') as datafile:
service_path_raw = utils.get_config_section(DropboxService.DESCRIPTION.fget(), constants.BACKUP_FOLDER)
normalized_service_path = utils.normalize_folder(service_path_raw)
final_path = normalized_service_path + utils.get_filename(data)
print ("File Destination: " + final_path)
try:
# db.put_file(<dest_path>, <object file>, <overwrite>)
response = self.dropbox_instance.put_file(final_path, datafile, True)
print (data + " was uploaded (" + response['size'] + ")")
except dbrest.ErrorResponse as e:
print ("Error on upload. Skipped file!")
print (e)
except OSError as e:
print ("Could not open file. Skipped!")
print (e)
elif value == "unavailable":
print("URL is unavailable")
else: # download and add this sucker
process(value)
# duplicates removal
for file_format in formats_to_download:
lines_seen = set()
file_name = 'blocklist_{}.txt'.format(file_format)
gz_file_name = 'blocklist_{}.gz'.format(file_format)
os.rename(file_name, '{}.ori'.format(file_name))
with open('{}.ori'.format(file_name), 'r') as old_file:
with open(file_name, 'w') as new_file:
for old_file_line in old_file:
if old_file_line not in lines_seen: # not a duplicate
new_file.write(old_file_line)
lines_seen.add(old_file_line)
os.remove('{}.ori'.format(file_name))
with open(file_name, 'rb') as f_in:
with gzip.open(gz_file_name, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
if token:
for file_format in formats_to_download:
file_name = 'blocklist_{}.txt'.format(file_format)
with open(file_name, 'rb') as file:
response = db_client.put_file(file_name, file, overwrite=True)
print('Uploaded {} to Dropbox!'.format(file_name))
class DropBoxStorage(Storage):
"""DropBox Storage class for Django pluggable storage system."""
def __init__(self, oauth2_access_token=None, root_path=None):
oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN')
self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
if oauth2_access_token is None:
raise ImproperlyConfigured("You must configure a token auth at"
"'settings.DROPBOX_OAUTH2_TOKEN'.")
self.client = DropboxClient(oauth2_access_token)
def _full_path(self, name):
if name == '/':
name = ''
return safe_join(self.root_path, name).replace('\\', '/')
def delete(self, name):
self.client.file_delete(self._full_path(name))
def exists(self, name):
try:
return bool(self.client.metadata(self._full_path(name)))
except ErrorResponse:
return False
def listdir(self, path):
directories, files = [], []
full_path = self._full_path(path)
metadata = self.client.metadata(full_path)
for entry in metadata['contents']:
entry['path'] = entry['path'].replace(full_path, '', 1)
entry['path'] = entry['path'].replace('/', '', 1)
if entry['is_dir']:
directories.append(entry['path'])
else:
files.append(entry['path'])
return directories, files
def size(self, name):
metadata = self.client.metadata(self._full_path(name))
return metadata['bytes']
def modified_time(self, name):
metadata = self.client.metadata(self._full_path(name))
mod_time = datetime.strptime(metadata['modified'], DATE_FORMAT)
return mod_time
def accessed_time(self, name):
metadata = self.client.metadata(self._full_path(name))
acc_time = datetime.strptime(metadata['client_mtime'], DATE_FORMAT)
return acc_time
def url(self, name):
media = self.client.media(self._full_path(name))
return media['url']
def _open(self, name, mode='rb'):
remote_file = DropBoxFile(self._full_path(name), self)
return remote_file
def _save(self, name, content):
self.client.put_file(self._full_path(name), content)
return name
# increment the motion counter
MOTION_COUNTER += 1
# check to see if the number of frames with consistent motion is high enough
if MOTION_COUNTER >= CONF["min_motion_frames"]:
# check to see if dropbox sohuld be used
if CONF["use_dropbox"]:
# write the image to temporary file
T = TempImage()
cv2.imwrite(T.path, FRAME)
# upload the image to Dropbox and cleanup the tempory image
print "[UPLOAD] {}".format(TS)
PATH = "{base_path}/{timestamp}.jpg".format(
base_path=CONF["dropbox_base_path"], timestamp=TS)
CLIENT.put_file(PATH, open(T.path, "rb"))
T.cleanup()
# update the last uploaded timestamp and reset the motion
# counter
LAST_UPLOADED = TIMESTAMP
MOTION_COUNTER = 0
# otherwise, the room is not occupied
else:
MOTION_COUNTER = 0
# check to see if the frames should be displayed to screen
if CONF["show_video"]:
# display the security feed
cv2.putText(frame, ts, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX,
0.35, (0, 0, 255), 1)
if text == "Occupied":
if (timestamp - lastUploaded).seconds >= conf["min_upload_seconds"]:
motionCounter += 1
if motionCounter >= conf["min_motion_frames"]:
if conf["use_dropbox"]:
t = TempImage()
cv2.imwrite(t.path, frame)
print ("[UPLOAD] {}".format(ts))
path = "{base_path}/{timestamp}.jpg".format(
base_path=conf["dropbox_base_path"], timestamp=ts)
client.put_file(path, open(t.path, 'rb'))
t.cleanup()
lastUploaded = timestamp
motionCounter = 0
else:
motionCounter = 0
#check to see if the frames should be displayed to screen
if conf["show_video"]:
cv2.imshow("Security Feed", frame)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
import os
import sys
from dropbox.client import DropboxClient
access_token, local_directory, dropbox_destination = sys.argv[1:4]
client = DropboxClient(access_token)
for root, dirs, files in os.walk(local_directory):
for filename in files:
local_path = os.path.join(root, filename)
relative_path = os.path.relpath(local_path, local_directory)
dropbox_path = os.path.join(dropbox_destination, relative_path)
with open(local_path, 'rb') as f:
client.put_file(dropbox_path, f)
# Create archives to be sent to dropbox
location = location + " /tmp/databases/"
print "Creating the backup archive..!"
os.system("tar -czvf %s/%s.tar.gz %s" % (temp_dir, current_time, location) )
print "Archive created successfully"
os.system("echo %s >> /root/backup_script/backup_history" %(current_time))
backup_file = "%s/%s.tar.gz" %(temp_dir, current_time)
destination = "%s.tar.gz" %(current_time)
print "The payload is:", backup_file
print "The destination is gonna be: ", destination
print "Sending files to dropbox"
f = open(backup_file, 'rb')
response = dbx.put_file('/your_dropbox_app_folder/' + destination, f)
print "Response: ", response
# Retention
head = ''
total_backups = get_line_count('/root/backup_script/backup_history')
print "Checking retention"
print "There are %s backups | retention is %s" %(total_backups, retention)
diff = int(total_backups) - int(retention)
if diff > 0 :
# Delete diff number of folders from the top of the file backup_history
print "There are files to be deleted in the destination"
with open("/root/backup_script/backup_history") as f:
head = [next(f) for x in xrange(diff)]
# delete everything in head
