Example #1
0
class DropboxHelper(object):
    def __init__(self, access_token):
        self.dropbox = Dropbox(oauth2_access_token=access_token)

    def upload(self, filename, file_path):
        with open(file_path, 'rb') as f:
            try:
                self.dropbox.files_upload(f.read(), '/' + filename)
            except Exception:
                os.remove(file_path)
                raise CommandError(
                    'Unable to upload file to Dropbox. Maybe access token is invalid.'
                )

    def delete_all_files(self):
        for i in self.dropbox.files_list_folder('').entries:
            self.dropbox.files_delete(i.path_lower)

    def download_last_backup(self, dir_path):
        entries = self.dropbox.files_list_folder('').entries

        if len(entries) == 0:
            raise CommandError('We could not find any backup.')

        entry = entries[-1]
        full_path = dir_path + entry.path_lower

        self.dropbox.files_download_to_file(full_path, entry.path_lower)
        return full_path, entry.content_hash
Example #2
0
class DropboxHelper(object):

    def __init__(self, access_token):
        self.dropbox = Dropbox(oauth2_access_token=access_token)

    def upload(self, filename, file_path):
        with open(file_path, 'rb') as f:
            try:
                self.dropbox.files_upload(f.read(), '/' + filename)
            except Exception:
                os.remove(file_path)
                raise CommandError('Unable to upload file to Dropbox. Maybe access token is invalid.')

    def delete_all_files(self):
        for i in self.dropbox.files_list_folder('').entries:
            self.dropbox.files_delete(i.path_lower)

    def download_last_backup(self, dir_path):
        entries = self.dropbox.files_list_folder('').entries

        if len(entries) == 0:
            raise CommandError('We could not find any backup.')

        entry = entries[-1]
        full_path = dir_path + entry.path_lower

        self.dropbox.files_download_to_file(full_path, entry.path_lower)
        return full_path, entry.content_hash
Example #3
0
class DropBoxDataProvider(DataProviderBase):
    smoke_url = DROPBOX_SMOKE_URL

    def __init__(self, acs_token):
        self.dbx = Dropbox(acs_token)

    def api_smoke(self) -> int:
        return len(self.dbx.files_list_folder('').entries)

    def get_list_of_objects(self, dbx_folder='') -> list:
        result = namedtuple('Result', ['filename', 'filepatch'])
        return [
            result(el.name, el.path_lower)
            for el in self.dbx.files_list_folder(dbx_folder).entries
        ]

    def file_delete(self, dbx_file) -> str:
        return self.dbx.files_delete_v2(dbx_file).metadata.path_lower

    def file_download(self, local_file, dbx_file) -> str:
        return self.dbx.files_download_to_file(local_file, dbx_file).path_lower

    def file_upload(self, local_file, dbx_file) -> str:
        if isinstance(local_file, str):
            if local_file.startswith("https://"):
                waiting_time = 0.1
                waiting_attempt = 100
                url_result = self.dbx.files_save_url(dbx_file, local_file)
                job_id = url_result.get_async_job_id()
                while waiting_attempt > 0:
                    st = self.dbx.files_save_url_check_job_status(job_id)
                    if st.is_complete():
                        return st.get_complete().path_lower
                    sleep(waiting_time)
                    waiting_attempt -= 1
            else:
                with open(local_file, 'rb') as f:
                    return self.dbx.files_upload(
                        f.read(),
                        dbx_file,
                        autorename=True,
                        strict_conflict=True).path_lower
        else:
            return self.dbx.files_upload(local_file.read(),
                                         dbx_file,
                                         autorename=True,
                                         strict_conflict=True).path_lower

    def file_move(self, dbx_file_from, dbx_file_to) -> str:
        return self.dbx.files_move_v2(dbx_file_from,
                                      dbx_file_to).metadata.path_lower

    def create_folder(self, dbx_folder) -> str:
        return self.dbx.files_create_folder_v2(dbx_folder).metadata.path_lower

    def get_file_tmp_link(self, dbx_path) -> str:
        return self.dbx.files_get_temporary_link(dbx_path).link
Example #4
0
class TestDropbox(unittest.TestCase):
    def setUp(self):
        self.dbx = Dropbox(oauth2_token)

    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with self.assertRaises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        self.assertIn('token is malformed', cm.exception.message)

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with self.assertRaises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        self.assertEqual(cm.exception.error['error']['.tag'],
                         'invalid_access_token')

    def test_rpc(self):
        self.dbx.files_list_folder('')

        # Test API error
        random_folder_path = '/' + \
                             ''.join(random.sample(string.ascii_letters, 15))
        with self.assertRaises(ApiError) as cm:
            self.dbx.files_list_folder(random_folder_path)
        self.assertIsInstance(cm.exception.error, ListFolderError)

    def test_upload_download(self):
        # Upload file
        timestamp = str(datetime.datetime.utcnow())
        random_filename = ''.join(random.sample(string.ascii_letters, 15))
        random_path = '/Test/%s/%s' % (timestamp, random_filename)
        test_contents = DUMMY_PAYLOAD
        self.dbx.files_upload(test_contents, random_path)

        # Download file
        metadata, resp = self.dbx.files_download(random_path)
        self.assertEqual(DUMMY_PAYLOAD, resp.content)

        # Cleanup folder
        self.dbx.files_delete('/Test/%s' % timestamp)

    def test_bad_upload_types(self):
        with self.assertRaises(TypeError):
            self.dbx.files_upload(BytesIO(b'test'), '/Test')

    @require_team_token
    def test_team(self, token):
        dbxt = DropboxTeam(token)
        dbxt.team_groups_list()
        r = dbxt.team_members_list()
        if r.members:
            # Only test assuming a member if there is a member
            dbxt.as_user(
                r.members[0].profile.team_member_id).files_list_folder('')
Example #5
0
    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with pytest.raises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        assert 'token is malformed' in cm.value.message

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with pytest.raises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        assert cm.value.error.is_invalid_access_token()
    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with self.assertRaises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        self.assertIn('token is malformed', cm.exception.message)

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with self.assertRaises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        self.assertTrue(cm.exception.error.is_invalid_access_token())
    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with self.assertRaises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        self.assertIn('token is malformed', cm.exception.message)

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with self.assertRaises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        self.assertTrue(cm.exception.error.is_invalid_access_token())
Example #8
0
class TestDropbox(unittest.TestCase):

    def setUp(self):
        self.dbx = Dropbox(oauth2_token)

    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with self.assertRaises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        self.assertIn('token is malformed', cm.exception.message)

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with self.assertRaises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        self.assertEqual(cm.exception.error['error']['.tag'],
                         'invalid_access_token')

    def test_rpc(self):
        self.dbx.files_list_folder('')

        # Test API error
        random_folder_path = '/' + \
                             ''.join(random.sample(string.ascii_letters, 15))
        with self.assertRaises(ApiError) as cm:
            self.dbx.files_list_folder(random_folder_path)
        self.assertIsInstance(cm.exception.error, ListFolderError)

    def test_upload_download(self):
        # Upload file
        timestamp = str(datetime.datetime.utcnow())
        random_filename = ''.join(random.sample(string.ascii_letters, 15))
        random_path = '/Test/%s/%s' % (timestamp, random_filename)
        test_contents = string.ascii_letters
        self.dbx.files_upload(test_contents, random_path)

        # Download file
        metadata, resp = self.dbx.files_download(random_path)
        self.assertEqual(string.ascii_letters, resp.text)

        # Cleanup folder
        self.dbx.files_delete('/Test/%s' % timestamp)

    @require_team_token
    def test_team(self, token):
        dbxt = DropboxTeam(token)
        dbxt.team_groups_list()
        r = dbxt.team_members_list()
        if r.members:
            # Only test assuming a member if there is a member
            dbxt.as_user(r.members[0].profile.team_member_id).files_list_folder('')
Example #9
0
class Cliente:

    def __init__(self):

        self.auth_flow = DropboxOAuth2FlowNoRedirect(APP_KEY, APP_SECRET)
        self.authorize_url = self.auth_flow.start()

        self.dbx = None
        self.tree = Tree()

        self.path_actual = '' # Si el string es vacío estamos en el root


    def get_auth(self, auth_code):
        try:
            access_token, user_id = self.auth_flow.finish(auth_code)
        except Exception as e:
            print('Error: %s' % (e,))
            return False
        else:
            self.dbx = Dropbox(access_token)
            self.setup_tree(self.tree.root, '')
            return True

    def setup_tree(self, nodo_actual, path):

        print(threading.current_thread().name)
        lista = self.dbx.files_list_folder(path, recursive=False).entries
        for entry in lista:
            nodo = Nodo(entry.name, entry)
            self.tree.agregar_nodo(nodo_padre=nodo_actual, nodo_hijo=nodo)
            if isinstance(entry, files.FolderMetadata):
                t = threading.Thread(name=entry.name ,target=self.setup_tree, args=(nodo, entry.path_lower))
                t.setDaemon(True)
                t.start()
Example #10
0
class DropboxLinkReaderWithToken(ReadOnlyMixin, DropboxPersister):
    def __init__(self, url, oauth2_access_token):
        self._con = Dropbox(oauth2_access_token)
        self.url = url
        self.shared_link = SharedLink(url=url)

    def _yield_from_files_list_folder(self, path, path_gen):
        """
        yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
        in a depth search manner.
        """
        for x in path_gen.entries:
            if _entry_is_file(x):
                yield x.path_display
            else:
                folder_path = _extend_path(path, x.name)
                yield from self._get_path_gen_from_path(path=folder_path)

        if path_gen.has_more:
            yield from self._get_path_gen_from_cursor(path_gen.cursor,
                                                      path=path)

    def _get_path_gen_from_path(self, path):
        path_gen = self._con.files_list_folder(path=path,
                                               recursive=False,
                                               shared_link=self.shared_link)
        yield from self._yield_from_files_list_folder(path, path_gen)

    def _get_path_gen_from_cursor(self, cursor, path):
        path_gen = self._con.files_list_folder_continue(cursor)
        yield from self._yield_from_files_list_folder(path, path_gen)

    def __iter__(self):
        yield from self._get_path_gen_from_path(path="")
Example #11
0
    def test_last_photos_picker_dropbox_uploader(self, filters,
                                                 expected_files):
        """
        Test with LastPhotosPicker and DropboxUploader

        :param array filters: filters to use
        :param dict expected_files: expected files with hash of their content
        """
        if 'DROPBOX_TOKEN' not in os.environ.keys():
            raise SkipTest("DROPBOX_TOKEN environment variable is not defined")

        api_token = os.environ['DROPBOX_TOKEN']

        picker = LastPhotosPicker(self.sample_dir, 5, -1)
        uploader = DropboxUploader(api_token, self.remote_test_dir)

        photo_picker = PhotosPicker(picker, filters, uploader)
        photo_picker.run()

        dbx = Dropbox(api_token)

        test_dir = '/' + self.remote_test_dir
        files = dbx.files_list_folder(test_dir)
        actual_files = {}
        for file_metadata in files.entries:
            fullpath = self.target_dir + '/' + file_metadata.name
            dbx.files_download_to_file(fullpath,
                                       test_dir + '/' + file_metadata.name)
            md5 = self._compute_file_md5(fullpath)
            actual_files[file_metadata.name] = md5
            os.remove(fullpath)

        self.assertEqual(expected_files, actual_files)
Example #12
0
def i_am_dropbox(token):
    from dropbox import Dropbox
    my_client = Dropbox(token)
    file_list = my_client.files_list_folder('')
    #WE DID NOT CHECK HASMORE!
    folder_list = [x.name for x in file_list.entries if 'size' not in dir(x)]
   #print folder_list
    return folder_list
Example #13
0
class TestDropbox(unittest.TestCase):

    def setUp(self):
        self.dbx = Dropbox(oauth2_token)

    def test_bad_auth(self):
        # Test malformed token
        malformed_token_dbx = Dropbox(MALFORMED_TOKEN)
        with self.assertRaises(BadInputError) as cm:
            malformed_token_dbx.files_list_folder('')
        self.assertIn('token is malformed', cm.exception.message)

        # Test reasonable-looking invalid token
        invalid_token_dbx = Dropbox(INVALID_TOKEN)
        with self.assertRaises(AuthError) as cm:
            invalid_token_dbx.files_list_folder('')
        self.assertEqual(cm.exception.reason['error']['.tag'],
                         'invalid_access_token')

    def test_rpc(self):
        self.dbx.files_list_folder('')

        # Test API error
        random_folder_path = '/' + \
                             ''.join(random.sample(string.ascii_letters, 15))
        with self.assertRaises(ApiError) as cm:
            self.dbx.files_list_folder(random_folder_path)
        self.assertIsInstance(cm.exception.reason, ListFolderError)

    def test_upload_download(self):
        # Upload file
        timestamp = str(datetime.datetime.utcnow())
        random_filename = ''.join(random.sample(string.ascii_letters, 15))
        random_path = '/Test/%s/%s' % (timestamp, random_filename)
        test_contents = string.ascii_letters
        self.dbx.files_upload(test_contents, random_path)

        # Download file
        metadata, resp = self.dbx.files_download(random_path)
        self.assertEqual(string.ascii_letters, resp.text)

        # Cleanup folder
        self.dbx.files_delete('/Test/%s' % timestamp)
Example #14
0
def pull_from_dropbox(branch_name, symbol, gui):
    global settings  # Read

    if settings["CONFIRM"]:
        confirm = simpledialog.askstring(
            "Confirm",
            "Type in 'YES' if you wish to proceed. This will override existing worlds"
            " if a conflict is found")

        if not confirm == "YES":
            return "Pull cancelled"

    saves_path = settings["SAVES_DIR"]
    temp_dir = settings["TEMP_DIR"]
    source = "/" + branch_name + "/"

    if settings["OAUTH"] == 'null':
        return "Please type in /login to use this feature"

    println("Starting download... ", gui)
    println(
        "Do not close the app until 'done downloading' message is shown on the console",
        gui)

    # clear temp_dir
    for path_temp in listdir(temp_dir):
        if path.isdir(path.join(temp_dir, path_temp)):
            rmtree(path.join(temp_dir, path_temp))
        else:
            remove(path.join(temp_dir, path_temp))

    # download zip files
    dbx = Dropbox(settings["OAUTH"].access_token)
    for entry in dbx.files_list_folder(source).entries:
        file = source + entry.name
        with open(path.join(temp_dir, entry.name), "wb") as f:
            metadata, res = dbx.files_download(path=file)
            f.write(res.content)
            f.close()

    for path_temp in listdir(temp_dir):
        file = path.join(temp_dir, path_temp)
        name, extension = path.splitext(file)
        file_name, ext = path.splitext(path_temp)

        if file_name[0] == symbol and extension == ".zip":
            with zipfile.ZipFile(file, 'r') as zip_ref:
                z = path.join(saves_path, file_name)
                zip_ref.extractall(z)

        remove(file)

    save(settings)

    return "Done Downloading"
Example #15
0
def list_files(dbx: dropbox.Dropbox, lookup_folder: list):
    all_folder_entries = []
    for folder in lookup_folder:
        response = dbx.files_list_folder(folder, recursive=True)
        entries = response.entries
        while response.has_more:
            print(dt.now(), 'Listing files... Found so far:', len(entries))
            response = dbx.files_list_folder_continue(response.cursor)
            entries += response.entries
        all_folder_entries += entries
    return all_folder_entries
Example #16
0
def files():
    if access_token:
        dbx = Dropbox(access_token)
    else:
        return redirect(url_for("oauth2_start", _external=True, _scheme="https"))
    if request.args.get("days"):
        days = int(request.args.get("days"))
    else:
        days = 100
    time_delta = datetime.now() - timedelta(days=days)
    file_objs = dbx.files_list_folder("", recursive=True).entries
    files_json = process_files(file_objs, time_delta)
    return render_template("files.html", selected_files=files_json[:50])
Example #17
0
    def sync():
        challenge = request.args.get('challenge')
        if challenge is not None:
            return challenge

        """Synchronize database with todo.txt"""
        dropbox = db.session.query(models.Dropbox).first()  # type: models.Dropbox

        # Make sure this is a valid request from Dropbox
        signature = request.headers.get('X-Dropbox-Signature')
        if not hmac.compare_digest(signature, hmac.new(dropbox.secret.encode(), request.data, sha256).hexdigest()):
            app.logger.warn('Invalid sync request attempted')
            abort(403)

        dbx = Dropbox(dropbox.access_token)
        if dropbox.cursor is None:
            result = dbx.files_list_folder(path=os.path.dirname(dropbox.file_location))
        else:
            result = dbx.files_list_folder_continue(cursor=dropbox.cursor)

        # Check if todo.txt was changed
        found = False
        for metadata in result.entries:  # type: Metadata
            if metadata.path_lower == dropbox.file_location.lower():
                found = True
                break
        if not found:
            dropbox.cursor = result.cursor
            db.session.merge(dropbox)
            db.session.commit()
            return ''

        app.logger.info('Sync request made')

        try:
            md, res = dbx.files_download(path=dropbox.file_location)
        except ApiError as err:
            if err.error.is_path() and err.error.get_path().is_not_found():
                return 'File not found: ' + dropbox.file_location
            return 'Other error occurred'
        update_todos(content=res.content)

        dropbox.cursor = result.cursor
        db.session.merge(dropbox)
        db.session.commit()
        return ''
Example #18
0
def process_user(uid):
    '''Call /delta for the given user ID and process any changes.'''

    # OAuth token for the user
    token = redis_client.hget('tokens', uid)

    # /delta cursor for the user (None the first time)
    cursor = redis_client.hget('cursors', uid)

    dbx = Dropbox(token)
    has_more = True
    trello_client = trello.TrelloClient(TRELLO_API_KEY, token=TRELLO_API_TOKEN)

    while has_more:
        if cursor is None:
            result = dbx.files_list_folder(path='/remote_workspace')
        else:
            result = dbx.files_list_folder_continue(cursor)

        for entry in result.entries:
            # Ignore deleted files, folders, and non-markdown files
            if (isinstance(entry, DeletedMetadata) or isinstance(entry, FolderMetadata)):
                continue

            card = get_card_by_name(trello_client, entry.name.encode('utf-8'))
            
            if(card == False):
                trello_post(trello_client, entry.name.encode('utf-8'))
                continue

            card.set_pos("top")
            card.comment("update! revision: %s" % entry.rev)

            revs = dbx.files_list_revisions(entry.path_lower)
            if(card.list_id == "577db30f129e87073996cc1a" and len(revs.entries) >= 2):
                card.change_list("577db3127b9a95030e956ab8")



        # Update cursor
        cursor = result.cursor
        redis_client.hset('cursors', uid, cursor)

        # Repeat only if there's more to do
        has_more = result.has_more
Example #19
0
def get_list_of_files():
    get_or_create_folder()

    dbx = Dropbox(get_token())
    response = dbx.files_list_folder(path='/Mainstay')

    files = []

    for file in response.entries:
        files.append({
            'name': file.name,
            'extension': file.name.split('.')[-1],
            'size': round(int(file.size) / (1024 * 1024), 2),
            'date_modified': file.server_modified,
            'checksum': file.content_hash
        })

    return files
Example #20
0
def download_dropboxfiles(payload):
    # Get the Project
    project = Project.objects.get(pk=payload['project_id'])
    project.set_status('downloading')

    # Check to see what files to download from Dropbox
    client = Dropbox(project.user.dropboxinfo.access_token)
    num_files = 0
    for x in client.files_list_folder(project.path).entries:
        if x.path_lower.endswith('.jpg') and x.size > 0:
            # Download the file from Dropbox to local disk
            local_filename = os.path.split(x.path_lower)[-1]
            local_filepath = os.path.join(project.originals_path,
                                          local_filename)
            num_files += 1
            if os.path.exists(local_filepath
                              ):  # and not payload.get('redownload') == True
                continue
            client.files_download_to_file(local_filepath, x.path_lower)

    # Get the metadata as a separate task
    new_task(project.user, {
        'action': 'extract_metadata',
        'project_id': project.pk
    })

    # schedule a thumbnail task
    new_task(project.user, {
        'action': 'makethumbnails',
        'project_id': project.pk
    })

    # Downloading files can take a long time
    # In the meantime this Project could have been changed by other tasks
    # Reload it before setting the status
    project = Project.objects.get(pk=payload['project_id'])
    project.num_files_on_dropbox = num_files
    project.status = 'layout'
    project.save()
    return {'downloaded_files_count': num_files}
Example #21
0
def download_dropboxfiles(payload):
    # Get the Project
    project = Project.objects.get(pk=payload['project_id'])
    project.set_status('downloading')

    # Check to see what files to download from Dropbox
    client = Dropbox(project.user.dropboxinfo.access_token)
    num_files = 0
    for x in client.files_list_folder(project.path).entries:
        if x.path_lower.endswith('.jpg') and x.size > 0:
            # Download the file from Dropbox to local disk
            local_filename = os.path.split(x.path_lower)[-1]
            local_filepath = os.path.join(project.originals_path, local_filename)
            num_files += 1
            if os.path.exists(local_filepath): # and not payload.get('redownload') == True
                continue
            client.files_download_to_file(local_filepath, x.path_lower)

    
    # Get the metadata as a separate task
    new_task(project.user, {
        'action': 'extract_metadata',
        'project_id': project.pk
    })

    # schedule a thumbnail task
    new_task(project.user, {
        'action': 'makethumbnails',
        'project_id': project.pk
    })


    # Downloading files can take a long time
    # In the meantime this Project could have been changed by other tasks
    # Reload it before setting the status
    project = Project.objects.get(pk=payload['project_id'])
    project.num_files_on_dropbox = num_files
    project.status = 'layout'
    project.save()
    return {'downloaded_files_count':num_files}
Example #22
0
def process_user(account):
    '''Call /files/list_folder for the given user ID and process any changes.'''

    # OAuth token for the user
    token = redis_client.hget('tokens', account)

    # cursor for the user (None the first time)
    cursor = redis_client.hget('cursors', account)

    dbx = Dropbox(token)
    has_more = True

    while has_more:
        if cursor is None:
            result = dbx.files_list_folder(path='')
        else:
            result = dbx.files_list_folder_continue(cursor)

        for entry in result.entries:
            # Ignore deleted files, folders, and non-markdown files
            if (isinstance(entry, DeletedMetadata)
                    or isinstance(entry, FolderMetadata)
                    or not entry.path_lower.endswith('.md')):
                continue

            # Convert to Markdown and store as <basename>.html
            _, resp = dbx.files_download(entry.path_lower)
            html = markdown(resp.content.decode("utf-8"))
            dbx.files_upload(bytes(html, encoding='utf-8'),
                             entry.path_lower[:-3] + '.html',
                             mode=WriteMode('overwrite'))

        # Update cursor
        cursor = result.cursor
        redis_client.hset('cursors', account, cursor)

        # Repeat only if there's more to do
        has_more = result.has_more
Example #23
0
def i_am_thumb(token):
    import dropbox
    from dropbox import Dropbox
    import base64
    
    my_client = Dropbox(token)
    folderfile_list = my_client.files_list_folder('',True,True)
    
    file_list  =  [x for x in folderfile_list.entries if 'media_info' in dir(x)]
    image_list =  [x for x in file_list if not x.media_info == None]
    
    print "IMAGES ", image_list
    
    img_data = []
    for image in image_list:
        m,f = my_client.files_get_thumbnail(image.path_lower,\
                                        dropbox.files.ThumbnailFormat('png', value=None),\
                                        dropbox.files.ThumbnailSize('w128h128', None))
                                        
        encoded = base64.b64encode(f.content) 
        img_data.append (encoded)
        
    return img_data
Example #24
0
class DropboxStorage(Storage):
    """
    A storage class providing access to resources in a Dropbox folder.
    """
    def __init__(self, token=ACCESS_TOKEN, location=ROOT_FOLDER):
        if not token:
            raise ImproperlyConfigured("You must configure an access token at "
                                       "'settings.DROPBOX_ACCESS_TOKEN'.")

        self.client = Dropbox(token)
        self.account_info = self.client.users_get_current_account()
        self.location = location or DEFAULT_ROOT_FOLDER
        self.base_url = 'https://dl.dropboxusercontent.com/'

    def _get_abs_path(self, name):
        return os.path.realpath(os.path.join(self.location, name))

    def _open(self, name, mode='rb'):
        name = self._get_abs_path(name)
        remote_file = DropboxFile(name, self, mode=mode)
        return remote_file

    def _save(self, name, content):
        name = self._get_abs_path(name)
        directory = os.path.dirname(name)
        if not self.exists(directory) and directory:
            self.client.files_create_folder(directory)
        # response = self.client.files_get_metadata(directory)
        # if not response['is_dir']:
        #     raise IOError("%s exists and is not a directory." % directory)
        abs_name = os.path.realpath(os.path.join(self.location, name))
        self.client.files_upload(content.read(), abs_name)
        return name

    def delete(self, name):
        name = self._get_abs_path(name)
        try:
            self.client.files_delete(name)
        except ApiError as e:
            if isinstance(e.error, DeleteError)\
                    and e.error.is_path_lookup()\
                    and e.error.get_path_lookup().is_not_found():
                # not found
                return False
            # error
            raise e
        # deleted
        return True

    def exists(self, name):
        name = self._get_abs_path(name)
        try:
            self.client.files_get_metadata(name)
        except ApiError as e:
            if hasattr(e.error, 'is_path')\
                    and e.error.is_path()\
                    and e.error.get_path().is_not_found():
                # not found
                return False
            # error
            raise e
        # found
        return True

    def listdir(self, path):
        path = self._get_abs_path(path)
        response = self.client.files_list_folder(path)
        directories = []
        files = []
        for entry in response.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(os.path.basename(entry.path_display))
            elif isinstance(entry, FileMetadata):
                files.append(os.path.basename(entry.path_display))
        return directories, files

    def size(self, name):
        name = self._get_abs_path(name)
        return self.client.files_get_metadata(name).size

    def url(self, name):
        name = self._get_abs_path(name)
        return self.client.files_get_temporary_link(name).link

    def modified_time(self, name):
        name = self._get_abs_path(name)
        return self.client.files_get_metadata(name).server_modified

    def accessed_time(self, name):
        name = self._get_abs_path(name)
        # Note to the unwary, this is actually an mtime
        return self.client.files_get_metadata(name).client_modified

    def get_available_name(self, name, max_length=None):
        """
        Returns a filename that's free on the target storage system, and
        available for new content to be written to.
        """
        name = self._get_abs_path(name)
        dir_name, file_name = os.path.split(name)
        file_root, file_ext = os.path.splitext(file_name)
        # If the filename already exists, add an underscore and a number (before
        # the file extension, if one exists) to the filename until the generated
        # filename doesn't exist.
        count = itertools.count(1)
        while self.exists(name):
            # file_ext includes the dot.
            _fn = "%s_%s%s" % (file_root, count.next(), file_ext)
            name = os.path.join(dir_name, _fn)

        return name
Example #25
0
def get_images_result():
    token = constants.token
    dbx = Dropbox(token)

    result = dbx.files_list_folder(constants.images_path)
    return result.entries
Example #26
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None):
        oauth2_access_token = oauth2_access_token or setting(
            'DROPBOX_OAUTH2_TOKEN')
        self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure a token auth at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")
        self.client = Dropbox(oauth2_access_token)

    def _full_path(self, path):
        path = PurePosixPath(self.root_path) / path
        path = str(path)

        if path == '/':
            path = ''

        return path

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)
        result = self.client.files_list_folder(full_path)

        for entry in result.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)

        assert not result.has_more, "FIXME: Not implemented!"

        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        # Note to the unwary, this is actually an mtime
        return metadata.client_modified

    def url(self, name):
        try:
            media = self.client.files_get_temporary_link(self._full_path(name))
            return media.link
        except ApiError:
            raise ValueError("This file is not accessible via a URL.")

    def _open(self, name, mode='rb'):
        return DropBoxFile(self._full_path(name), self)

    def _save(self, name, content):
        try:
            content.open()

            if content.size <= self.CHUNK_SIZE:
                self.client.files_upload(content.read(), self._full_path(name))
            else:
                self._chunked_upload(content, self._full_path(name))

        finally:
            content.close()

        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE))
        cursor = UploadSessionCursor(session_id=upload_session.session_id,
                                     offset=content.tell())
        commit = CommitInfo(path=dest_path)

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit)
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor)
                cursor.offset = content.tell()
Example #27
0
class dropboxClient(object):

    ''' a class of methods to manage file storage on Dropbox API '''

    # https://www.dropbox.com/developers/documentation/http/documentation

    _class_fields = {
        'schema': {
            'access_token': '',
            'collection_name': 'labPack',
            'record_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.ogg',
            'record_key_path': '/home/user/.config/collective-acuity-labpack/user-data/obs/terminal',
            'record_key_comp': 'obs',
            'previous_key': 'obs/terminal/2016-03-17T17-24-51-687845Z.yaml',
            'secret_key': '6tZ0rUexOiBcOse2-dgDkbeY',
            'prefix': 'obs/terminal',
            'delimiter': '2016-03-17T17-24-51-687845Z.yaml',
            'max_results': 1
        },
        'components': {
            '.collection_name': {
                'max_length': 255,
                'must_not_contain': ['/', '^\\.']
            },
            '.record_key': {
                'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
            },
            '.record_key_path': {
                'max_length': 32767
            },
            '.record_key_comp': {
                'max_length': 255
            },
            '.secret_key': {
                'must_not_contain': [ '[\\t\\n\\r]' ]
            },
            '.max_results': {
                'min_value': 1,
                'integer_data': True
            },
            '.previous_key': {
                'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
            },
            '.prefix': {
                'must_not_contain': [ '[^\\w\\-\\./]', '^\\.', '\\.$', '^/', '//' ]
            }
        },
        'metadata': {
            'record_optimal_bytes': 10000 * 1024,
            'record_max_bytes': 150000 * 1024
        }
    }
    
    def __init__(self, access_token, collection_name=''):
        
        '''
            a method to initialize the dropboxClient class
            
        :param access_token: string with oauth2 access token for users account
        '''    

        title = '%s.__init__' % self.__class__.__name__
    
    # construct input validation model
        self.fields = jsonModel(self._class_fields)
        
    # validate inputs
        input_fields = {
            'access_token': access_token,
            'collection_name': collection_name
        }
        for key, value in input_fields.items():
            object_title = '%s(%s=%s)' % (title, key, str(value))
            self.fields.validate(value, '.%s' % key, object_title)
    
    # workaround for module namespace conflict
        from sys import path as sys_path
        sys_path.append(sys_path.pop(0))
        from dropbox import Dropbox
        from dropbox.files import FileMetadata, WriteMode, DeleteArg
        from dropbox.exceptions import ApiError
        sys_path.insert(0, sys_path.pop())
    
    # construct dropbox client
        from labpack.compilers.objects import _method_constructor
        self.dropbox = Dropbox(oauth2_access_token=access_token)
    
    # construct dropbox objects
        self.objects = _method_constructor({
            'FileMetadata': FileMetadata,
            'ApiError': ApiError,
            'WriteMode': WriteMode,
            'DeleteArg': DeleteArg
        })
    
    # construct collection name
        self.collection_name = collection_name
    
    def _import(self, record_key, record_data, overwrite=True, last_modified=0.0, **kwargs):
        
        '''
            a helper method for other storage clients to import into appdata
            
        :param record_key: string with key for record
        :param record_data: byte data for body of record
        :param overwrite: [optional] boolean to overwrite existing records
        :param last_modified: [optional] float to record last modified date
        :param kwargs: [optional] keyword arguments from other import methods 
        :return: boolean indicating whether record was imported
        '''
        
        title = '%s._import' % self.__class__.__name__
    
    # check overwrite
        if not overwrite:
            if self.exists(record_key):
                return False
    
    # check max size
        import sys
        record_max = self.fields.metadata['record_max_bytes']
        record_size = sys.getsizeof(record_data)
        error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
        if record_size > record_max:
            raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
    
    # TODO: apply session upload for files greater than record_max
            
    # construct upload kwargs
        upload_kwargs = {
            'f': record_data,
            'path': '/%s' % record_key,
            'mute': True,
            'mode': self.objects.WriteMode.overwrite
        }
    
    # modify file time
        import re
        if re.search('\\.drep$', record_key):
            from labpack.records.time import labDT
            drep_time = labDT.fromEpoch(1)
            upload_kwargs['client_modified'] = drep_time
        elif last_modified:
            from labpack.records.time import labDT
            mod_time = labDT.fromEpoch(last_modified)
            upload_kwargs['client_modified'] = mod_time
    
    # send upload request
        try:
            self.dropbox.files_upload(**upload_kwargs)
        except:
            raise DropboxConnectionError(title)
        
        return True
    
    def _walk(self, root_path=''):
        ''' an iterator method which walks the file structure of the dropbox collection '''
        title = '%s._walk' % self.__class__.__name__
        if root_path:
            root_path = '/%s' % root_path
        try:
            response = self.dropbox.files_list_folder(path=root_path, recursive=True)
            for record in response.entries:
                if not isinstance(record, self.objects.FileMetadata):
                    continue
                yield record.path_display[1:]
            if response.has_more:
                while response.has_more:
                    response = self.dropbox.files_list_folder_continue(response.cursor)
                    for record in response.entries:
                        if not isinstance(record, self.objects.FileMetadata):
                            continue
                        yield record.path_display[1:]
        except:
            raise DropboxConnectionError(title)
    
    def exists(self, record_key):
        
        ''' 
            a method to determine if a record exists in collection

        :param record_key: string with key of record
        :return: boolean reporting status
        '''
        
        title = '%s.exists' % self.__class__.__name__
    
    # validate inputs
        input_fields = {
            'record_key': record_key
        }
        for key, value in input_fields.items():
            object_title = '%s(%s=%s)' % (title, key, str(value))
            self.fields.validate(value, '.%s' % key, object_title)
    
    # send get metadata request
        file_path = '/%s' % record_key
        try:
            self.dropbox.files_get_metadata(file_path)
        except Exception as err:
            if str(err).find("LookupError('not_found'") > -1:
                return False
            else:
                raise DropboxConnectionError(title)

        return True
        
    def save(self, record_key, record_data, overwrite=True, secret_key=''):

        ''' 
            a method to create a record in the collection folder

        :param record_key: string with name to assign to record (see NOTES below)
        :param record_data: byte data for record body
        :param overwrite: [optional] boolean to overwrite records with same name
        :param secret_key: [optional] string with key to encrypt data
        :return: string with name of record

        NOTE:   record_key may only contain alphanumeric, /, _, . or -
                characters and may not begin with the . or / character.

        NOTE:   using one or more / characters splits the key into
                separate segments. these segments will appear as a
                sub directories inside the record collection and each
                segment is used as a separate index for that record
                when using the list method
                eg. lab/unittests/1473719695.2165067.json is indexed:
                [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
        '''

        title = '%s.save' % self.__class__.__name__
            
    # validate inputs
        input_fields = {
            'record_key': record_key,
            'secret_key': secret_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)
    
    # validate byte data
        if not isinstance(record_data, bytes):
            raise ValueError('%s(record_data=b"...") must be byte data.' % title)
        
    # construct and validate file path
        file_root, file_name = os.path.split(record_key)
        self.fields.validate(file_name, '.record_key_comp')
        while file_root:
            file_root, path_node = os.path.split(file_root)
            self.fields.validate(path_node, '.record_key_comp')

    # check overwrite exception
        if not overwrite:
            if self.exists(record_key):
                raise Exception('%s(record_key="%s") already exists. To overwrite, set overwrite=True' % (title, record_key))
    
    # check size of file
        import sys
        record_optimal = self.fields.metadata['record_optimal_bytes']
        record_max = self.fields.metadata['record_max_bytes']
        record_size = sys.getsizeof(record_data)
        error_prefix = '%s(record_key="%s", record_data=b"...")' % (title, record_key)
        if record_size > record_max:
            raise ValueError('%s exceeds maximum record data size of %s bytes.' % (error_prefix, record_max))
        elif record_size > record_optimal:
            print('[WARNING] %s exceeds optimal record data size of %s bytes.' % (error_prefix, record_optimal))
    
    # TODO add upload session for support of files over 150MB
    # http://dropbox-sdk-python.readthedocs.io/en/latest/moduledoc.html#dropbox.dropbox.Dropbox.files_upload_session_start
            
    # encrypt data
        if secret_key:
            from labpack.encryption import cryptolab
            record_data, secret_key = cryptolab.encrypt(record_data, secret_key)
    
    # construct upload kwargs
        upload_kwargs = {
            'f': record_data,
            'path': '/%s' % record_key,
            'mute': True,
            'mode': self.objects.WriteMode.overwrite
        }
    
    # modify file time
        import re
        if re.search('\\.drep$', file_name):
            from labpack.records.time import labDT
            drep_time = labDT.fromEpoch(1)
            upload_kwargs['client_modified'] = drep_time
    
    # send upload request
        try:
            self.dropbox.files_upload(**upload_kwargs)
        except:
            raise DropboxConnectionError(title)
        
        return record_key
    
    def load(self, record_key, secret_key=''):

        ''' 
            a method to retrieve byte data of appdata record

        :param record_key: string with name of record
        :param secret_key: [optional] string used to decrypt data
        :return: byte data for record body
        '''

        title = '%s.load' % self.__class__.__name__
    
    # validate inputs
        input_fields = {
            'record_key': record_key,
            'secret_key': secret_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)

    # construct file path
        file_path = '/%s' % record_key
    
    # request file data
        try:
            metadata, response = self.dropbox.files_download(file_path)
        except Exception as err:
            if str(err).find("LookupError('not_found'") > -1:
                raise Exception('%s(record_key=%s) does not exist.' % (title, record_key))
            else:
                raise DropboxConnectionError(title)
        record_data = response.content
    
    # decrypt (if necessary)
        if secret_key:
            from labpack.encryption import cryptolab
            record_data = cryptolab.decrypt(record_data, secret_key)
    
        return record_data
    
    def conditional_filter(self, path_filters):

        ''' a method to construct a conditional filter function for list method

        :param path_filters: dictionary or list of dictionaries with query criteria
        :return: filter_function object

        path_filters:
        [ { 0: { conditional operators }, 1: { conditional_operators }, ... } ]

        conditional operators:
            "byte_data": false,
            "discrete_values": [ "" ],
            "excluded_values": [ "" ],
            "greater_than": "",
            "less_than": "",
            "max_length": 0,
            "max_value": "",
            "min_length": 0,
            "min_value": "",
            "must_contain": [ "" ],
            "must_not_contain": [ "" ],
            "contains_either": [ "" ]
        '''

        title = '%s.conditional_filter' % self.__class__.__name__
        
        from labpack.compilers.filters import positional_filter
        filter_function = positional_filter(path_filters, title)
        
        return filter_function

    def list(self, prefix='', delimiter='', filter_function=None, max_results=1, previous_key=''):
        
        ''' 
            a method to list keys in the dropbox collection

        :param prefix: string with prefix value to filter results
        :param delimiter: string with value which results must not contain (after prefix)
        :param filter_function: (positional arguments) function used to filter results
        :param max_results: integer with maximum number of results to return
        :param previous_key: string with key in collection to begin search after
        :return: list of key strings

            NOTE:   each key string can be divided into one or more segments
                    based upon the / characters which occur in the key string as
                    well as its file extension type. if the key string represents
                    a file path, then each directory in the path, the file name
                    and the file extension are all separate indexed values.

                    eg. lab/unittests/1473719695.2165067.json is indexed:
                    [ 'lab', 'unittests', '1473719695.2165067', '.json' ]

                    it is possible to filter the records in the collection according
                    to one or more of these path segments using a filter_function.

            NOTE:   the filter_function must be able to accept an array of positional
                    arguments and return a value that can evaluate to true or false.
                    while searching the records, list produces an array of strings
                    which represent the directory structure in relative path of each
                    key string. if a filter_function is provided, this list of strings
                    is fed to the filter function. if the function evaluates this input
                    and returns a true value the file will be included in the list
                    results.
        '''
        
        title = '%s.list' % self.__class__.__name__
        
    # validate input
        input_fields = {
            'prefix': prefix,
            'delimiter': delimiter,
            'max_results': max_results,
            'previous_key': previous_key
        }
        for key, value in input_fields.items():
            if value:
                object_title = '%s(%s=%s)' % (title, key, str(value))
                self.fields.validate(value, '.%s' % key, object_title)

    # validate filter function
        if filter_function:
            try:
                path_segments = [ 'lab', 'unittests', '1473719695.2165067', '.json' ]
                filter_function(*path_segments)
            except:
                err_msg = '%s(filter_function=%s)' % (title, filter_function.__class__.__name__)
                raise TypeError('%s must accept positional arguments.' % err_msg)

    # construct empty results list
        results_list = []
        check_key = True
        if previous_key: 
            check_key = False
    
    # determine root path
        root_path = ''
        if prefix:
            from os import path
            root_path, file_name = path.split(prefix)

    # iterate over dropbox files
        for file_path in self._walk(root_path):
            path_segments = file_path.split(os.sep)
            record_key = os.path.join(*path_segments)
            record_key = record_key.replace('\\','/')
            if record_key == previous_key:
                check_key = True
    
    # find starting point
            if not check_key:
                continue
                
    # apply prefix filter
            partial_key = record_key
            if prefix:
                if record_key.find(prefix) == 0:
                    partial_key = record_key[len(prefix):]
                else:
                    continue
    
    # apply delimiter filter
            if delimiter:
                if partial_key.find(delimiter) > -1:
                    continue
    
    # apply filter function
            if filter_function:
                if filter_function(*path_segments):
                    results_list.append(record_key)
            else:
                results_list.append(record_key)

    # return results list
            if len(results_list) == max_results:
                return results_list

        return results_list
    
    def delete(self, record_key):

        ''' a method to delete a file

        :param record_key: string with name of file
        :return: string reporting outcome
        '''

        title = '%s.delete' % self.__class__.__name__

    # validate inputs
        input_fields = {
            'record_key': record_key
        }
        for key, value in input_fields.items():
            object_title = '%s(%s=%s)' % (title, key, str(value))
            self.fields.validate(value, '.%s' % key, object_title)

    # validate existence of file
        if not self.exists(record_key):
            exit_msg = '%s does not exist.' % record_key
            return exit_msg
            
    # remove file
        current_dir = os.path.split(record_key)[0]
        try:
            file_path = '/%s' % record_key
            self.dropbox.files_delete(file_path)
        except:
            raise DropboxConnectionError(title)

    # remove empty directories in path to file
        try:
            while current_dir:
                folder_path = '/%s' % current_dir
                response = self.dropbox.files_list_folder(folder_path)
                if not response.entries:
                    self.dropbox.files_delete(folder_path)
                    current_dir = os.path.split(current_dir)[0]
                else:
                    break
        except:
            raise DropboxConnectionError(title)

        exit_msg = '%s has been deleted.' % record_key
        return exit_msg
    
    def remove(self):
        
        ''' 
            a method to remove all records in the collection

        NOTE:   this method removes all the files in the collection, but the
                collection folder itself created by oauth2 cannot be removed.
                only the user can remove the app folder
                
        :return: string with confirmation of deletion
        '''

        title = '%s.remove' % self.__class__.__name__
    
    # get contents in root
        try:
            response = self.dropbox.files_list_folder(path='')
        except:
            raise DropboxConnectionError(title)

    # populate delete list
        delete_list = []
        for file in response.entries:
            delete_list.append(self.objects.DeleteArg(path=file.path_display))

    # continue retrieval if folder is large
        if response.has_more:
            try:
                while response.has_more:
                    response = self.dropbox.files_list_folder_continue(response.cursor)
                    for file in response.entries:
                        delete_list.append(self.objects.DeleteArg(path=file.path_display))
            except:
                raise DropboxConnectionError(title)

    # send batch delete request
        try:
            self.dropbox.files_delete_batch(delete_list)
        except:
            raise DropboxConnectionError(title)
    
    # return outcome
        insert = 'collection'
        if self.collection_name:
            insert = self.collection_name
        exit_msg = 'Contents of %s will been removed from Dropbox.' % insert
        return exit_msg

    def export(self, storage_client, overwrite=True):
        
        '''
            a method to export all the records in collection to another platform
            
        :param storage_client: class object with storage client methods
        :return: string with exit message
        '''
        
        title = '%s.export' % self.__class__.__name__
        
    # validate storage client
        method_list = [ 'save', 'load', 'list', 'export', 'delete', 'remove', '_import', 'collection_name' ]
        for method in method_list:
            if not getattr(storage_client, method, None):
                from labpack.parsing.grammar import join_words
                raise ValueError('%s(storage_client=...) must be a client object with %s methods.' % (title, join_words(method_list)))
            
    # walk collection folder to find files
        import os
        count = 0
        skipped = 0
        for file_path in self._walk():
            path_segments = file_path.split(os.sep)
            record_key = os.path.join(*path_segments)
            record_key = record_key.replace('\\','/')
            file_path = '/%s' % file_path
            
    # retrieve data and metadata
            try:
                metadata, response = self.dropbox.files_download(file_path)
            except:
                raise DropboxConnectionError(title)
            record_data = response.content
            client_modified = metadata.client_modified
            
    # import record into storage client
            last_modified = 0.0
            if client_modified:
                from dateutil.tz import tzutc
                from labpack.records.time import labDT
                last_modified = labDT.fromPython(client_modified.replace(tzinfo=tzutc())).epoch()
            outcome = storage_client._import(record_key, record_data, overwrite=overwrite, last_modified=last_modified)
            if outcome:
                count += 1
            else:
                skipped += 1
            
    # report outcome
        plural = ''
        skip_insert = ''
        new_folder = storage_client.collection_name
        if count != 1:
            plural = 's'
        if skipped > 0:
            skip_plural = ''
            if skipped > 1:
                skip_plural = 's'
            skip_insert = ' %s record%s skipped to avoid overwrite.' % (str(skipped), skip_plural)
        exit_msg = '%s record%s exported to %s.%s' % (str(count), plural, new_folder, skip_insert)
        return exit_msg
Example #28
0
def process_user(account):
    '''
    Call /files/list_folder for the given user ID and process any changes.
    '''
    print("*" * 30, " - PROCESS_USER - ", "*" * 30)
    # OAuth token for the user
    token = redis_client.hget('tokens', account)
    print("token: ", token)

    # cursor for the user (None the first time)
    cursor = redis_client.hget('cursors', account)
    print("cursor: ", cursor)

    output_file_extension = ".csv"
    input_file_extensions = [
        ".textgrid",
        ".TextGrid",
    ]
    processed_marker = "_cleaned"

    dbx = Dropbox(token.decode())
    has_more = True

    while has_more:
        print("there's more!")
        if cursor is None:
            print("cursor is 'None'!")
            result = dbx.files_list_folder(path='')
            print("result: ", result)
        else:
            print("entering files_list_folder_continue...")
            result = dbx.files_list_folder_continue(cursor.decode())
            print("result: ", result)

        for entry in result.entries:
            print("entry: ", entry)
            # yapf: disable
            if (
                isinstance(entry, DeletedMetadata) or
                isinstance(entry, FolderMetadata) or
                not any(entry.path_lower.endswith(e)
                        for e in input_file_extensions)
            ):
                # yapf: enable
                print("skipping.")
                continue

            _, resp = dbx.files_download(entry.path_lower)

            print("processing data...")
            processed_data = clean_data(resp.content)

            dbx.files_upload(processed_data,
                             entry.path_lower[:-4] + processed_marker +
                             output_file_extension,
                             mode=WriteMode('add'))
            #mode=WriteMode('overwrite'))

        # Update cursor
        cursor = result.cursor
        redis_client.hset('cursors', account, cursor)

        # Repeat only if there's more to do
        has_more = result.has_more
Example #29
0
class DropboxFS(FS):
    _meta = {
        "case_insensitive": False,
        "invalid_path_chars": "\0",
        "network": True,
        "read_only": False,
        "thread_safe": True,
        "unicode_paths": True,
        "virtual": False,
    }

    def __init__(self, accessToken, session=None):
        super(DropboxFS, self).__init__()
        self._lock = threading.RLock()
        self.dropbox = Dropbox(accessToken, session=session)

    def fix_path(self, path):

        if isinstance(path, bytes):
            try:
                path = path.decode("utf-8")
            except AttributeError:
                pass
        if not path.startswith("/"):
            path = "/" + path
        if path == "." or path == "./":
            path = "/"
        path = self.validatepath(path)

        return path

    def __repr__(self):
        return "<DropboxDriveFS>"

    def _infoFromMetadata(self, metadata):

        rawInfo = {
            "basic": {
                "name": metadata.name,
                "is_dir": isinstance(metadata, FolderMetadata),
            }
        }
        if isinstance(metadata, FileMetadata):
            rawInfo.update(
                {"details": {"size": metadata.size, "type": ResourceType.file}}
            )
        else:
            rawInfo.update({"details": {"type": ResourceType.directory}})

        return Info(rawInfo)

    def getinfo(self, path, namespaces=None):
        _path = self.fix_path(path)
        if _path == "/":
            info_dict = {
                "basic": {"name": "", "is_dir": True},
                "details": {"type": ResourceType.directory},
            }
            return Info(info_dict)

        try:

            metadata = self.dropbox.files_get_metadata(
                _path, include_media_info=True
            )
        except ApiError as e:
            raise errors.ResourceNotFound(path=path, exc=e)
        return self._infoFromMetadata(metadata)

    def setinfo(self, path, info):
        if not self.exists(path):
            raise errors.ResourceNotFound(path)

    def listdir(self, path):
        _path = self.fix_path(path)

        if _path == "/":
            _path = ""
        if not self.exists(_path):
            raise errors.ResourceNotFound(path)
        meta = self.getinfo(_path)
        if meta.is_file:
            raise errors.DirectoryExpected(path)

        result = self.dropbox.files_list_folder(_path, include_media_info=True)
        allEntries = result.entries
        while result.has_more:
            result = self.dropbox.files_list_folder_continue(result.cursor)
            allEntries += result.entries
        return [x.name for x in allEntries]

    def makedir(self, path, permissions=None, recreate=False):
        path = self.fix_path(path)
        if self.exists(path) and not recreate:
            raise errors.DirectoryExists(path)
        if path == "/":
            return SubFS(self, path)

        if self.exists(path):
            meta = self.getinfo(path)
            if meta.is_dir:
                if recreate == False:
                    raise errors.DirectoryExists(path)
                else:
                    return SubFS(self, path)
            if meta.is_file:
                raise errors.DirectoryExpected(path)

        ppath = self.get_parent(path)
        if not self.exists(ppath):

            raise errors.ResourceNotFound(ppath)

        try:

            folderMetadata = self.dropbox.files_create_folder_v2(path)
        except ApiError as e:

            raise errors.DirectoryExpected(path=path)

        return SubFS(self, path)

    def openbin(self, path, mode="r", buffering=-1, **options):

        path = self.fix_path(path)
        _mode = Mode(mode)
        mode = _mode
        _mode.validate_bin()
        _path = self.validatepath(path)

        log.debug("openbin: %s, %s", path, mode)
        with self._lock:
            try:
                info = self.getinfo(_path)
                log.debug("Info: %s", info)
            except errors.ResourceNotFound:
                if not _mode.create:
                    raise errors.ResourceNotFound(path)
                # Check the parent is an existing directory
                if not self.getinfo(self.get_parent(_path)).is_dir:
                    raise errors.DirectoryExpected(path)
            else:
                if info.is_dir:
                    raise errors.FileExpected(path)
            if _mode.exclusive:
                raise errors.FileExists(path)

        return DropboxFile(self.dropbox, path, mode)

    def remove(self, path):
        _path = self.fix_path(path)

        try:
            info = self.getinfo(path)
            if info.is_dir:
                raise errors.FileExpected(path=path)
            self.dropbox.files_delete_v2(_path)
        except ApiError as e:
            if isinstance(e.error._value, LookupError):
                raise errors.ResourceNotFound(path=path)
            log.debug(e)
            raise errors.FileExpected(path=path, exc=e)

    def removedir(self, path):
        _path = self.fix_path(path)

        if _path == "/":
            raise errors.RemoveRootError()

        try:
            info = self.getinfo(path)
            if not info.is_dir:
                raise errors.DirectoryExpected(path=path)
            if len(self.listdir(path)) > 0:
                raise errors.DirectoryNotEmpty(path=path)
            self.dropbox.files_delete_v2(_path)
        except ApiError as e:
            if isinstance(e.error._value, LookupError):
                raise errors.ResourceNotFound(path=path)

            raise errors.FileExpected(path=path, exc=e)

    def copy(self, src_path, dst_path, overwrite=False):

        src_path = self.fix_path(src_path)
        dst_path = self.fix_path(dst_path)
        try:
            src_meta = self.getinfo(src_path)
            if src_meta.is_dir:
                raise errors.FileExpected(src_path)

        except ApiError as e:
            raise errors.ResourceNotFound
        dst_meta = None
        try:
            dst_meta = self.getinfo(dst_path)
        except Exception as e:
            pass

        if dst_meta is not None:
            if overwrite == True:
                self.remove(dst_path)
            else:
                raise errors.DestinationExists(dst_path)
        parent_path = self.get_parent(dst_path)

        if not self.exists(parent_path):
            raise errors.ResourceNotFound(dst_path)

        self.dropbox.files_copy_v2(src_path, dst_path)

    def get_parent(self, dst_path):
        import os

        parent_path = os.path.abspath(os.path.join(dst_path, ".."))
        return parent_path

    def exists(self, path):
        path = self.fix_path(path)

        try:

            self.getinfo(path)
            return True

        except Exception as e:
            return False

    def move(self, src_path, dst_path, overwrite=False):
        _src_path = self.fix_path(src_path)
        _dst_path = self.fix_path(dst_path)

        if not self.getinfo(_src_path).is_file:
            raise errors.FileExpected(src_path)
        if not overwrite and self.exists(_dst_path):
            raise errors.DestinationExists(dst_path)
        if "/" in dst_path and not self.exists(self.get_parent(_dst_path)):
            raise errors.ResourceNotFound(src_path)
        with self._lock:
            try:
                if overwrite:
                    try:
                        # remove file anyways
                        self.dropbox.files_delete_v2(_dst_path)
                    except Exception as e:
                        pass

                self.dropbox.files_move_v2(_src_path, _dst_path)
            except ApiError as e:

                raise errors.ResourceNotFound(src_path, exc=e)

    def apierror_map(self, error):
        log.debug(error)

    def geturl(self, path, purpose='download'):
        url = self.dropbox.sharing_create_shared_link(path).url
        url = url.replace('?dl=0', '?raw=1')
        return url
Example #30
0
class DPBXBackend(duplicity.backend.Backend):
    """Connect to remote store using Dr*pB*x service"""
    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        self.api_account = None
        self.api_client = None
        self.auth_flow = None

        self.login()

    def user_authenticated(self):
        try:
            account = self.api_client.users_get_current_account()
            log.Debug("User authenticated as ,%s" % account)
            return True
        except:
            log.Debug('User not authenticated')
            return False

    def load_access_token(self):
        return os.environ.get('DPBX_ACCESS_TOKEN', None)

    def save_access_token(self, access_token):
        raise BackendException(
            'dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' %
            access_token)

    def obtain_access_token(self):
        log.Info("dpbx: trying to obtain access token")
        for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
            if env_var not in os.environ:
                raise BackendException(
                    'dpbx: %s environment variable not set' % env_var)

        app_key = os.environ['DPBX_APP_KEY']
        app_secret = os.environ['DPBX_APP_SECRET']

        if not sys.stdout.isatty() or not sys.stdin.isatty():
            log.FatalError(
                'dpbx error: cannot interact, but need human attention',
                log.ErrorCode.backend_command_error)

        auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
        log.Debug('dpbx,auth_flow.start()')
        authorize_url = auth_flow.start()
        print
        print '-' * 72
        print "1. Go to: " + authorize_url
        print "2. Click \"Allow\" (you might have to log in first)."
        print "3. Copy the authorization code."
        print '-' * 72
        auth_code = raw_input("Enter the authorization code here: ").strip()
        try:
            log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
            authresult = auth_flow.finish(auth_code)
        except Exception as e:
            raise BackendException('dpbx: Unable to obtain access token: %s' %
                                   e)
        log.Info("dpbx: Authentication successfull")
        self.save_access_token(authresult.access_token)

    def login(self):
        if self.load_access_token() is None:
            self.obtain_access_token()

        self.api_client = Dropbox(self.load_access_token())
        self.api_account = None
        try:
            log.Debug('dpbx,users_get_current_account([token])')
            self.api_account = self.api_client.users_get_current_account()
            log.Debug("dpbx,%s" % self.api_account)

        except (BadInputError, AuthError) as e:
            log.Debug('dpbx,exception: %s' % e)
            log.Info(
                "dpbx: Authentication failed. Trying to obtain new access token"
            )

            self.obtain_access_token()

            # We're assuming obtain_access_token will throw exception.
            # So this line should not be reached
            raise BackendException(
                "dpbx: Please update DPBX_ACCESS_TOKEN and try again")

        log.Info("dpbx: Successfully authenticated as %s" %
                 self.api_account.name.display_name)

    def _error_code(self, operation, e):
        if isinstance(e, ApiError):
            err = e.error

            if isinstance(err, GetMetadataError) and err.is_path():
                if err.get_path().is_not_found():
                    return log.ErrorCode.backend_not_found
            elif isinstance(err, DeleteError) and err.is_path_lookup():
                lookup = e.error.get_path_lookup()
                if lookup.is_not_found():
                    return log.ErrorCode.backend_not_found

    @command()
    def _put(self, source_path, remote_filename):
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()

        file_size = os.path.getsize(source_path.name)
        progress.report_transfer(0, file_size)

        if file_size < DPBX_UPLOAD_CHUNK_SIZE:
            # Upload whole file at once to avoid extra server request
            res_metadata = self.put_file_small(source_path, remote_path)
        else:
            res_metadata = self.put_file_chunked(source_path, remote_path)

        # A few sanity checks
        if res_metadata.path_display != remote_path:
            raise BackendException(
                'dpbx: result path mismatch: %s (expected: %s)' %
                (res_metadata.path_display, remote_path))
        if res_metadata.size != file_size:
            raise BackendException(
                'dpbx: result size mismatch: %s (expected: %s)' %
                (res_metadata.size, file_size))

    def put_file_small(self, source_path, remote_path):
        if not self.user_authenticated():
            self.login()

        file_size = os.path.getsize(source_path.name)
        f = source_path.open('rb')
        try:
            log.Debug('dpbx,files_upload(%s, [%d bytes])' %
                      (remote_path, file_size))

            res_metadata = self.api_client.files_upload(
                f.read(),
                remote_path,
                mode=WriteMode.overwrite,
                autorename=False,
                client_modified=None,
                mute=True)
            log.Debug('dpbx,files_upload(): %s' % res_metadata)
            progress.report_transfer(file_size, file_size)
            return res_metadata
        finally:
            f.close()

    def put_file_chunked(self, source_path, remote_path):
        if not self.user_authenticated():
            self.login()

        file_size = os.path.getsize(source_path.name)
        f = source_path.open('rb')
        try:
            buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
            log.Debug(
                'dpbx,files_upload_session_start([%d bytes]), total: %d' %
                (len(buf), file_size))
            upload_sid = self.api_client.files_upload_session_start(buf)
            log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
            upload_cursor = UploadSessionCursor(upload_sid.session_id,
                                                f.tell())
            commit_info = CommitInfo(remote_path,
                                     mode=WriteMode.overwrite,
                                     autorename=False,
                                     client_modified=None,
                                     mute=True)
            res_metadata = None
            progress.report_transfer(f.tell(), file_size)

            requested_offset = None
            current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
            retry_number = globals.num_retries
            is_eof = False

            # We're doing our own error handling and retrying logic because
            # we can benefit from Dpbx chunked upload and retry only failed
            # chunk
            while not is_eof or not res_metadata:
                try:
                    if requested_offset is not None:
                        upload_cursor.offset = requested_offset

                    if f.tell() != upload_cursor.offset:
                        f.seek(upload_cursor.offset)
                    buf = f.read(current_chunk_size)

                    is_eof = f.tell() >= file_size
                    if not is_eof and len(buf) == 0:
                        continue

                    # reset temporary status variables
                    requested_offset = None
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
                    retry_number = globals.num_retries

                    if not is_eof:
                        assert len(buf) != 0
                        log.Debug(
                            'dpbx,files_upload_sesssion_append([%d bytes], offset=%d)'
                            % (len(buf), upload_cursor.offset))
                        self.api_client.files_upload_session_append(
                            buf, upload_cursor.session_id,
                            upload_cursor.offset)
                    else:
                        log.Debug(
                            'dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)'
                            % (len(buf), upload_cursor.offset))
                        res_metadata = self.api_client.files_upload_session_finish(
                            buf, upload_cursor, commit_info)

                    upload_cursor.offset = f.tell()
                    log.Debug('progress: %d of %d' %
                              (upload_cursor.offset, file_size))
                    progress.report_transfer(upload_cursor.offset, file_size)
                except ApiError as e:
                    error = e.error
                    if isinstance(error, UploadSessionLookupError
                                  ) and error.is_incorrect_offset():
                        # Server reports that we should send another chunk.
                        # Most likely this is caused by network error during
                        # previous upload attempt. In such case we'll get
                        # expected offset from server and it's enough to just
                        # seek() and retry again
                        new_offset = error.get_incorrect_offset(
                        ).correct_offset
                        log.Debug(
                            'dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)'
                            % (upload_cursor.offset, new_offset))
                        if requested_offset is not None:
                            # chunk failed even after seek attempt. Something
                            # strange and no safe way to recover
                            raise BackendException(
                                "dpbx: unable to chunk upload")
                        else:
                            # will seek and retry
                            requested_offset = new_offset
                        continue
                    raise
                except ConnectionError as e:
                    log.Debug('dpbx,files_upload_session_append: %s' % e)

                    retry_number -= 1

                    if not self.user_authenticated():
                        self.login()

                    if retry_number == 0:
                        raise

                    # We don't know for sure, was partial upload successful or
                    # not. So it's better to retry smaller amount to avoid extra
                    # reupload
                    log.Info('dpbx: sleeping a bit before chunk retry')
                    time.sleep(30)
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
                    requested_offset = None
                    continue

            if f.tell() != file_size:
                raise BackendException('dpbx: something wrong')

            log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
            progress.report_transfer(f.tell(), file_size)

            return res_metadata

        finally:
            f.close()

    @command()
    def _get(self, remote_filename, local_path):
        if not self.user_authenticated():
            self.login()

        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()

        log.Debug('dpbx,files_download(%s)' % remote_path)
        res_metadata, http_fd = self.api_client.files_download(remote_path)
        log.Debug('dpbx,files_download(%s): %s, %s' %
                  (remote_path, res_metadata, http_fd))
        file_size = res_metadata.size
        to_fd = None
        progress.report_transfer(0, file_size)
        try:
            to_fd = local_path.open('wb')
            for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
                to_fd.write(c)
                progress.report_transfer(to_fd.tell(), file_size)

        finally:
            if to_fd:
                to_fd.close()
            http_fd.close()

        # It's different from _query() check because we're not querying metadata
        # again. Since this check is free, it's better to have it here
        local_size = os.path.getsize(local_path.name)
        if local_size != file_size:
            raise BackendException("dpbx: wrong file size: %d (expected: %d)" %
                                   (local_size, file_size))

        local_path.setdata()

    @command()
    def _list(self):
        # Do a long listing to avoid connection reset
        if not self.user_authenticated():
            self.login()
        remote_dir = '/' + urllib.unquote(
            self.parsed_url.path.lstrip('/')).rstrip()

        log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
        res = []
        try:
            resp = self.api_client.files_list_folder(remote_dir)
            log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))

            while True:
                res.extend([entry.name for entry in resp.entries])
                if not resp.has_more:
                    break
                resp = self.api_client.files_list_folder_continue(resp.cursor)
        except ApiError as e:
            if (isinstance(e.error, ListFolderError) and e.error.is_path()
                    and e.error.get_path().is_not_found()):
                log.Debug('dpbx.list(%s): ignore missing folder (%s)' %
                          (remote_dir, e))
            else:
                raise

        # Warn users of old version dpbx about automatically renamed files
        self.check_renamed_files(res)

        return res

    @command()
    def _delete(self, filename):
        if not self.user_authenticated():
            self.login()

        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, filename).rstrip()

        log.Debug('dpbx.files_delete(%s)' % remote_path)
        self.api_client.files_delete(remote_path)

        # files_permanently_delete seems to be better for backup purpose
        # but it's only available for Business accounts
        # self.api_client.files_permanently_delete(remote_path)

    @command()
    def _close(self):
        """close backend session? no! just "flush" the data"""
        log.Debug('dpbx.close():')

    @command()
    def _query(self, filename):
        if not self.user_authenticated():
            self.login()
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, filename).rstrip()

        log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
        info = self.api_client.files_get_metadata(remote_path)
        log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
        return {'size': info.size}

    def check_renamed_files(self, file_list):
        if not self.user_authenticated():
            self.login()
        bad_list = [
            x for x in file_list
            if DPBX_AUTORENAMED_FILE_RE.search(x) is not None
        ]
        if len(bad_list) == 0:
            return
        log.Warn('-' * 72)
        log.Warn(
            'Warning! It looks like there are automatically renamed files on backend'
        )
        log.Warn(
            'They were probably created when using older version of duplicity.'
        )
        log.Warn('')
        log.Warn(
            'Please check your backup consistency. Most likely you will need to choose'
        )
        log.Warn(
            'largest file from duplicity-* (number).gpg and remove brackets from its name.'
        )
        log.Warn('')
        log.Warn(
            'These files are not managed by duplicity at all and will not be')
        log.Warn('removed/rotated automatically.')
        log.Warn('')
        log.Warn('Affected files:')
        for x in bad_list:
            log.Warn('\t%s' % x)
        log.Warn('')
        log.Warn('In any case it\'s better to create full backup.')
        log.Warn('-' * 72)
Example #31
0
class Dropbox(BlackboxStorage):
    """Storage handler that uploads backups to Dropbox."""

    required_fields = ("access_token", )

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

        self.upload_base = self.config.get("upload_directory") or "/"
        self.client = DropboxClient(self.config["access_token"])
        self.valid = self._validate_token()

    def _validate_token(self):
        """Check if dropbox token is valid."""
        try:
            return self.client.check_user("test").result == "test"
        except AuthError:
            return False

    def sync(self, file_path: Path) -> None:
        """Sync a file to Dropbox."""
        # Check if Dropbox token is valid.
        if self.valid is False:
            error = "Dropbox token is invalid!"
            self.success = False
            self.output = error
            log.error(error)
            return None

        # This is size what can be uploaded as one chunk.
        # When file is bigger than that, this will be uploaded
        # in multiple parts.
        chunk_size = 4 * 1024 * 1024

        temp_file, recompressed = self.compress(file_path)
        upload_path = f"{self.upload_base}{file_path.name}{'.gz' if recompressed else ''}"

        try:
            with temp_file as f:
                file_size = os.stat(f.name).st_size
                log.debug(file_size)
                if file_size <= chunk_size:
                    self.client.files_upload(f.read(), upload_path,
                                             WriteMode.overwrite)
                else:
                    session_start = self.client.files_upload_session_start(
                        f.read(chunk_size))
                    cursor = UploadSessionCursor(session_start.session_id,
                                                 offset=f.tell())
                    # Commit contains path in Dropbox and write mode about file
                    commit = CommitInfo(upload_path, WriteMode.overwrite)

                    while f.tell() < file_size:
                        if (file_size - f.tell()) <= chunk_size:
                            self.client.files_upload_session_finish(
                                f.read(chunk_size), cursor, commit)
                        else:
                            self.client.files_upload_session_append(
                                f.read(chunk_size), cursor.session_id,
                                cursor.offset)
                            cursor.offset = f.tell()
            self.success = True
        except (ApiError, HttpError) as e:
            log.error(e)
            self.success = False
            self.output = str(e)

    def rotate(self, database_id: str) -> None:
        """
        Rotate the files in the Dropbox directory.

        All files in base directory of backups will be deleted when they
        are older than `retention_days`, and because of this,
        it's better to have backups in isolated folder.
        """
        # Check if Dropbox token is valid.
        if self.valid is False:
            log.error("Dropbox token is invalid - Can't delete old backups!")
            return None
        # Let's rotate only this type of database
        db_type_regex = rf"{database_id}_blackbox_\d{{2}}_\d{{2}}_\d{{4}}.+"

        # Receive first batch of files.
        files_result = self.client.files_list_folder(
            self.upload_base if self.upload_base != "/" else "")
        entries = [
            entry for entry in files_result.entries
            if self._is_backup_file(entry, db_type_regex)
        ]

        # If there is more files, receive all of them.
        while files_result.has_more:
            cursor = files_result.cursor
            files_result = self.client.files_list_folder_continue(cursor)
            entries += [
                entry for entry in files_result.entries
                if self._is_backup_file(entry, db_type_regex)
            ]

        retention_days = 7
        if Blackbox.retention_days:
            retention_days = Blackbox.retention_days

        # Find all old files and delete them.
        for item in entries:
            last_modified = item.server_modified
            now = datetime.now(tz=last_modified.tzinfo)
            delta = now - last_modified
            if delta.days >= retention_days:
                self.client.files_delete(item.path_lower)

    @staticmethod
    def _is_backup_file(entry, db_type_regex) -> bool:
        """Check if file is actually this kind of database backup."""
        return isinstance(entry, FileMetadata) and re.match(
            db_type_regex, entry.name)
Example #32
0
class DropboxPersister(Persister):
    """
    A persister for dropbox.
    You need to have the python connector (if you don't: pip install dropbox)
    You also need to have a token for your dropbox app. If you don't it's a google away.
    Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
    dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.

    >>> import json
    >>> import os
    >>>
    >>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
    >>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
    >>> if '/py2store_data/test/_can_remove' in s:
    ...     del s['/py2store_data/test/_can_remove']
    ...
    >>>
    >>> n = len(s)
    >>> if n == 1:
    ...     assert list(s) == ['/py2store_data/test/_can_remove']
    ...
    >>> s['/py2store_data/test/_can_remove'] = b'this is a test'
    >>> assert len(s) == n + 1
    >>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
    >>> '/py2store_data/test/_can_remove' in s
    True
    >>> del s['/py2store_data/test/_can_remove']
    """
    def __init__(
        self,
        rootdir,
        oauth2_access_token,
        connection_kwargs=None,
        files_upload_kwargs=None,
        files_list_folder_kwargs=None,
        rev=None,
    ):

        if connection_kwargs is None:
            connection_kwargs = {}
        if files_upload_kwargs is None:
            files_upload_kwargs = {"mode": WriteMode.overwrite}
        if files_list_folder_kwargs is None:
            files_list_folder_kwargs = {
                "recursive": True,
                "include_non_downloadable_files": False,
            }

        self._prefix = rootdir
        self._con = Dropbox(oauth2_access_token, **connection_kwargs)
        self._connection_kwargs = connection_kwargs
        self._files_upload_kwargs = files_upload_kwargs
        self._files_list_folder_kwargs = files_list_folder_kwargs
        self._rev = rev

    # TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!

    def __iter__(self):
        r = self._con.files_list_folder(self._prefix)
        yield from (x.path_display for x in r.entries)
        cursor = r.cursor
        if r.has_more:
            r = self._con.files_list_folder_continue(cursor)
            yield from (x.path_display for x in r.entries)

    def __getitem__(self, k):
        try:
            metadata, contents_response = self._con.files_download(k)
        except ApiError as e:
            if _is_file_not_found_error(e):
                raise KeyError(f"Key doesn't exist: {k}")
            raise

        if not contents_response.status_code:
            raise ValueError(
                "Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
            )

        return contents_response.content

    def __setitem__(self, k, v):
        return self._con.files_upload(v, k, **self._files_upload_kwargs)

    def __delitem__(self, k):
        return self._con.files_delete_v2(k, self._rev)
Example #33
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None, timeout=None):
        oauth2_access_token = oauth2_access_token or setting(
            'DROPBOX_OAUTH2_TOKEN')
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure an auth token at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")

        self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
        timeout = timeout or setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
        self.client = Dropbox(oauth2_access_token, timeout=timeout)

    def _full_path(self, name):
        if name == '/':
            name = ''
        return safe_join(self.root_path, name).replace('\\', '/')

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)

        if full_path == '/':
            full_path = ''

        metadata = self.client.files_list_folder(full_path)
        for entry in metadata.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)
        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.client_modified

    def url(self, name):
        media = self.client.files_get_temporary_link(self._full_path(name))
        return media.link

    def _open(self, name, mode='rb'):
        remote_file = DropBoxFile(self._full_path(name), self)
        return remote_file

    def _save(self, name, content):
        content.open()
        if content.size <= self.CHUNK_SIZE:
            self.client.files_upload(content.read(), self._full_path(name))
        else:
            self._chunked_upload(content, self._full_path(name))
        content.close()
        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE))
        cursor = UploadSessionCursor(session_id=upload_session.session_id,
                                     offset=content.tell())
        commit = CommitInfo(path=dest_path)

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit)
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor)
                cursor.offset = content.tell()
Example #34
0
class DropboxStorage(Storage):
    """
    A storage class providing access to resources in a Dropbox Public folder.
    """

    def __init__(self, location='/Public'):
        self.client = Dropbox(ACCESS_TOKEN)
        self.account_info = self.client.users_get_current_account()
        self.location = location
        self.base_url = 'https://dl.dropboxusercontent.com/'

    def _get_abs_path(self, name):
        return os.path.realpath(os.path.join(self.location, name))

    def _open(self, name, mode='rb'):
        name = self._get_abs_path(name)
        remote_file = DropboxFile(name, self, mode=mode)
        return remote_file

    def _save(self, name, content):
        name = self._get_abs_path(name)
        directory = os.path.dirname(name)
        if not self.exists(directory) and directory:
            self.client.files_create_folder(directory)
        # response = self.client.files_get_metadata(directory)
        # if not response['is_dir']:
        #     raise IOError("%s exists and is not a directory." % directory)
        abs_name = os.path.realpath(os.path.join(self.location, name))
        foo = self.client.files_upload(content.read(), abs_name)
        return name

    def delete(self, name):
        name = self._get_abs_path(name)
        self.client.files_delete(name)

    def exists(self, name):
        name = self._get_abs_path(name)
        try:
            self.client.files_get_metadata(name)
        except ApiError as e:
            if e.error.is_path() and e.error.get_path().is_not_found():  # not found
                return False
            raise e
        return True

    def listdir(self, path):
        path = self._get_abs_path(path)
        response = self.client.files_list_folder(path)
        directories = []
        files = []
        for entry in response.entries:
            if type(entry) == FolderMetadata:
                directories.append(os.path.basename(entry.path_display))
            elif type(entry) == FileMetadata:
                files.append(os.path.basename(entry.path_display))
        return directories, files

    def size(self, name):
        cache_key = 'django-dropbox-size:{}'.format(filepath_to_uri(name))
        size = cache.get(cache_key)

        if not size:
            size = self.client.files_get_metadata(name).size
            cache.set(cache_key, size, CACHE_TIMEOUT)
        return size

    def url(self, name):
        if name.startswith(self.location):
            name = name[len(self.location) + 1:]

        name = os.path.basename(self.location) + "/" + name

        if self.base_url is None:
            raise ValueError("This file is not accessible via a URL.")

        myurl = urlparse.urljoin(self.base_url, filepath_to_uri(name))

        if "static" not in self.location:
            # Use a dynamic URL for "non-static" files.
            try:
                new_name = os.path.dirname(self.location) + "/" + name
                fp = filepath_to_uri(new_name)
                cache_key = 'django-dropbox-size:{}'.format(fp)
                myurl = cache.get(cache_key)
                if not myurl:
                    try:
                        shared_link = self.client.sharing_create_shared_link(fp)
                        myurl = shared_link.url + '&raw=1'
                        logger.debug("shared link: {0}, myurl: {1}".format(shared_link, myurl))
                    except Exception,e:
                        logger.exception(e)
                    if myurl is None:
                        temp_link = self.client.files_get_temporary_link(fp)
                        myurl = temp_link.link
                        logger.debug("temp link: {0}, myurl: {1}".format(temp_link, myurl))
                    cache.set(cache_key, myurl, SHARE_LINK_CACHE_TIMEOUT)
            except Exception,e:
                logger.exception(e)

        return myurl

        """
Example #35
0
class DPBXBackend(duplicity.backend.Backend):
    """Connect to remote store using Dr*pB*x service"""

    def __init__(self, parsed_url):
        duplicity.backend.Backend.__init__(self, parsed_url)

        self.api_account = None
        self.api_client = None
        self.auth_flow = None

        self.login()

    def load_access_token(self):
        return os.environ.get('DPBX_ACCESS_TOKEN', None)

    def save_access_token(self, access_token):
        raise BackendException('dpbx: Please set DPBX_ACCESS_TOKEN=\"%s\" environment variable' % access_token)

    def obtain_access_token(self):
        log.Info("dpbx: trying to obtain access token")
        for env_var in ['DPBX_APP_KEY', 'DPBX_APP_SECRET']:
            if env_var not in os.environ:
                raise BackendException('dpbx: %s environment variable not set' % env_var)

        app_key = os.environ['DPBX_APP_KEY']
        app_secret = os.environ['DPBX_APP_SECRET']

        if not sys.stdout.isatty() or not sys.stdin.isatty():
            log.FatalError('dpbx error: cannot interact, but need human attention', log.ErrorCode.backend_command_error)

        auth_flow = DropboxOAuth2FlowNoRedirect(app_key, app_secret)
        log.Debug('dpbx,auth_flow.start()')
        authorize_url = auth_flow.start()
        print
        print '-' * 72
        print "1. Go to: " + authorize_url
        print "2. Click \"Allow\" (you might have to log in first)."
        print "3. Copy the authorization code."
        print '-' * 72
        auth_code = raw_input("Enter the authorization code here: ").strip()
        try:
            log.Debug('dpbx,auth_flow.finish(%s)' % auth_code)
            access_token, _ = auth_flow.finish(auth_code)
        except Exception as e:
            raise BackendException('dpbx: Unable to obtain access token: %s' % e)
        log.Info("dpbx: Authentication successfull")
        self.save_access_token(access_token)

    def login(self):
        if self.load_access_token() is None:
            self.obtain_access_token()

        self.api_client = Dropbox(self.load_access_token())
        self.api_account = None
        try:
            log.Debug('dpbx,users_get_current_account([token])')
            self.api_account = self.api_client.users_get_current_account()
            log.Debug("dpbx,%s" % self.api_account)

        except (BadInputError, AuthError) as e:
            log.Debug('dpbx,exception: %s' % e)
            log.Info("dpbx: Authentication failed. Trying to obtain new access token")

            self.obtain_access_token()

            # We're assuming obtain_access_token will throw exception. So this line should not be reached
            raise BackendException("dpbx: Please update DPBX_ACCESS_TOKEN and try again")

        log.Info("dpbx: Successfully authenticated as %s" % self.api_account.name.display_name)

    def _error_code(self, operation, e):
        if isinstance(e, ApiError):
            err = e.error

            if isinstance(err, GetMetadataError) and err.is_path():
                if err.get_path().is_not_found():
                    return log.ErrorCode.backend_not_found
            elif isinstance(err, DeleteError) and err.is_path_lookup():
                lookup = e.error.get_path_lookup()
                if lookup.is_not_found():
                    return log.ErrorCode.backend_not_found

    @command()
    def _put(self, source_path, remote_filename):
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()

        file_size = os.path.getsize(source_path.name)
        f = source_path.open('rb')
        try:
            progress.report_transfer(0, file_size)
            buf = f.read(DPBX_UPLOAD_CHUNK_SIZE)
            log.Debug('dpbx,files_upload_session_start([%d bytes]), total: %d' % (len(buf), file_size))
            upload_sid = self.api_client.files_upload_session_start(buf)
            log.Debug('dpbx,files_upload_session_start(): %s' % upload_sid)
            upload_cursor = UploadSessionCursor(upload_sid.session_id, f.tell())
            commit_info = CommitInfo(remote_path, mode=WriteMode.overwrite, autorename=False, client_modified=None, mute=True)
            res_metadata = None
            progress.report_transfer(f.tell(), file_size)

            requested_offset = None
            current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
            retry_number = globals.num_retries

            # We're doing our own error handling and retrying logic because
            # we can benefit from Dpbx chunked upload and retry only failed chunk
            while (f.tell() < file_size) or not res_metadata:
                try:
                    if requested_offset is not None:
                        upload_cursor.offset = requested_offset

                    if f.tell() != upload_cursor.offset:
                        f.seek(upload_cursor.offset)
                    buf = f.read(current_chunk_size)

                    # reset temporary status variables
                    requested_offset = None
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE
                    retry_number = globals.num_retries

                    if len(buf) != 0:
                        log.Debug('dpbx,files_upload_sesssion_append([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
                        self.api_client.files_upload_session_append(buf, upload_cursor.session_id, upload_cursor.offset)
                    else:
                        log.Debug('dpbx,files_upload_sesssion_finish([%d bytes], offset=%d)' % (len(buf), upload_cursor.offset))
                        res_metadata = self.api_client.files_upload_session_finish(buf, upload_cursor, commit_info)

                    upload_cursor.offset = f.tell()
                    log.Debug('progress: %d of %d' % (upload_cursor.offset, file_size))
                    progress.report_transfer(upload_cursor.offset, file_size)
                except ApiError as e:
                    error = e.error
                    if isinstance(error, UploadSessionLookupError) and error.is_incorrect_offset():
                        # Server reports that we should send another chunk. Most likely this is caused by
                        # network error during previous upload attempt. In such case we'll get expected offset
                        # from server and it's enough to just seek() and retry again
                        new_offset = error.get_incorrect_offset().correct_offset
                        log.Debug('dpbx,files_upload_session_append: incorrect offset: %d (expected: %s)' % (upload_cursor.offset, new_offset))
                        if requested_offset is not None:
                            # chunk failed even after seek attempt. Something strange and no safe way to recover
                            raise BackendException("dpbx: unable to chunk upload")
                        else:
                            # will seek and retry
                            requested_offset = new_offset
                        continue
                    raise
                except ConnectionError as e:
                    log.Debug('dpbx,files_upload_session_append: %s' % e)

                    retry_number -= 1
                    if retry_number == 0:
                        raise

                    # We don't know for sure, was partial upload successfull or not. So it's better to retry smaller amount to avoid extra reupload
                    log.Info('dpbx: sleeping a bit before chunk retry')
                    time.sleep(30)
                    current_chunk_size = DPBX_UPLOAD_CHUNK_SIZE / 5
                    requested_offset = None
                    continue

            if f.tell() != file_size:
                raise BackendException('dpbx: something wrong')

            log.Debug('dpbx,files_upload_sesssion_finish(): %s' % res_metadata)
            progress.report_transfer(f.tell(), file_size)

            # A few sanity checks
            if res_metadata.path_display != remote_path:
                raise BackendException('dpbx: result path mismatch: %s (expected: %s)' % (res_metadata.path_display, remote_path))
            if res_metadata.size != file_size:
                raise BackendException('dpbx: result size mismatch: %s (expected: %s)' % (res_metadata.size, file_size))

        finally:
            f.close()

    @command()
    def _get(self, remote_filename, local_path):
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, remote_filename).rstrip()

        log.Debug('dpbx,files_download(%s)' % remote_path)
        res_metadata, http_fd = self.api_client.files_download(remote_path)
        log.Debug('dpbx,files_download(%s): %s, %s' % (remote_path, res_metadata, http_fd))
        file_size = res_metadata.size
        to_fd = None
        progress.report_transfer(0, file_size)
        try:
            to_fd = local_path.open('wb')
            for c in http_fd.iter_content(DPBX_DOWNLOAD_BUF_SIZE):
                to_fd.write(c)
                progress.report_transfer(to_fd.tell(), file_size)

        finally:
            if to_fd:
                to_fd.close()
            http_fd.close()

        # It's different from _query() check because we're not querying metadata again.
        # Since this check is free, it's better to have it here
        local_size = os.path.getsize(local_path.name)
        if local_size != file_size:
            raise BackendException("dpbx: wrong file size: %d (expected: %d)" % (local_size, file_size))

        local_path.setdata()

    @command()
    def _list(self):
        # Do a long listing to avoid connection reset
        remote_dir = '/' + urllib.unquote(self.parsed_url.path.lstrip('/')).rstrip()

        log.Debug('dpbx.files_list_folder(%s)' % remote_dir)
        resp = self.api_client.files_list_folder(remote_dir)
        log.Debug('dpbx.list(%s): %s' % (remote_dir, resp))

        res = []
        while True:
            res.extend([entry.name for entry in resp.entries])
            if not resp.has_more:
                break
            resp = self.api_client.files_list_folder_continue(resp.cursor)

        # Warn users of old version dpbx about automatically renamed files
        self.check_renamed_files(res)

        return res

    @command()
    def _delete(self, filename):
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, filename).rstrip()

        log.Debug('dpbx.files_delete(%s)' % remote_path)
        self.api_client.files_delete(remote_path)

        # files_permanently_delete seems to be better for backup purpose
        # but it's only available for Business accounts
        # self.api_client.files_permanently_delete(remote_path)

    @command()
    def _close(self):
        """close backend session? no! just "flush" the data"""
        log.Debug('dpbx.close():')

    @command()
    def _query(self, filename):
        remote_dir = urllib.unquote(self.parsed_url.path.lstrip('/'))
        remote_path = '/' + os.path.join(remote_dir, filename).rstrip()

        log.Debug('dpbx.files_get_metadata(%s)' % remote_path)
        info = self.api_client.files_get_metadata(remote_path)
        log.Debug('dpbx.files_get_metadata(%s): %s' % (remote_path, info))
        return {'size': info.size}

    def check_renamed_files(self, file_list):
        bad_list = [x for x in file_list if DPBX_AUTORENAMED_FILE_RE.search(x) is not None]
        if len(bad_list) == 0:
            return
        log.Warn('-' * 72)
        log.Warn('Warning! It looks like there are automatically renamed files on backend')
        log.Warn('They were probably created when using older version of duplicity.')
        log.Warn('')
        log.Warn('Please check your backup consistency. Most likely you will need to choose')
        log.Warn('largest file from duplicity-* (number).gpg and remove brackets from its name.')
        log.Warn('')
        log.Warn('These files are not managed by duplicity at all and will not be')
        log.Warn('removed/rotated automatically.')
        log.Warn('')
        log.Warn('Affected files:')
        for x in bad_list:
            log.Warn('\t%s' % x)
        log.Warn('')
        log.Warn('In any case it\'s better to create full backup.')
        log.Warn('-' * 72)
Example #36
0
class DropboxConnection(object):
    class Parameters(object):
        def __init__(self, auth_token, remote_dir_path):
            self._auth_token = auth_token
            self._remote_dir_path = remote_dir_path

        @property
        def auth_token(self):
            return self._auth_token

        @property
        def remote_dir_path(self):
            return constants.DROPBOX_APP_PATH_PREFIX / self._remote_dir_path

        def __str__(self):
            return """
                        remote_dir_path: {}
                   """.format(self.remote_dir_path)

    @classmethod
    def get_client_from_params(cls, dropbox_parameters):
        #Dropbox connection placeholder
        dropbox = None

        if dropbox_parameters:
            dropbox_params = DropboxConnection.Parameters(
                dropbox_parameters[0], dropbox_parameters[1])
            dropbox = DropboxConnection(dropbox_params)

        return dropbox

    @classmethod
    def get_client(cls, auth_token, remote_dir_path):
        #Initialize the paramters
        params = DropboxConnection.Parameters(auth_token, remote_dir_path)

        #Create dropbox client
        client = DropboxConnection(params)

        return client

    def __init__(self, params):
        #Required parameters
        self._params = params

        #Derived parameters
        self._client = Dropbox(self._params.auth_token)

        #Logging
        self._logger = logging.get_logger(__name__)

    def upload(self, source_file_path):
        """It upload the source files to the dropbox.

        Arguments:
            source_file_path {string} -- The source file path.

        Raises:
            ValueError -- It raise value error for invalid files.
        """

        #Validate parameters
        if not source_file_path:
            raise ValueError("Invalid source file path")

        with open(source_file_path, 'rb') as handle:
            #Source file name
            source_file_name = Path(source_file_path).name

            #Remote path
            remote_file_path = (self._params.remote_dir_path /
                                source_file_name).as_posix()

            #File size
            upload_size = path.getsize(source_file_path)

            #Upload the files based on the upload size
            if upload_size <= constants.DROPBOX_CHUNK_SIZE:
                self._logger.info(
                    'Preparing to upload small file: %s with size: %d to: %s',
                    source_file_path, upload_size, remote_file_path)

                self._upload_small_file(handle, remote_file_path)
            else:
                self._logger.info(
                    'Preparing to upload large file: %s with size: %d to: %s',
                    source_file_path, upload_size, remote_file_path)

                self._upload_large_file(handle, upload_size, remote_file_path)

            self._logger.info('Uploaded: %s', source_file_path)

    def _upload_small_file(self, handle, remote_file_path):
        """It uploads a small source files to the dropbox.

        Arguments:
            handle {A File handle} -- The source file handle.
            remote_file_path {string} -- The destination path of the file.
        """
        self._client.files_upload(handle.read(),
                                  remote_file_path,
                                  mode=Dropbox_WriteMode.overwrite)

    def _upload_large_file(self, handle, upload_size, remote_file_path):
        """It uploads a large source files to the dropbox.

        Arguments:
            handle {A File handle} -- The source file handle.
            upload_size {int} -- The number of bytes to be uploaded.
            remote_file_path {string} -- The destination path of the file.
        """
        #Upload session
        session = self._client.files_upload_session_start(
            handle.read(constants.DROPBOX_CHUNK_SIZE))
        cursor = Dropbox_UploadSessionCursor(session_id=session.session_id,
                                             offset=handle.tell())

        #Upload look
        with tqdm(desc='Uploading: {}'.format(remote_file_path),
                  total=upload_size) as pbar:
            #Update the progress bar for the session start reads
            pbar.update(handle.tell())

            while handle.tell() < upload_size:
                #Calculate remaining bytes
                remaining_bytes = upload_size - handle.tell()

                #If it is the last chunk, finalize the upload
                if remaining_bytes <= constants.DROPBOX_CHUNK_SIZE:
                    #Commit info
                    commit = Dropbox_CommitInfo(
                        path=remote_file_path,
                        mode=Dropbox_WriteMode.overwrite)

                    #Finish upload
                    self._client.files_upload_session_finish(
                        handle.read(remaining_bytes), cursor, commit)

                    #Update progress
                    pbar.update(remaining_bytes)
                #More than chunk size remaining to upload
                else:
                    self._client.files_upload_session_append_v2(
                        handle.read(constants.DROPBOX_CHUNK_SIZE), cursor)

                    #Update the cursor
                    cursor.offset = handle.tell()

                    #Update the progress
                    pbar.update(constants.DROPBOX_CHUNK_SIZE)

                #Refresh the progress bar
                pbar.refresh()

    def download(self, remote_file_path):
        """It downloads the remote files from the dropbox.

        Arguments:
            remote_file_path {Path} -- The path to the remote file.

        Raises:
            ValueError -- It raise value error for invalid file name.
        """
        #Validate parameters
        if not remote_file_path:
            raise ValueError("Invalid remote file path")

        #Destination file path
        dest_file_path = remote_file_path

        #Full remote file path
        remote_file_path = self._params.remote_dir_path / remote_file_path

        #Download file size placeholder
        download_size = 0

        try:
            download_size = self._client.files_get_metadata(
                remote_file_path.as_posix()).size
        except ApiError as e:
            raise FileNotFoundError(
                'File: {} is not found'.format(remote_file_path.as_posix()), e)

        self._logger.info('Preparing file download: %s with size: %d to: %s',
                          remote_file_path, download_size, dest_file_path)

        #Download the file
        self._download_file(dest_file_path, remote_file_path, download_size)

        self._logger.info('Completed the file download: %s to: %s',
                          remote_file_path, dest_file_path)

    def _download_file(self, dest_file_path, remote_file_path, download_size):
        """It downloads the remote files from the dropbox.

        Arguments:
            remote_file_path {A Path object} -- The path of the remote file.
            dest_file_path {string} -- The destination file path.
            download_size {int} -- The number of bytes to be downloaded.
        """
        #Download
        _, result = self._client.files_download(remote_file_path.as_posix())

        #Temporary dest_file_name
        tmp_dest_file_path = "{}.tmp".format(dest_file_path)

        with open(tmp_dest_file_path, 'wb') as handle:
            with tqdm(desc='Downloading: {}'.format(
                    remote_file_path.as_posix()),
                      total=download_size) as pbar:
                for bytes_read in result.iter_content(
                        constants.DROPBOX_CHUNK_SIZE):
                    handle.write(bytes_read)

                    #Update the progress
                    pbar.update(len(bytes_read))

        if Path(tmp_dest_file_path).exists():
            rename(tmp_dest_file_path, dest_file_path)

    def list(self, dir_path=Path(), file_name_prefix=''):
        """It lists the files in the dropbox folder that starts with the given prefix.

        Arguments:
            file_name_prefix {string} -- The prefix to filter the results.
        """
        #Candidate directory whose contents are to be listed
        candidate_dir_path = self._params.remote_dir_path / dir_path
        self._logger.info('Enumerating: %s with file_name_prefix: %s',
                          candidate_dir_path, file_name_prefix)

        #Call the downstream API
        response = self._client.files_list_folder(
            candidate_dir_path.as_posix())

        #Output list placeholder
        files = []
        sizes = []

        if response.entries:
            #Log the response summary
            self._logger.info('Got %d files in: %s', len(response.entries),
                              candidate_dir_path)

            #Extract the name of files satisfying the input criteria from the response entries.
            file_infos = [(dir_path / entry.name, entry.size)
                          for entry in response.entries
                          if entry.name.startswith(file_name_prefix)]

            files, sizes = zip(*file_infos)

        return files, sizes
Example #37
0
class DropboxFS(FS):
    def __init__(self, accessToken):
        super().__init__()
        self.dropbox = Dropbox(accessToken)
        _meta = self._meta = {
            "case_insensitive": False,  # I think?
            "invalid_path_chars": ":",  # not sure what else
            "max_path_length": None,  # don't know what the limit is
            "max_sys_path_length": None,  # there's no syspath
            "network": True,
            "read_only": False,
            "supports_rename": False  # since we don't have a syspath...
        }

    def __repr__(self):
        return "<DropboxDriveFS>"

    def _infoFromMetadata(self, metadata):  # pylint: disable=no-self-use
        rawInfo = {
            "basic": {
                "name": metadata.name,
                "is_dir": isinstance(metadata, FolderMetadata),
            }
        }
        if isinstance(metadata, FileMetadata):
            rawInfo.update({
                "details": {
                    "accessed": None,  # not supported by Dropbox API
                    "created": None,  # not supported by Dropbox API?,
                    "metadata_changed": None,  # not supported by Dropbox
                    "modified": datetime_to_epoch(
                        metadata.server_modified
                    ),  # API documentation says that this is reliable
                    "size": metadata.size,
                    "type": 0
                },
                "dropbox": {
                    "content_hash": metadata.
                    content_hash,  # see https://www.dropbox.com/developers/reference/content-hash
                    "rev": metadata.rev,
                    "client_modified": metadata.
                    client_modified  # unverified value coming from dropbox clients
                }
            })
            if metadata.media_info is not None and metadata.media_info.is_metadata(
            ) is True:
                media_info_metadata = metadata.media_info.get_metadata()
                if media_info_metadata.time_taken is not None:
                    rawInfo.update({
                        "media_info": {
                            "taken_date_time":
                            datetime_to_epoch(media_info_metadata.time_taken)
                        }
                    })
                if media_info_metadata.location is not None:
                    rawInfo.update({
                        "media_info": {
                            "location_latitude":
                            media_info_metadata.location.latitude,
                            "location_longitude":
                            media_info_metadata.location.longitude
                        }
                    })
                # Dropbox doesn't parse some jpgs properly
                if media_info_metadata.dimensions is not None:
                    rawInfo.update({
                        "media_info": {
                            "dimensions_height":
                            media_info_metadata.dimensions.height,
                            "dimensions_width":
                            media_info_metadata.dimensions.width
                        }
                    })
        elif isinstance(metadata, FolderMetadata):
            rawInfo.update({
                "details": {
                    "accessed": None,  # not supported by Dropbox API
                    "created": None,  # not supported by Dropbox API,
                    "metadata_changed": None,  # not supported by Dropbox
                    "modified": None,  # not supported for folders
                    "size": None,  # not supported for folders
                    "type": 1
                }
            })
        else:
            assert False, f"{metadata.name}, {metadata}, {type(metadata)}"
        return Info(rawInfo)

    def getinfo(self, path, namespaces=None):
        if path == "/":
            return Info({"basic": {"name": "", "is_dir": True}})
        try:
            if not path.startswith("/"):
                path = "/" + path
            metadata = self.dropbox.files_get_metadata(path,
                                                       include_media_info=True)
        except ApiError as e:
            raise ResourceNotFound(path=path, exc=e)
        return self._infoFromMetadata(metadata)

    def setinfo(self, path, info):  # pylint: disable=too-many-branches
        # dropbox doesn't support changing any of the metadata values
        pass

    def listdir(self, path):
        return [x.name for x in self.scandir(path)]

    def makedir(self, path, permissions=None, recreate=False):
        try:
            folderMetadata = self.dropbox.files_create_folder(path)
        except ApiError as e:
            assert isinstance(e.reason, CreateFolderError)
            # TODO - there are other possibilities
            raise DirectoryExpected(path=path)
        # don't need to close this filesystem so we return the non-closing version
        return SubFS(self, path)

    def openbin(self, path, mode="r", buffering=-1, **options):
        mode = Mode(mode)
        exists = True
        isDir = False
        try:
            isDir = self.getinfo(path).is_dir
        except ResourceNotFound:
            exists = False
        if mode.exclusive and exists:
            raise FileExists(path)
        elif mode.reading and not mode.create and not exists:
            raise ResourceNotFound(path)
        elif isDir:
            raise FileExpected(path)
        return DropboxFile(self.dropbox, path, mode)

    def remove(self, path):
        try:
            self.dropbox.files_delete(path)
        except ApiError as e:
            raise FileExpected(path=path, exc=e)

    def removedir(self, path):
        try:
            self.dropbox.files_delete(path)
        except ApiError as e:
            assert e.reason is DeleteError
            raise DirectoryExpected(path=path, exc=e)

    # non-essential method - for speeding up walk
    def scandir(self, path, namespaces=None, page=None):
        #
        if path == "/":
            path = ""
        # get all the avaliable metadata since it's cheap
        # TODO - this call has a recursive flag so we can either use that and cache OR override walk
        result = self.dropbox.files_list_folder(path, include_media_info=True)
        allEntries = result.entries
        while result.has_more:
            result = self.dropbox.files_list_folder_continue(result.cursor)
            allEntries += result.entries
        return [self._infoFromMetadata(x) for x in allEntries]
Example #38
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""
    location = setting('DROPBOX_ROOT_PATH', '/')
    oauth2_access_token = setting('DROPBOX_OAUTH2_TOKEN')
    timeout = setting('DROPBOX_TIMEOUT', _DEFAULT_TIMEOUT)
    write_mode = setting('DROPBOX_WRITE_MODE', _DEFAULT_MODE)

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=oauth2_access_token, root_path=location, timeout=timeout,
                 write_mode=write_mode):
        if oauth2_access_token is None:
            raise ImproperlyConfigured("You must configure an auth token at"
                                       "'settings.DROPBOX_OAUTH2_TOKEN'.")

        self.root_path = root_path
        self.write_mode = write_mode
        self.client = Dropbox(oauth2_access_token, timeout=timeout)

    def _full_path(self, name):
        if name == '/':
            name = ''
        
        # If the machine is windows do not append the drive letter to file path
        if os.name == 'nt':
            final_path = os.path.join(self.root_path, name).replace('\\', '/')
            
            # Separator on linux system
            sep = '//'
            base_path = self.root_path

            if (not os.path.normcase(final_path).startswith(os.path.normcase(base_path + sep)) and
                    os.path.normcase(final_path) != os.path.normcase(base_path) and
                    os.path.dirname(os.path.normcase(base_path)) != os.path.normcase(base_path)):
                raise SuspiciousFileOperation(
                    'The joined path ({}) is located outside of the base path '
                    'component ({})'.format(final_path, base_path))
            
            return final_path
        
        else:
            return safe_join(self.root_path, name).replace('\\', '/')

    def delete(self, name):
        self.client.files_delete(self._full_path(name))

    def exists(self, name):
        try:
            return bool(self.client.files_get_metadata(self._full_path(name)))
        except ApiError:
            return False

    def listdir(self, path):
        directories, files = [], []
        full_path = self._full_path(path)

        if full_path == '/':
            full_path = ''

        metadata = self.client.files_list_folder(full_path)
        for entry in metadata.entries:
            if isinstance(entry, FolderMetadata):
                directories.append(entry.name)
            else:
                files.append(entry.name)
        return directories, files

    def size(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.size

    def modified_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.server_modified

    def accessed_time(self, name):
        metadata = self.client.files_get_metadata(self._full_path(name))
        return metadata.client_modified

    def url(self, name):
        media = self.client.files_get_temporary_link(self._full_path(name))
        return media.link

    def _open(self, name, mode='rb'):
        remote_file = DropBoxFile(self._full_path(name), self)
        return remote_file

    def _save(self, name, content):
        content.open()
        if content.size <= self.CHUNK_SIZE:
            self.client.files_upload(content.read(), self._full_path(name), mode=WriteMode(self.write_mode))
        else:
            self._chunked_upload(content, self._full_path(name))
        content.close()
        return name

    def _chunked_upload(self, content, dest_path):
        upload_session = self.client.files_upload_session_start(
            content.read(self.CHUNK_SIZE)
        )
        cursor = UploadSessionCursor(
            session_id=upload_session.session_id,
            offset=content.tell()
        )
        commit = CommitInfo(path=dest_path, mode=WriteMode(self.write_mode))

        while content.tell() < content.size:
            if (content.size - content.tell()) <= self.CHUNK_SIZE:
                self.client.files_upload_session_finish(
                    content.read(self.CHUNK_SIZE), cursor, commit
                )
            else:
                self.client.files_upload_session_append_v2(
                    content.read(self.CHUNK_SIZE), cursor
                )
                cursor.offset = content.tell()

    def get_available_name(self, name, max_length=None):
        """Overwrite existing file with the same name."""
        name = self._full_path(name)
        if self.write_mode == 'overwrite':
            return get_available_overwrite_name(name, max_length)
        return super().get_available_name(name, max_length)
Example #39
0
class DropboxPersister(Persister):
    """
    A persister for dropbox.
    You need to have the python connector (if you don't: pip install dropbox)
    You also need to have a token for your dropbox app. If you don't it's a google away.
    Finally, for the test below, you need to put this token in ~/.py2store_configs.json' under key
    dropbox.__init__kwargs, and have a folder named /py2store_data/test/ in your app space.

    >>> import json
    >>> import os
    >>>
    >>> configs = json.load(open(os.path.expanduser('~/.py2store_configs.json')))
    >>> s = DropboxPersister('/py2store_data/test/', **configs['dropbox']['__init__kwargs'])
    >>> if '/py2store_data/test/_can_remove' in s:
    ...     del s['/py2store_data/test/_can_remove']
    ...
    >>>
    >>> n = len(s)
    >>> if n == 1:
    ...     assert list(s) == ['/py2store_data/test/_can_remove']
    ...
    >>> s['/py2store_data/test/_can_remove'] = b'this is a test'
    >>> assert len(s) == n + 1
    >>> assert s['/py2store_data/test/_can_remove'] == b'this is a test'
    >>> '/py2store_data/test/_can_remove' in s
    True
    >>> del s['/py2store_data/test/_can_remove']
    """
    def __init__(self,
                 rootdir,
                 oauth2_access_token,
                 connection_kwargs=None,
                 files_upload_kwargs=None,
                 files_list_folder_kwargs=None,
                 rev=None):

        if connection_kwargs is None:
            connection_kwargs = {}
        if files_upload_kwargs is None:
            files_upload_kwargs = {'mode': WriteMode.overwrite}
        if files_list_folder_kwargs is None:
            files_list_folder_kwargs = {
                'recursive': True,
                'include_non_downloadable_files': False
            }

        self._prefix = rootdir
        self._con = Dropbox(oauth2_access_token, **connection_kwargs)
        self._connection_kwargs = connection_kwargs
        self._files_upload_kwargs = files_upload_kwargs
        self._files_list_folder_kwargs = files_list_folder_kwargs
        self._rev = rev

    # TODO: __len__ is taken from Persister, which iterates and counts. Not efficient. Find direct api for this!

    def __iter__(self):
        r = self._con.files_list_folder(self._prefix)
        yield from (x.path_display for x in r.entries)
        cursor = r.cursor
        if r.has_more:
            r = self._con.files_list_folder_continue(cursor)
            yield from (x.path_display for x in r.entries)

    def __getitem__(self, k):
        try:
            metadata, contents_response = self._con.files_download(k)
        except ApiError as err:
            if _is_file_not_found_error(err):
                raise KeyError(f"Key doesn't exist: {k}")
            else:
                raise ValueError(
                    "Some unknown error happened (sorry, the lazy dev didn't tell me more than that)."
                )
        if contents_response.status_code:
            return contents_response.content
        else:
            raise ValueError(
                "Response code wasn't 200 when trying to download a file (yet the file seems to exist)."
            )

    def __setitem__(self, k, v):
        return self._con.files_upload(v, k, **self._files_upload_kwargs)

    def __delitem__(self, k):
        return self._con.files_delete_v2(k, self._rev)


# def _entry_is_dir(entry):
#     return not hasattr(entry, 'is_downloadable')
#
#
# def _entry_is_file(entry):
#     return hasattr(entry, 'is_downloadable')
#
#
# def _extend_path(path, extension):
#     extend_path = '/' + path + '/' + extension + '/'
#     extend_path.replace('//', '/')
#     return extend_path
#
#
# class DropboxLinkPersister(DropboxPersister):
#     def __init__(self, url, oauth2_access_token):
#         self._con = Dropbox(oauth2_access_token)
#         self.url = url
#         self.shared_link = SharedLink(url=url)
#
#     def _yield_from_files_list_folder(self, path, path_gen):
#         """
#         yield paths from path_gen, which can be a files_list_folder or a files_list_folder_continue,
#         in a depth search manner.
#         """
#         for x in path_gen.entries:
#             try:
#                 if _entry_is_file(x):
#                     yield x.name
#                 else:
#                     folder_path = _extend_path(path, x.name)
#                     yield from self._get_path_gen_from_path(path=folder_path)
#             except Exception as e:
#                 print(e)
#         if path_gen.has_more:
#             yield from self._get_path_gen_from_cursor(path_gen.cursor, path=path)
#
#     def _get_path_gen_from_path(self, path):
#         path_gen = self._con.files_list_folder(path=path, recursive=False, shared_link=self.shared_link)
#         yield from self._yield_from_files_list_folder(path, path_gen)
#
#     def _get_path_gen_from_cursor(self, cursor, path):
#         path_gen = self._con.files_list_folder_continue(cursor)
#         yield from self._yield_from_files_list_folder(path, path_gen)
#
#     def __iter__(self):
#         yield from self._get_path_gen_from_path(path='')
Example #40
0
class rpiImageDbxClass(rpiBaseClass):
	"""
	Implements the rpiImageDb class to manage images in a remote directory (dropbox).
	"""

	def __init__(self, name, rpi_apscheduler, rpi_events, rpi_config, cam_rpififo=None):

		### Get the Dbx error event
		#self._eventDbErr 	= rpi_events.eventErrList["DBXJob"]

		### Get the custom config parameters
		self._config = rpi_config

		### Get FIFO buffer for images from the camera (deque)
		self._imageFIFO = cam_rpififo

		### The FIFO buffer for the uploaded images (deque)
		self.imageUpldFIFO = rpififo.rpiFIFOClass([], 576)
		self.imageUpldFIFO.crtSubDir = ''

		### Init base class
		super().__init__(name, rpi_apscheduler, rpi_events)

	def __repr__(self):
		return "<%s (name=%s, rpi_apscheduler=%s, rpi_events=dict(), rpi_config=%s, dbuff_rpififo=%s)>" % (self.__class__.__name__, self.name, self._sched, self._config, self._imageFIFO)

	def __str__(self):
		msg = super().__str__()
		return "%s::: dbinfo: %s, config: %s\nimageUpldFIFO: %s\n%s" % \
				(self.name, self.dbinfo, self._config, self.imageUpldFIFO, msg)

	def __del__(self):
		### Clean base class
		super().__del__()



	#
	# Main interface methods
	#

	def jobRun(self):

		try:
			# Lock the buffer
			self._imageFIFO.acquireSemaphore()

			# Get the current images in the FIFO
			# Refresh the last remote image when available
			if len(self._imageFIFO):

				# Update remote cam image with the current (last) image
				if not (self._imageFIFO[-1] == self.crt_image_snap):
					self._putImage(self._imageFIFO[-1], self._config['image_snap'], True)
					self.crt_image_snap = self._imageFIFO[-1]
					self.numImgUpdDb += 1
					logging.info("Updated remote %s with %s" % (self._config['image_snap'], self._imageFIFO[-1]) )


				# Lock the upload buffer
				self.imageUpldFIFO.acquireSemaphore()

				# Check if a new upload sub-folder has to be used
				if not (self.imageUpldFIFO.crtSubDir == self._imageFIFO.crtSubDir):
					self.imageUpldFIFO.crtSubDir = self._imageFIFO.crtSubDir
					self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
					self._mkdirImage(self.upldir)

				# Upload only images in the FIFO which have not been uploaded yet
				for img in self._imageFIFO:
					if not img in self.imageUpldFIFO:
						self._putImage(img, os.path.join(self.upldir, os.path.basename(img)))
						logging.info("Uploaded %s" % img )

				# Release the upload buffer
				self.imageUpldFIFO.releaseSemaphore()

				# Update status
				self.statusUpdate = (self.name, self.numImgUpdDb)

			else:
				# Update status
				self.statusUpdate = (self.name, ERRNONE)

				logging.info('Nothing to upload')


		# Handle exceptions, mostly HTTP/SSL related!
		except exceptions.Timeout as e:
			# Catching this error will catch both ReadTimeout and ConnectTimeout.
			raise rpiBaseClassError("%s::: jobRun(): Connect/ReadTimeoutError:\n%s" % (self.name, str(e)), ERRLEV2)

		except exceptions.ConnectionError as e:
			# A Connection error occurred.
			raise rpiBaseClassError("%s::: jobRun(): ConnectionError:\n%s" % (self.name, str(e)), ERRLEV2)

		except exceptions.HTTPError as e:
			# An HTTP error occurred.
			raise rpiBaseClassError("%s::: jobRun(): HTTPError:\n%s" % (self.name, str(e)), ERRLEV2)

		except exceptions.RequestException as e:
			# There was an ambiguous exception that occurred while handling your request.
			raise rpiBaseClassError("%s::: jobRun(): RequestException:\n%s" % (self.name, str(e)), ERRLEV2)

# 			except BadStatusLine as e:
# 				self.eventErr_set('run()')
# 				logging.debug("BadStatusLine:\n%s" % str(e))
# 				pass

		except rpiBaseClassError as e:
			if e.errval == ERRCRIT:
				self.endDayOAM()
			raise rpiBaseClassError("%s::: jobRun(): %s" % (self.name, e.errmsg), e.errval)

		except RuntimeError as e:
			self.endDayOAM()
			raise rpiBaseClassError("%s::: jobRun(): RuntimeError:\n%s" % (self.name, str(e)), ERRCRIT)

		except:
			self.endDayOAM()
			raise rpiBaseClassError("%s::: jobRun(): Unhandled Exception:\n%s" % (self.name, str(sys.exc_info())), ERRCRIT)

		finally:
			# Release the buffer
			self._imageFIFO.releaseSemaphore()


	def initClass(self):
		""""
		(re)Initialize the class.
		"""

		#self.imageDbHash = None
		self._imageDbCursor = None
		self.imageDbList = []
		self.numImgUpdDb = 0

		self.crt_image_snap = None
		self.imgid = self._imageFIFO.camID + '.jpg'
		self.upldir = os.path.normpath(os.path.join(self._config['image_dir'], self.imageUpldFIFO.crtSubDir))
		self.logfile = './upldlog.json'

		### When there are already images listed in the upload log file, then
		# make sure we don't upload them to the remote folder again
		# Else, create the file with an empty list; to be updated in endDayOAM()
		try:
			self.imageUpldFIFO.acquireSemaphore()
			self.imageUpldFIFO.clear()

			if os.path.isfile(self.logfile):
				with open(self.logfile,'r') as logf:
					upldimg = json.load(logf)

				for img in upldimg:
					self.imageUpldFIFO.append(img)

				del upldimg

				logging.info("%s::: Local log file %s found and loaded." % (self.name, self.logfile))
			else:
				with open(self.logfile,'w') as logf:
					json.dump([], logf)
					logging.info("%s::: Local log file %s initialized." % (self.name, self.logfile))

		except IOError:
			raise rpiBaseClassError("%s::: initClass(): Local log file %s was not found or could not be created." % (self.name, self.logfile), ERRCRIT)

		finally:
			# Release the upload buffer
			self.imageUpldFIFO.releaseSemaphore()

		### Init Dropbox API client
		self._token_file = self._config['token_file']
		self._dbx = None
		self.dbinfo = None
		try:
			with open(self._token_file, 'r') as token:
				self._dbx = Dropbox(token.read())

			info = self._dbx.users_get_current_account()
			# info._all_field_names_ =
			# {'account_id', 'is_paired', 'locale', 'email', 'name', 'team', 'country', 'account_type', 'referral_link'}
			self.dbinfo ={'email': info.email, 'referral_link': info.referral_link}

			logging.info("%s::: Loaded access token from ''%s''" % (self.name, self._token_file) )

			### Create remote root folder (relative to app root) if it does not exist yet
			self._mkdirImage(os.path.normpath(self._config['image_dir']))

		except rpiBaseClassError as e:
			if e.errval == ERRCRIT:
				self.endDayOAM()
			raise rpiBaseClassError("initClass(): %s" % e.errmsg, e.errval)

		except IOError:
			self.endDayOAM()
			raise rpiBaseClassError("initClass(): Token file ''%s'' could not be read." % (self.name, self._token_file), ERRCRIT)

		except AuthError as e:
			self.endDayOAM()
			raise rpiBaseClassError("initClass(): AuthError:\n%s" % e.error, ERRCRIT)

		except DropboxException as e:
			self.endDayOAM()
			raise rpiBaseClassError("initClass(): DropboxException:\n%s" %  str(e), ERRCRIT)

		except InternalServerError as e:
			self.endDayOAM()
			raise rpiBaseClassError("initClass(): InternalServerError:\n%s" % str(e.status_code),  ERRCRIT)


	def endDayOAM(self):
		"""
		End-of-Day Operation and Maintenance sequence.
		"""

		self._lsImage(self.upldir)
		logging.info("%s::: %d images in the remote folder %s" % (self.name, len(self.imageDbList), self.upldir))

		# Lock the uplaod buffer
		self.imageUpldFIFO.acquireSemaphore()

		try:
			upldimg=[]
			for img in self.imageUpldFIFO:
				upldimg.append(img)

			with open(self.logfile,'w') as logf:
				json.dump(upldimg, logf)

			del upldimg

			logging.info("%s::: Local log file %s updated." % (self.name, self.logfile))

		except IOError:
			raise rpiBaseClassError("endDayOAM(): Local log file %s was not found." % self.logfile,  ERRCRIT)

		finally:
			# Release the upload buffer
			self.imageUpldFIFO.releaseSemaphore()

#	def endOAM(self):
#		"""
#		End OAM procedure.
#		"""
	@atexit.register
	def atexitend():
		self.endDayOAM()

	def _lsImage(self,from_path):
		"""
		List the image/video files in the remote directory.
		Stores the found file names in self.imageDbList.
		"""
		try:
			if self._imageDbCursor is None:
				self.ls_ref = self._dbx.files_list_folder('/' + os.path.normpath(from_path), recursive=False, include_media_info=True )
			else:
				new_ls = self._dbx.files_list_folder_continue(self._imageDbCursor)
				if new_ls.entries == []:
					logging.debug("%s::: _lsImage():: No changes on the server." % self.name)
				else:
					self.ls_ref = new_ls

			# Select only images and only the ones for the current imgid (camid)
			foundImg = False
			for f in self.ls_ref.entries:
				if 'media_info' in f._all_field_names_ and \
					f.media_info is not None:
					if self.imgid in f.path_lower:
						img = '.%s' % f.path_lower
						foundImg = True
						if not img in self.imageDbList:
							self.imageDbList.append(img)


			if not foundImg:
				self.imageDbList = []

			### Store the hash of the folder
			self._imageDbCursor = self.ls_ref.cursor

			if len(self.imageDbList) > 0:
				logging.debug("%s::: _lsImage():: imageDbList[0..%d]: %s .. %s" % (self.name, len(self.imageDbList)-1, self.imageDbList[0], self.imageDbList[-1]) )
			else:
				logging.debug("%s::: _lsImage():: imageDbList[]: empty" % self.name)

		except ApiError as e:
			raise rpiBaseClassError("_lsImage(): %s" % e.error, ERRLEV2)


	def _putImage(self, from_path, to_path, overwrite=False):
		"""
		Copy local file to remote file.
		Stores the uploaded files names in self.imageUpldFIFO.

		Examples:
		_putImage('./path/test.jpg', '/path/dropbox-upload-test.jpg')
		"""
		try:
			mode = (WriteMode.overwrite if overwrite else WriteMode.add)

			with open(from_path, "rb") as from_file:
				self._dbx.files_upload( from_file, '/' + os.path.normpath(to_path), mode)

			if not overwrite:
				self.imageUpldFIFO.append(from_path)

			logging.debug("%s::: _putImage(): Uploaded file from %s to remote %s" % (self.name, from_path, to_path))

		except IOError:
			raise rpiBaseClassError("_putImage(): Local img file %s could not be opened." %  from_path, ERRCRIT)

		except ApiError as e:
			raise rpiBaseClassError("_putImage(): %s" % e.error, ERRLEV2)


	def _mkdirImage(self, path):
		"""
		Create a new remote directory.

		Examples:
		_mkdirImage('/dropbox_dir_test')
		"""
		try:
			self._dbx.files_create_folder('/' + os.path.normpath(path))

			logging.debug("%s::: Remote output folder /%s created." % (self.name, path))

		except ApiError as e:
			noerr = False
			# dropbox.files.CreateFolderError
			if e.error.is_path():
				# dropbox.files.WriteError
				we = e.error.get_path()
				if we.is_conflict():
					# dropbox.files.WriteConflictError
					wce = we.get_conflict()
					# union tag is 'folder'
					if wce.is_folder():
						logging.info("%s::: Remote output folder /%s already exist!" % (self.name, path))
						noerr = True

			if not noerr:
				raise rpiBaseClassError("_mkdirImage(): Remote output folder /%s was not created! %s" % (path, e.error), ERRCRIT)
			else:
				pass


	def _mvImage(self, from_path, to_path):
		"""
		Move/rename a remote file or directory.

		Examples:
		_mvImage('./path1/dropbox-move-test.jpg', '/path2/dropbox-move-test.jpg')
		"""
		try:
			self._dbx.files_move( '/' + os.path.normpath(from_path), '/' +  os.path.normpath(to_path) )

			logging.debug("%s::: _mvImage(): Moved file from %s to %s" % (self.name, from_path, to_path))

		except ApiError as e:
			raise rpiBaseClassError("_mvImage(): Image %s could not be moved to %s! %s" % (from_path, to_path, e.error), ERRLEV2)
Example #41
0
 def test_dbx(self):
     dbx = Dropbox(settings.DROPBOX_ACCESS_TOKEN)
     result = dbx.files_list_folder(path='/roasts')
     print(result)
Example #42
0
class DropBoxStorage(Storage):
    """DropBox Storage class for Django pluggable storage system."""

    CHUNK_SIZE = 4 * 1024 * 1024

    def __init__(self, oauth2_access_token=None, root_path=None):
        oauth2_access_token = DROPBOX_OAUTH2_TOKEN
        self.root_path = DROPBOX_ROOT_PATH
        if oauth2_access_token is None:
            raise ImproperlyConfigured("Você deve configurar um token em DROPBOX_OAUTH2_TOKEN ou em settings.py")
        self.dbx = Dropbox(oauth2_access_token)

    def user_profile(self):
        self.dt = self.dbx.users_get_current_account()
        print(self.dt)

    def list_dir_and_files_all(self):
        try:
            self.dt = self.dbx.files_list_folder(self.root_path)
            print('DIRETÓRIOS\n')
            self.list_subdirs(self.dt)
        except:
            self.dt = self.dbx.files_get_metadata(self.root_path)
            print('ARQUIVOS\n')
            if (isinstance(self.dt, dropbox.files.FileMetadata)):
                print('FUNCIONA')
            file = self.list_files(self.dt)
            return file

    def list_subdirs(self, dt):
        for entry in self.dt.entries:
            i = entry.path_display
            print(i)

    def list_files(self):
        self.dt = self.dbx.files_get_metadata(self.root_path)
        file = self.dt.path_display
        print(file)
        return file

    def upload_file(self):
        #print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW)
        time = datetime.datetime.now()
        time = time.strftime("%Y%m%d%H%M%S")
        FILEPATH = self.simple_backup()



        with open(FILEPATH, 'rb') as f:
            self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump')
        link = self.dbx.sharing_create_shared_link_with_settings(DROPBOX_ROOT_PATH_NEW + '/' + time + '.dump')
        url = link.url
        dl_url = re.sub(r"\?dl\=0", "?dl=1", url)
        return dl_url

    def upload_file_compress(self, filename=''):
        print('Uploading para pasta ', DROPBOX_ROOT_PATH_NEW)
        t = datetime.datetime.now()
        t.strftime("%Y%m%d%H%M%S")
        new_basename = os.path.basename(filename)
        with open(filename, 'rb') as f:
            self.dbx.files_upload(f.read(), DROPBOX_ROOT_PATH_NEW + '/' + new_basename)
        link = self.dbx.sharing_create_shared_link_with_settings(
            DROPBOX_ROOT_PATH_NEW + '/' + new_basename)
        url = link.url
        dl_url = re.sub(r"\?dl\=0", "?dl=1", url)
        return dl_url

    def download_file(self, file=''):
        self.file = (DROPBOX_ROOT_PATH + '/' + file)
        file_name = self.file.replace('/sistemaweb/backup/', '')
        print('\nDownloading... /data/backup/' + file_name)
        try:
            metadata, res = self.dbx.files_download(self.file)
        except:
            pass
            metadata, res = self.dbx.files_download(file)
        final_path = ROOT_DIR + '/data/backup/' + file_name
        f = open(final_path, "wb")
        f.write(res.content)
        f.close()
        if '.zip' in final_path or '.gz' in final_path:
            print('Arquivo compactado...efetuando descompressão de dados.')
            self.uncompress_file(final_path)
            new_basename = os.path.basename(final_path).replace('.gz', '')
            return new_basename
        return final_path

    def list_dirs_root_path(self):
        self.dt = self.dbx.files_list_folder(self.root_path)
        #print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n')
        dir = self.download_file(self.dt.entries[-1].path_display) #self.list_files_root_path(self.dt)
        return dir

    def list_files_all(self):
        self.dt = self.dbx.files_list_folder(self.root_path)
        #print('ARQUIVOS ENCONTRADOS SERÃO LISTADOS ABAIXO:\n')
        self.data = []
        for entry in self.dt.entries:
            data = {}
            data['backup_link']
            data['client_modified'] = entry.client_modified
            data['size'] = str(entry.size)+" bytes"
            t = entry.client_modified
            time = datetime.timedelta(hours=2)
            hora = datetime.datetime.strptime(str(t), '%Y-%m-%d %H:%M:%S')
            now = hora - time
            size = entry.size
            size = str(size)+' bytes'
            display = entry.path_display
            print(display, now , size)
            self.data.append(data)
        return self.data

    def simple_backup(self):
        g = dbbackup.get_connector()
        execute_from_command_line(["manage.py", "dbbackup", "-v", "1"])
        filename = (ROOT_DIR + '/data/backup/' + g.generate_filename())
        #print(filename)
        return filename

    def compress_file(self, filename='', n=''):
        if n == '':
            self.compress_all(filename)
        elif n == '1':
            file = self.compress_all(filename)
            action = self.upload_file_compress(file)
            return action

    def compress_all(self, filename):
        if 'C:' in filename:
            new_basename = os.path.basename(filename)
            file = (ROOT_DIR + '/data/backup/' + new_basename + '.gz')
            f = open(filename, 'rb')
            data = f.read()
            f = gzip.open(file, 'wb')
            f.write(data)
            f.close()
            #print('Arquivo compactado com sucesso!!!')
            return file
        else:
            filepath = (ROOT_DIR + '/data/backup/' + filename)
            file = (ROOT_DIR + '/data/backup/' + filename + '.gz')
            f = open(filepath, 'rb')
            data = f.read()
            f = gzip.open(file, 'wb')
            f.write(data)
            f.close()
            #print('Arquivo compactado com sucesso!!!')
            return file

    def uncompress_file(self, filename):
        if 'C:' in filename:
            new_basename = os.path.basename(filename).replace('.gz', '')
            local_file = (ROOT_DIR + '/data/backup/' + new_basename)
            f = gzip.open(filename, 'rb')
            data = f.read()
            f.close()
            f = open(local_file, 'wb')
            f.write(data)
            f.close()
            print('Arquivo descomprimido com sucesso!!!')
        else:
            local_file = (ROOT_DIR + '/data/backup/' + filename)
            new_basename = os.path.basename(filename).replace('.gz', '')
            file = (ROOT_DIR + '/data/backup/' + new_basename)
            f = gzip.open(local_file, 'rb')
            data = f.read()
            f.close()
            f = open(file, 'wb')
            f.write(data)
            f.close()
            print('Arquivo descomprimido com sucesso!!!')

    def restore_db(self, filepath=''):
        print("VEJA O FILEPATH: ",filepath)
        execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", filepath])

    def restore(self):  # faz a restauração do banco de dados a partir de um backup salvo na dropbox.
        file_name = DropBoxStorage().list_dirs_root_path()
        new_basename = file_name
        if '/' in file_name:
            basename = shutil.copy(file_name, ROOT_DIR + '/data/backup/')
            new_basename = os.path.basename(basename)
        DropBoxStorage().restore_db(new_basename)

        from django.core.management import call_command
        import django
        django.setup()

        call_command('dbrestore', '-v', '1',  "--noinput", "-i", new_basename)

        #execute_from_command_line(["manage.py", "dbrestore", "-v", "1", "--noinput", "-i", new_basename])

        """