예제 #1
0
    def test_all_nodes_fail(self):
        # Create container1
        container1 = 'container-%s' % uuid4()
        cpart, cnodes = self.container_ring.get_nodes(self.account, container1)
        client.put_container(self.url, self.token, container1)
        client.put_object(self.url, self.token, container1, 'obj1', 'data1')

        # All primaries go down
        for cnode in cnodes:
            kill_server((cnode['ip'], cnode['port']), self.ipport2server)

        # Can't GET the container
        with self.assertRaises(client.ClientException) as caught:
            client.get_container(self.url, self.token, container1)
        self.assertEqual(caught.exception.http_status, 503)

        # But we can still write objects! The old info is still in memcache
        client.put_object(self.url, self.token, container1, 'obj2', 'data2')

        # Can't POST the container, either
        with self.assertRaises(client.ClientException) as caught:
            client.post_container(self.url, self.token, container1, {})
        self.assertEqual(caught.exception.http_status, 503)

        # Though it *does* evict the cache
        with self.assertRaises(client.ClientException) as caught:
            client.put_object(self.url, self.token, container1, 'obj3', 'x')
        self.assertEqual(caught.exception.http_status, 503)
 def test_param_end_marker(self):
     c.http_connection = self.fake_http_connection(
         204, query_string="format=json&end_marker=end_marker")
     c.get_container('http://www.test.com',
                     'asdf',
                     'asdf',
                     end_marker='end_marker')
예제 #3
0
def _get_total_objects(request, container, objectname):
    """ Return the total number of objects under a pseudofolder. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')
    prefix = request.session.get('prefix')

    if prefix:
        prefix += objectname
    else:
        prefix = objectname

    try:
        if prefix:
            meta, objects = client.get_container(
                storage_url, auth_token, container,
                prefix=prefix, full_listing=True,
                headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})

        else:
            meta, objects = client.get_container(
                storage_url, auth_token, container,
                headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')},
                full_listing=True)

    except client.ClientException:
        return 0

    pseudofolders, objs = pseudofolder_object_list(objects, prefix)

    return len(objs)
예제 #4
0
def create_container(request):
    """ Creates a container (empty object of type application/directory) """

    storage_url = request.session.get("storage_url", "")
    auth_token = request.session.get("auth_token", "")

    headers = {
        "X-Container-Meta-Access-Control-Expose-Headers": "Access-Control-Allow-Origin",
        "X-Container-Meta-Access-Control-Allow-Origin": settings.BASE_URL,
    }

    form = CreateContainerForm(request.POST or None)
    if form.is_valid():
        container = form.cleaned_data["containername"]

        # Check container does not already exist
        try:
            client.get_container(storage_url, auth_token, container)
            messages.add_message(request, messages.ERROR, _("Container {0} already exists.".format(container)))
        except:
            try:
                client.put_container(storage_url, auth_token, container, headers)
                messages.add_message(request, messages.INFO, _("Container created."))
            except client.ClientException:
                messages.add_message(request, messages.ERROR, _("Access denied."))

        return redirect(containerview)

    return render_to_response(
        "create_container.html", {"session": request.session}, context_instance=RequestContext(request)
    )
예제 #5
0
    def test_post_trumped_by_prior_delete(self):
        # new metadata and content-type posted to subset of nodes should not
        # cause object to persist after replication of an earlier delete on
        # other nodes.
        self.brain.put_container()
        # incomplete put
        self.brain.stop_primary_half()
        self.container_brain.stop_primary_half()
        self._put_object(
            headers={
                'Content-Type': 'oldest',
                'X-Object-Sysmeta-Test': 'oldest',
                'X-Object-Meta-Test': 'oldest'
            })
        self.brain.start_primary_half()
        self.container_brain.start_primary_half()

        # incomplete put then delete
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._put_object(
            headers={
                'Content-Type': 'oldest',
                'X-Object-Sysmeta-Test': 'oldest',
                'X-Object-Meta-Test': 'oldest'
            })
        self._delete_object()
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        # handoff post
        self.brain.stop_primary_half()
        self.container_brain.stop_primary_half()
        self._post_object(
            headers={
                'Content-Type': 'newest',
                'X-Object-Sysmeta-Test': 'ignored',
                'X-Object-Meta-Test': 'newest'
            })

        # check object metadata
        metadata = self._get_object_metadata()
        self.assertEqual(metadata['x-object-sysmeta-test'], 'oldest')
        self.assertEqual(metadata['x-object-meta-test'], 'newest')
        self.assertEqual(metadata['content-type'], 'newest')

        self.brain.start_primary_half()
        self.container_brain.start_primary_half()

        # delete trumps later post
        self.get_to_final_state()

        # check object is now deleted
        self.assertRaises(UnexpectedResponse, self._get_object_metadata)
        container_metadata, objs = client.get_container(
            self.url, self.token, self.container_name)
        self.assertEqual(0, len(objs))
        self._assert_consistent_container_dbs()
        self._assert_consistent_deleted_object()
        self._assert_consistent_suffix_hashes()
예제 #6
0
def public_objectview(request, account, container, prefix=None):
    """ Returns list of all objects in current container. """
    storage_url = settings.STORAGE_URL + account
    auth_token = ' '
    try:
        _meta, objects = client.get_container(storage_url,
                                              auth_token,
                                              container,
                                              delimiter='/',
                                              prefix=prefix)

    except client.ClientException:
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    base_url = get_base_url(request)
    account = storage_url.split('/')[-1]

    return render_to_response("publicview.html", {
        'container': container,
        'objects': objs,
        'folders': pseudofolders,
        'prefix': prefix,
        'prefixes': prefixes,
        'base_url': base_url,
        'storage_url': storage_url,
        'account': account
    },
                              context_instance=RequestContext(request))
예제 #7
0
def delete_given_folder(request, container, foldername):
    """ Recursively delete all the objects in the given folder and then
    delete the folder itself. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    # Get all objects within folder.
    meta, objects = client.get_container(
        storage_url, auth_token, container,
        headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')},
        delimiter='/', prefix=foldername)

    # Recursive call to delete subfolders.
    pseudofolders, objs = pseudofolder_object_list(objects, foldername)
    for folder in pseudofolders:
        delete_given_folder(request, container, folder[0])

    # Delete all objects.
    for obj in objs:
        delete_given_object(request, container, obj["name"])

    # Delete the folder itself.
    try:
        client.get_object(
            storage_url, auth_token, container, foldername,
            headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
        delete_given_object(request, container, foldername)
    except:
        # If folder does not exist, pass
        pass
예제 #8
0
파일: views.py 프로젝트: globocom/vault
def objectview(request, container, prefix=None):
    """ Returns list of all objects in current container. """

    storage_url = get_endpoint(request, 'adminURL')
    public_url = get_endpoint(request, 'publicURL') + '/' + container
    auth_token = get_token_id(request)
    http_conn = client.http_connection(storage_url,
                                       insecure=settings.SWIFT_INSECURE)

    page = request.GET.get('page', 1)

    try:
        _, objects = client.get_container(storage_url, auth_token,
                                             container, delimiter='/',
                                             prefix=prefix,
                                             http_conn=http_conn)
    except client.ClientException as err:
        log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
        messages.add_message(request, messages.ERROR, ugettext('Access denied.'))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    object_list = pseudofolder_object_list(objects, prefix, public_url)
    context = utils.update_default_context(request, {
        'container': container,
        'objects': utils.generic_pagination(object_list, page),
        'prefix': prefix,
        'prefixes': prefixes,
    })

    return render_to_response("objectview.html", context,
                              context_instance=RequestContext(request))
예제 #9
0
파일: views.py 프로젝트: globocom/vault
def delete_container(request, container, force=True):
    """
    Deletes a container. If force is True, it will deletes all objects first.
    """

    storage_url = get_endpoint(request, 'adminURL')
    auth_token = get_token_id(request)
    http_conn = client.http_connection(storage_url,
                                       insecure=settings.SWIFT_INSECURE)

    if force:
        try:
            _, objects = client.get_container(storage_url,
                                           auth_token,
                                           container,
                                           http_conn=http_conn)

        except client.ClientException as err:
            log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
            return False

        for obj in objects:
            delete_object(request=request,
                          container=container,
                          objectname=obj['name'])

    try:
        client.delete_container(storage_url, auth_token,
                                container, http_conn=http_conn)
        actionlog.log(request.user.username, "delete", container)
    except client.ClientException as err:
        log.exception('{}{}'.format(ugettext('Exception:').encode('UTF-8'), err))
        return False

    return True
예제 #10
0
def subdir_proc(storage_url, token, container, subdir=None, nest=0):
    #print "DEBUG %s" % subdir
    (c_headers, objects) = get_container(storage_url, token,
                                         container,
                                         prefix=subdir,
                                         delimiter='/')
    spacer = ' ' * ((nest + 1) * 2)
    for obj in objects:
        if 'subdir' in obj:
            sdir_name = obj['subdir']
            l = len(sdir_name)
            idx = sdir_name.rfind('/', 0, l - 1)
            if idx >= 0:
                sdir_name = sdir_name[idx + 1:]
            print "%-30s" % (spacer + sdir_name)
            if subdir is None:
                new_subdir = obj['subdir']
            else:
                new_subdir = obj['subdir']
            subdir_proc(storage_url, token, container, new_subdir, nest + 1)
        else:
            if subdir is None:
                l = 0
            else:
                l = len(subdir)
            objname = obj['name'][l:]
            print "%-30s  %20s bytes" % (spacer + objname, obj['bytes'])
예제 #11
0
    def getMemberNames(self):
        _stat, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              delimiter='/',
                                              prefix=self.prefix,
                                              http_conn=self.http_connection)

        self.objects = {}

        childs = []
        for obj in objects:
            name = obj.get('name')
            if name and name != self.prefix:
                name = name.encode("utf8")
                childs.append(name)
                self.objects[name] = obj
            subdir = obj.get('subdir')
            if subdir and subdir != self.prefix:
                subdir = subdir.rstrip('/')
                subdir = subdir.encode("utf8")
                # there might be two entries:
                # 1. object with type application/directory and no trailing '/'
                # 2. subdir entry with trailing '/'
                if subdir not in childs:
                    childs.append(subdir)
                    self.objects[subdir] = obj
        return childs
def delete_container(request):
    auth_token = request.session['auth_token']
    storage_url = request.session['storage_url']

    container = ''
    if 'container' in request.GET.keys():
        container = request.GET['container']
    else:
        return redirect(containers)

    try:
        http_conn = (urlparse(storage_url),
                     client.HTTPConnection(storage_url, insecure=settings.SWIFT_SSL_INSECURE))
        _m, objects = client.get_container(storage_url, auth_token, container, http_conn=http_conn)
        for obj in objects:
            logger.info("Deleting object %s in container %s" % (obj['name'], container))
            client.delete_object(storage_url, auth_token,
                                 container, obj['name'], http_conn=http_conn)
        logger.info("Deleting container %s" % (container))
        client.delete_container(storage_url, auth_token, container, http_conn=http_conn)
        messages.add_message(request, messages.INFO, "Container deleted.")
    except client.ClientException:
        messages.add_message(request, messages.ERROR, "Access denied.")

    return redirect(containers)
예제 #13
0
 def list(self):
     swift_objects = []
     for cont in self.list_containers():
         objs = [{'container': cont, 'name': obj['name']} for obj in
                 swift_client.get_container(self.endpoint, self.token, cont)[1]]
         swift_objects.extend(objs)
     return swift_objects
예제 #14
0
def get_object_table(request):
    """ Returns json object of all objects in current container. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')
    container = request.session.get('container')
    prefix = request.session.get('prefix')

    try:
        meta, objects = client.get_container(
            storage_url, auth_token, container, delimiter='/', prefix=prefix,
            headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})

    except client.ClientException:
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    pseudofolders, objs = pseudofolder_object_list(objects, prefix)

    # Check for incomplete SLO uploads. The object table will display them
    # allowing the user to continue uploading or delete them.
    incomplete_slo = check_incomplete_slo(
        request, storage_url, auth_token, container, prefix)

    return JsonResponse({
        'success': True,
        "data": {
            'container': container,
            'objects': objs,
            'folders': pseudofolders,
            'folder_prefix': prefix,
            'incomplete_slo': incomplete_slo
        }
    })
예제 #15
0
def public_objectview(request, account, container, prefix=None):
    """ Returns list of all objects in current container. """
    storage_url = settings.STORAGE_URL + account
    auth_token = ' '
    try:
        _meta, objects = client.get_container(
            storage_url, auth_token, container, delimiter='/', prefix=prefix)

    except client.ClientException:
        traceback.print_exc()
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    base_url = get_base_url(request)
    account = storage_url.split('/')[-1]

    return render_to_response("publicview.html", {
        'container': container,
        'objects': objs,
        'folders': pseudofolders,
        'prefix': prefix,
        'prefixes': prefixes,
        'base_url': base_url,
        'storage_url': storage_url,
        'account': account},
        context_instance=RequestContext(request))
    def test_async_updates_after_PUT_and_POST(self):
        # verify correct update values when PUT update and POST updates are
        # missed but then async updates are sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # PUT and POST to object while one container server is stopped so that
        # we force async updates to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url,
                          self.token,
                          'c1',
                          'o1',
                          contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(self.account, 'c1', 'o1',
                                       {'X-Object-Meta-Fruit': 'Tomato'})
        self.assertEqual('Tomato',
                         int_client.get_object_metadata(
                             self.account, 'c1',
                             'o1')['x-object-meta-fruit'])  # sanity

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # Run the object-updaters to send the async pendings
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
예제 #17
0
def public_objectview(request, account, container, prefix=None):
    """ Returns list of all objects in current container. """
    storage_url = settings.STORAGE_URL + account
    auth_token = " "
    try:
        _meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix)

    except client.ClientException:
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    base_url = get_base_url(request)
    account = storage_url.split("/")[-1]

    return render_to_response(
        "publicview.html",
        {
            "container": container,
            "objects": objs,
            "folders": pseudofolders,
            "prefix": prefix,
            "prefixes": prefixes,
            "base_url": base_url,
            "storage_url": storage_url,
            "account": account,
        },
        context_instance=RequestContext(request),
    )
예제 #18
0
    def getMemberNames(self):
        _stat, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              delimiter='/',
                                              prefix=self.prefix,
                                              http_conn=self.http_connection)

        self.objects = {}

        childs = []
        for obj in objects:
            name = obj.get('name')
            if name and name != self.prefix:
                name = name.encode("utf8")
                childs.append(name)
                self.objects[name] = obj
            subdir = obj.get('subdir')
            if subdir and subdir != self.prefix:
                subdir = subdir.rstrip('/')
                subdir = subdir.encode("utf8")
                # there might be two entries:
                # 1. object with type application/directory and no trailing '/'
                # 2. subdir entry with trailing '/'
                if subdir not in childs:
                    childs.append(subdir)
                    self.objects[subdir] = obj
        return childs
예제 #19
0
def get_all(request, container):
    """ Returns List of all Objects present in specified container
    :param container: Name of Swift Container
    :return: meta data and id's of all the objects
    """
    try:
        response_dict = dict()
        storage_url, auth_token = _get_auth_data(request.session)
        data_container = client.get_container(storage_url, auth_token, container)
        for obj in data_container[1]:
            meta_object = client.head_object(storage_url, auth_token, container, obj['name'])
            if not meta_object['x-object-meta-deleted']:
                if meta_object['x-object-meta-type'] in ['thumbnail', 'original-thumbnail']:
                    if meta_object['x-object-meta-format'] not in response_dict:
                        form = meta_object['x-object-meta-format']
                        response_dict[form] = []
                    if meta_object['x-object-meta-format'] in response_dict:
                        new_obj = {'thumbnail_id': obj['name'],
                                   'name': meta_object['x-object-meta-name'],
                                   'type': meta_object['x-object-meta-type'],
                                   'resolution': meta_object['x-object-meta-resolution']}
                        if meta_object['x-object-meta-format'] is 'thumbnail':
                            new_obj['original_id'] = meta_object['x-object-meta-original']
                        response_dict[form].append(new_obj)
        return Response(response_dict)
    except swift_exception.ClientException as e:
        print e
        return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
    except Exception as e:
        print e
        return Response('Please contact your admininstrator', status=status.HTTP_500_INTERNAL_SERVER_ERROR)
예제 #20
0
    def list_container(self,container_name,prefix=None,delimiter=None):

        try:
            return client.get_container(self.storage_url,self.token,container_name,prefix=prefix,delimiter=delimiter,http_conn=self.http_conn)

        except:
            return None
예제 #21
0
 def test_create_swift_client(self):
     mock_resp = mock(dict)
     when(swiftclient.client.Connection).get_container("bob").thenReturn(["text", mock_resp])
     client = remote.create_swift_client(TroveContext(tenant="123"))
     headers, container = client.get_container("bob")
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #22
0
 def get_container_detail(self, account_id, token, s_type, container):
     cnx = self.get_cnx(account_id, s_type)
     return sclient.get_container(None,
                                  token,
                                  container,
                                  http_conn=cnx,
                                  full_listing=True)
예제 #23
0
def get_segment_number(file_name, request, container, prefix=None):
    '''Return the segment number a given file should create next. If it is 0,
    create a pseudo folder for the file. The folder is created if it doesn't
    exist. '''

    if prefix:
        foldername = prefix + '/' + file_name + '_segments'
    else:
        foldername = file_name + '_segments'

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    foldername = os.path.normpath(foldername)
    foldername = foldername.strip('/')
    foldername += '/'

    content_type = 'application/directory'
    obj = None

    client.put_object(storage_url, auth_token, container, foldername, obj,
                      content_type=content_type)

    meta, objects = client.get_container(storage_url, auth_token, container,
                                         delimiter='/', prefix=foldername)

    pseudofolders, objs = pseudofolder_object_list(objects, prefix)

    return get_first_nonconsecutive(objs)
예제 #24
0
def get_object_table(request):
    """ Returns json object of all objects in current container. """

    storage_url = request.session.get("storage_url", "")
    auth_token = request.session.get("auth_token", "")
    container = request.session.get("container")
    prefix = request.session.get("prefix")

    try:
        meta, objects = client.get_container(storage_url, auth_token, container, delimiter="/", prefix=prefix)

    except client.ClientException:
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    base_url = get_base_url(request)
    account = storage_url.split("/")[-1]

    return JsonResponse(
        {
            "success": True,
            "data": {"container": container, "objects": objs, "folders": pseudofolders, "folder_prefix": prefix},
        }
    )
예제 #25
0
파일: api.py 프로젝트: accelazh/magnetodb
 def delete_data(self, context, container_name):
     url = self._get_endpoint_url(context)
     token = context.auth_token
     try:
         headers, info = swift_client.get_container(
             url, token, container_name
         )
         for obj in info:
             try:
                 swift_client.delete_object(
                     url, token, container_name, obj['name']
                 )
             except swift_exceptions.ClientException as e:
                 LOG.error(e)
                 if e.http_status != 404:
                     raise
         swift_client.delete_container(url, token, container_name)
     except swift_exceptions.ClientException as e:
         LOG.error(e)
         if e.http_status == 404:
             raise exception.ContainerNotExists(
                 container_name=container_name
             )
         else:
             raise exception.ContainerDeletionError(
                 container_name=container_name
             )
예제 #26
0
def delete_given_folder(request, container, foldername):

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    # Get all objects within folder.
    meta, objects = client.get_container(
        storage_url, auth_token, container, delimiter='/', prefix=foldername)

    # Recursive call to delete subfolders.
    pseudofolders, objs = pseudofolder_object_list(objects, foldername)
    for folder in pseudofolders:
        delete_given_folder(request, container, folder[0])

    # Delete all objects.
    for obj in objs:
        delete_given_object(request, container, obj["name"])

    # Delete the folder itself.
    try:
        delete_given_object(request, container, foldername)
    except:
        #Except a failure to delete if the pseudo folder was not created
        #manually.
        pass
예제 #27
0
def get_temp_key(storage_url, auth_token, container):
    """ Tries to get meta-temp-url key from account.
    If not set, generate tempurl and save it to acocunt.
    This requires at least account owner rights. """

    try:
        account = client.get_account(storage_url, auth_token)
    except client.ClientException:

        #Try to get the container temp url key instead
        try:
            container = client.get_container(
                storage_url, auth_token, container)
            return container[0].get('x-container-meta-temp-url-key')
        except client.ClientException:
            return None
        return None

    key = account[0].get('x-account-meta-temp-url-key')

    if not key:
        chars = string.ascii_lowercase + string.digits
        key = ''.join(random.choice(chars) for x in range(32))
        headers = {'x-account-meta-temp-url-key': key}
        try:
            client.post_account(storage_url, auth_token, headers)
        except client.ClientException:
            return None
    return key
예제 #28
0
파일: views.py 프로젝트: globocom/vault
def object_versioning(request, container, prefix=None):
    storage_url = get_endpoint(request, 'adminURL')
    auth_token = get_token_id(request)
    public_url = get_endpoint(request, 'publicURL') + '/' + container
    http_conn = client.http_connection(storage_url,
                                       insecure=settings.SWIFT_INSECURE)

    objects = []

    page = request.GET.get('page', 1)

    if request.method == 'GET':
        headers = client.head_container(storage_url,
                                auth_token,
                                container,
                                http_conn=http_conn)

        version_location = headers.get('x-versions-location', None)

        if version_location:
            try:
                _, objects = client.get_container(storage_url,
                                                  auth_token,
                                                  version_location,
                                                  prefix=prefix,
                                                  delimiter='/',
                                                  http_conn=http_conn)
            except client.ClientException:
                pass

        prefixes = prefix_list(prefix)
        object_list = pseudofolder_object_list(objects, prefix, public_url)

        context = utils.update_default_context(request, {
            'container': container,
            'objects': utils.generic_pagination(object_list, page),
            'version_location': version_location,
            'prefix': prefix,
            'prefixes': prefixes,
        })

        return render_to_response('container_versioning.html',
                                  dictionary=context,
                                  context_instance=RequestContext(request))

    if request.method == 'POST':

        action = request.POST.get('action', None)

        if action == 'enable':
            enable_versioning(request, container)
            actionlog.log(request.user.username, "enable", 'Versioning. Container: %s' % container)
        elif action == 'disable':
            disable_versioning(request, container)
            actionlog.log(request.user.username, "disable", 'Versioning. Container: %s' % container)
        else:
            messages.add_message(request, messages.ERROR, ugettext('Action is required.'))

        return redirect(object_versioning, container=container)
예제 #29
0
파일: test_remote.py 프로젝트: sitle/trove
 def test_create_swift_client(self):
     mock_resp = MagicMock()
     swiftclient.client.Connection.get_container = MagicMock(
         return_value=["text", mock_resp])
     client = remote.create_swift_client(TroveContext(tenant='123'))
     headers, container = client.get_container('bob')
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #30
0
    def find_container(self, name):
        if not self.http_conn:
            self.connect()

        retval = swift.get_container(url=self.swift_url, token=self.token, http_conn=self.http_conn, container=name)
        if self.debug:
            print(retval)
        return retval
예제 #31
0
 def test_create_swift_client(self):
     mock_resp = MagicMock()
     swiftclient.client.Connection.get_container = MagicMock(
         return_value=["text", mock_resp])
     client = remote.create_swift_client(TroveContext(tenant='123'))
     headers, container = client.get_container('bob')
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #32
0
    def test_first_two_nodes_fail(self):
        container = 'container-%s' % uuid4()
        client.put_container(self.url, self.token, container)
        self.assert_(container in [c['name'] for c in
                     client.get_account(self.url, self.token)[1]])

        object1 = 'object1'
        client.put_object(self.url, self.token, container, object1, 'test')
        self.assert_(container in [c['name'] for c in
                     client.get_account(self.url, self.token)[1]])
        self.assert_(object1 in [o['name'] for o in
                     client.get_container(self.url, self.token, container)[1]])

        cpart, cnodes = self.container_ring.get_nodes(self.account, container)
        for x in xrange(2):
            kill(self.pids[self.port2server[cnodes[x]['port']]], SIGTERM)

        client.delete_object(self.url, self.token, container, object1)
        self.assert_(container in [c['name'] for c in
                     client.get_account(self.url, self.token)[1]])
        self.assert_(object1 not in [o['name'] for o in
                     client.get_container(self.url, self.token, container)[1]])

        for x in xrange(2):
            self.pids[self.port2server[cnodes[x]['port']]] = \
                Popen(['swift-container-server',
                       '/etc/swift/container-server/%d.conf' %
                        ((cnodes[x]['port'] - 6001) / 10)]).pid
        sleep(2)
        self.assert_(container in [c['name'] for c in
                     client.get_account(self.url, self.token)[1]])
        # This okay because the first node hasn't got the update that the
        # object was deleted yet.
        self.assert_(object1 in [o['name'] for o in
                     direct_client.direct_get_container(cnodes[0], cpart,
                     self.account, container)[1]])

        # This fails because all three nodes have to indicate deletion before
        # we tell the user it worked. Since the first node 409s (it hasn't got
        # the update that the object was deleted yet), the whole must 503
        # (until every is synced up, then the delete would work).
        exc = None
        try:
            client.delete_container(self.url, self.token, container)
        except client.ClientException, err:
            exc = err
예제 #33
0
 def test_create_swift_client(self):
     mock_resp = mock(dict)
     when(swiftclient.client.Connection).get_container('bob').thenReturn(
         ["text", mock_resp])
     client = remote.create_swift_client(TroveContext(tenant='123'))
     headers, container = client.get_container('bob')
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #34
0
    def clean_container(self, container_name):
        _, objects = client.get_container(
            self.url, self.token, container_name,
            full_listing=True)

        # delete all objects inside the container
        # N.B. this cleanup could run in parallel but currently we have a few
        # objects in the user testing container so that, currently this does
        # as sequential simply
        for obj_dict in objects:
            client.delete_object(
                self.url, self.token,
                container_name, obj_dict['name'])
        client.get_container(self.url, self.token, container_name)

        # delete the container
        client.delete_container(self.url, self.token, container_name)    
예제 #35
0
 def test_create_swift_client(self):
     mock_resp = mock(dict)
     when(swiftclient.client.Connection).get_container('bob').thenReturn(
         ["text", mock_resp])
     client = remote.create_swift_client(ReddwarfContext(tenant='123'))
     headers, container = client.get_container('bob')
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #36
0
def download_collection(request, container, prefix=None, non_recursive=False):
    """ Download the content of an entire container/pseudofolder
    as a Zip file. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    delimiter = '/' if non_recursive else None
    try:
        x, objects = client.get_container(
            storage_url,
            auth_token,
            container,
            delimiter=delimiter,
            prefix=prefix
        )
    except client.ClientException:
        return HttpResponseForbidden()

    x, objs = pseudofolder_object_list(objects, prefix)

    # Do not provide download when the folder is empty.
    if len(x) + len(objs) == 0:
        messages.add_message(
            request, messages.ERROR, _("Unable to download, no files found."))

        if prefix:  # Return user to object view
            # remove the last prefix. ex "dir1/dir2/" -> "dir1"
            prefix = prefix[0:prefix.rfind('/', 0, prefix.rfind('/'))]
            return redirect(swiftbrowser.views.objectview, container=container,
                            prefix=prefix)
        else:  # Return user to the container view
            return redirect(swiftbrowser.views.containerview)

    output = StringIO()
    zipf = zipfile.ZipFile(output, 'w')
    for o in objs:
        name = o['name']
        try:
            x, content = client.get_object(storage_url, auth_token, container,
                                           name)
        except client.ClientException:
            return HttpResponseForbidden()

        if prefix:
            name = name[len(prefix):]
        zipf.writestr(name, content)
    zipf.close()

    if prefix:
        filename = prefix.split('/')[-2]
    else:
        filename = container
    response = HttpResponse(output.getvalue(), 'application/zip')
    response['Content-Disposition'] = 'attachment; filename="%s.zip"'\
        % (filename)
    output.close()
    return response
예제 #37
0
파일: views.py 프로젝트: stormers/vault
def delete_pseudofolder(request, container, pseudofolder):
    """ Deletes an empty object, used as a pseudofolder """

    storage_url = get_endpoint(request, 'adminURL')
    auth_token = get_token_id(request)
    http_conn = client.http_connection(storage_url,
                                       insecure=settings.SWIFT_INSECURE)

    objects = client.get_container(storage_url,
                                   auth_token,
                                   container,
                                   prefix=pseudofolder,
                                   http_conn=http_conn)[1]

    count_deletes = 0
    for obj in objects:
        try:
            client.delete_object(storage_url,
                                 token=auth_token,
                                 container=container,
                                 name=obj['name'],
                                 http_conn=http_conn)
            count_deletes += 1
        except client.ClientException as err:
            log.exception('{}{}'.format(_('Exception:').encode('UTF-8'), err))

    # Empty pseudofolder
    if count_deletes == 1 and count_deletes == len(objects):
        messages.add_message(request, messages.SUCCESS,
                             _('Pseudofolder deleted.'))

    # Non empty pseudofolder
    elif count_deletes > 1 and count_deletes == len(objects):
        messages.add_message(
            request, messages.SUCCESS,
            _('Pseudofolder and {0} objects deleted.').format(count_deletes -
                                                              1))
    elif count_deletes > 0 and count_deletes < len(objects):
        messages.add_message(request, messages.SUCCESS,
                             _('It was not possible to delete all objects.'))
    else:
        messages.add_message(request, messages.ERROR,
                             _('Fail to delete pseudofolder.'))

    if pseudofolder[-1] == '/':  # deleting a pseudofolder, move one level up
        pseudofolder = pseudofolder[:-1]

    prefix = '/'.join(pseudofolder.split('/')[:-1])

    if prefix:
        prefix += '/'

    actionlog.log(request.user.username, "delete", pseudofolder)

    if prefix:
        return redirect(objectview, container=container, prefix=prefix)
    else:
        return redirect(objectview, container=container)
예제 #38
0
    def test_put_ctype_replicated_when_subsequent_post(self):
        # primary half                     handoff half
        # ------------                     ------------
        # t0.data: ctype = foo
        #                                  t1.data: ctype = bar
        # t2.meta:
        #
        #              ...run replicator and expect...
        #
        #               t1.data: ctype = bar
        #               t2.meta:
        self.brain.put_container(policy_index=0)

        # incomplete write
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._put_object(headers={'Content-Type': 'foo'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        # handoff write
        self.brain.stop_primary_half()
        self.container_brain.stop_primary_half()
        self._put_object(headers={'Content-Type': 'bar'})
        self.brain.start_primary_half()
        self.container_brain.start_primary_half()

        # metadata update with newest data unavailable
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        self.get_to_final_state()

        # check object metadata
        metadata = client.head_object(self.url, self.token,
                                      self.container_name,
                                      self.object_name)

        # check container listing metadata
        container_metadata, objs = client.get_container(self.url, self.token,
                                                        self.container_name)

        for obj in objs:
            if obj['name'] == self.object_name:
                break
        else:
            self.fail('obj not found in container listing')
        expected = 'bar'
        self.assertEqual(obj['content_type'], expected)
        self.assertEqual(metadata['x-object-meta-color'], 'Blue')
        self._assert_object_metadata_matches_listing(obj, metadata)
        self._assert_consistent_container_dbs()
        self._assert_consistent_object_metadata()
        self._assert_consistent_suffix_hashes()
    def test_put_ctype_replicated_when_subsequent_post(self):
        # primary half                     handoff half
        # ------------                     ------------
        # t0.data: ctype = foo
        #                                  t1.data: ctype = bar
        # t2.meta:
        #
        #              ...run replicator and expect...
        #
        #               t1.data: ctype = bar
        #               t2.meta:
        self.brain.put_container()

        # incomplete write
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._put_object(headers={'Content-Type': 'foo'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        # handoff write
        self.brain.stop_primary_half()
        self.container_brain.stop_primary_half()
        self._put_object(headers={'Content-Type': 'bar'})
        self.brain.start_primary_half()
        self.container_brain.start_primary_half()

        # metadata update with newest data unavailable
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._post_object(headers={'X-Object-Meta-Color': 'Blue'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        self.get_to_final_state()

        # check object metadata
        metadata = client.head_object(self.url, self.token,
                                      self.container_name,
                                      self.object_name)

        # check container listing metadata
        container_metadata, objs = client.get_container(self.url, self.token,
                                                        self.container_name)

        for obj in objs:
            if obj['name'] == self.object_name:
                break
        else:
            self.fail('obj not found in container listing')
        expected = 'bar'
        self.assertEqual(obj['content_type'], expected)
        self.assertEqual(metadata['x-object-meta-color'], 'Blue')
        self._assert_object_metadata_matches_listing(obj, metadata)
        self._assert_consistent_container_dbs()
        self._assert_consistent_object_metadata()
        self._assert_consistent_suffix_hashes()
예제 #40
0
    def test_update_during_POST_only(self):
        # verify correct update values when PUT update is missed but then a
        # POST update succeeds *before* the PUT async pending update is sent
        cpart, cnodes = self.container_ring.get_nodes(self.account, "c1")
        client.put_container(self.url, self.token, "c1", headers={"X-Storage-Policy": self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        content = u"stuff"
        client.put_object(self.url, self.token, "c1", "o1", contents=content, content_type="test/ctype")
        meta = client.head_object(self.url, self.token, "c1", "o1")

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]["ip"], cnodes[0]["port"]), self.ipport2server)
        self.assertFalse(direct_client.direct_get_container(cnodes[0], cpart, self.account, "c1")[1])

        # use internal client for POST so we can force fast-post mode
        int_client = self.make_internal_client(object_post_as_copy=False)
        int_client.set_object_metadata(self.account, "c1", "o1", {"X-Object-Meta-Fruit": "Tomato"})
        self.assertEqual(
            "Tomato", int_client.get_object_metadata(self.account, "c1", "o1")["x-object-meta-fruit"]
        )  # sanity

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]["bytes"])
            self.assertEqual("test/ctype", listing[0]["content_type"])
            listing_etags.add(listing[0]["hash"])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, "c1")
        self.assertEqual(1, len(listing))
        self.assertEqual("o1", listing[0]["name"])
        self.assertEqual(len(content), listing[0]["bytes"])
        self.assertEqual(meta["etag"], listing[0]["hash"])
        self.assertEqual("test/ctype", listing[0]["content_type"])

        # Run the object-updaters to send the async pending from the PUT
        Manager(["object-updater"]).once()

        # check container listing metadata is still correct
        for cnode in cnodes:
            listing = direct_client.direct_get_container(cnode, cpart, self.account, "c1")[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]["bytes"])
            self.assertEqual("test/ctype", listing[0]["content_type"])
            listing_etags.add(listing[0]["hash"])
        self.assertEqual(1, len(listing_etags))
예제 #41
0
def create_container(request):
    """ Creates a container on swift. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    # We need the CORS header set to the base url of this instance of
    # swiftbrowser. Otherwise posts from the browser to swift will not work.
    headers = {
        'X-Container-Meta-Access-Control-Expose-Headers':
        'Access-Control-Allow-Origin',
        'X-Container-Meta-Access-Control-Allow-Origin': settings.BASE_URL
    }

    form = CreateContainerForm(request.POST or None)
    if form.is_valid():
        container = form.cleaned_data['containername']

        #Check container does not already exist
        try:
            client.get_container(
                storage_url, auth_token, container,
                headers={"X-Forwarded-For": request.META.get('REMOTE_ADDR')})
            messages.add_message(
                request,
                messages.ERROR,
                _("Container {0} already exists.".format(container)))
        except:
            try:
                client.put_container(
                    storage_url, auth_token, container, headers)
                messages.add_message(request, messages.INFO,
                                     _("Container created."))
            except client.ClientException:
                messages.add_message(
                    request, messages.ERROR, _("Access denied."))

        return redirect(containerview)

    return render_to_response(
        'create_container.html',
        {'session': request.session},
        context_instance=RequestContext(request))
예제 #42
0
    def is_subdir(self, name):
        """Checks if given name is a subdir.

        This is a workaround for Swift (and other object storages).

        There are several possibilites for a given URL in the form /ABC/XYZ.
        1. /ABC/XYZ is the full path to an object. In this case there are
            more possibilites, for example get the object, check if it exists,
            put a new object under this name.
        2. /ABC/XYZ should list the contents of container ABC with a prefix XYZ

        The latter one will return en empty result set, thus this will be used
        to differentiate between the possibilites.
        """

        name = name.strip('/')

        obj = self.objects.get(name, self.objects.get(name + '/'))
        if not obj:
            _, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              http_conn=self.http_connection)
            for obj in objects:
                objname = obj.get('name')
                self.objects[objname] = obj

            _, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              delimiter='/',
                                              prefix=name,
                                              http_conn=self.http_connection)
            for obj in objects:
                objname = obj.get('name', obj.get('subdir'))
                self.objects[objname] = obj

        obj = self.objects.get(name, self.objects.get(name + '/', {}))
        if obj.get('subdir') or \
                obj.get('content_type') == 'application/directory':
            return True

        return False
예제 #43
0
 def list_container(self, container_name, prefix=None, delimiter=None):
     try:
         return client.get_container(self.storage_url,
                                     self.token,
                                     container_name,
                                     prefix=prefix,
                                     delimiter=delimiter,
                                     http_conn=self.http_conn)
     except:
         return None
예제 #44
0
    def is_subdir(self, name):
        """Checks if given name is a subdir.

        This is a workaround for Swift (and other object storages).

        There are several possibilites for a given URL in the form /ABC/XYZ.
        1. /ABC/XYZ is the full path to an object. In this case there are
            more possibilites, for example get the object, check if it exists,
            put a new object under this name.
        2. /ABC/XYZ should list the contents of container ABC with a prefix XYZ

        The latter one will return en empty result set, thus this will be used
        to differentiate between the possibilites.
        """
        name = name.strip('/')
        name = name.replace(self.container + '/', '')

        obj = self.objects.get(name, self.objects.get(name + '/'))
        if not obj:
            _, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              http_conn=self.http_connection)
            for obj in objects:
                objname = obj.get('name')
                self.objects[objname] = obj

            _, objects = client.get_container(self.storage_url,
                                              self.auth_token,
                                              container=self.container,
                                              delimiter='/',
                                              prefix=name,
                                              http_conn=self.http_connection)
            for obj in objects:
                objname = obj.get('name', obj.get('subdir'))
                self.objects[objname] = obj

        obj = self.objects.get(name, self.objects.get(name + '/', {}))
        if obj.get('subdir') or \
                obj.get('content_type') == 'application/directory':
            return True

        return False
예제 #45
0
 def list(self):
     swift_objects = []
     for cont in self.list_containers():
         objs = [{'container': cont, 'name': obj['name']} for obj in
                 swift_client.get_container(self.endpoint,
                                            self.token,
                                            cont,
                                            http_conn=self.http_conn)[1]]
         swift_objects.extend(objs)
     return swift_objects
예제 #46
0
    def test_skip_sync_when_misconfigured(self):
        source_container, dest_container = self._setup_synced_containers()

        container_name = 'versioned-%s' % uuid.uuid4()
        version_hdr = {'X-Versions-Enabled': 'true'}

        client.put_container(self.url, self.token, container_name,
                             headers=version_hdr)

        # some sanity checks
        object_name = 'object-%s' % uuid.uuid4()
        client.put_object(self.url, self.token, container_name, object_name,
                          'version1')
        client.put_object(self.url, self.token, container_name, object_name,
                          'version2')

        resp_headers, listing = client.get_container(
            self.url, self.token, container_name,
            query_string='versions')

        self.assertEqual(2, len(listing))

        sync_headers = {}
        sync_to = '//%s/%s/%s/%s' % (self.realm, self.cluster, self.account,
                                     dest_container)
        sync_headers['X-Container-Sync-To'] = sync_to
        sync_headers['X-Container-Sync-Key'] = 'secret'

        # use internal client to set container-sync headers
        # since it doesn't have container_sync middleware in pipeline
        # allowing us to bypass checks
        int_client = self.make_internal_client()
        # TODO: what a terrible hack, maybe we need to extend internal
        # client to allow caller to become a swift_owner??
        int_client.app.app.app.app.swift_owner_headers = []
        int_client.set_container_metadata(self.account, container_name,
                                          metadata=sync_headers)

        headers = client.head_container(self.url, self.token,
                                        container_name)

        # This should never happen, but if it does because of eventual
        # consistency or a messed up pipeline, container-sync should
        # skip syncing container.
        self.assertEqual('True', headers.get('x-versions-enabled'))
        self.assertEqual('secret', headers.get('x-container-sync-key'))
        self.assertEqual(sync_to, headers.get('x-container-sync-to'))

        # cycle container-sync
        Manager(['container-sync']).once()

        with self.assertRaises(ClientException) as cm:
            client.get_object(
                self.url, self.token, dest_container, object_name)
        self.assertEqual(404, cm.exception.http_status)  # sanity check
예제 #47
0
    def moveRecursive(self, newname):
        """ Move/rename a container.

        This is only working for empty containers and objects with a size of 0.
        Required by some clients because they create a folder "New folder" first.

        For all other requests this will simply return HTTP_FORBIDDEN. """
        oldname = self.path.lstrip('/')
        newname = newname.lstrip('/')

        if '/' not in oldname:
            try:
                # Container deletion will fail if not empty
                client.delete_container(self.storage_url,
                                        self.auth_token,
                                        oldname)
                client.put_container(self.storage_url,
                                     self.auth_token,
                                     newname)
            except client.ClientException:
                raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)

        else:
            old_container, _, old_object = oldname.partition('/')
            new_container, _, new_object = newname.partition('/')
            if old_container != new_container:
                raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)

            # If it is a pseudofolder, check that it is empty
            _, objects = client.get_container(
                self.storage_url,
                self.auth_token,
                container=self.container,
                delimiter='/',
                prefix=sanitize(old_object).rstrip('/') + '/',
                http_conn=self.http_connection)
            if len(objects) != 1:  # first object is the pseudofolder entry
                raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)

            # also check if the object is empty
            if objects[0].get('bytes') != 0:
                raise dav_error.DAVError(dav_error.HTTP_FORBIDDEN)

            # Do a COPY to preserve existing metadata and content-type
            client.put_object(self.storage_url,
								  self.auth_token,
								  self.container,
								  sanitize(new_object),
                                  headers = {'X-Copy-From': '/' + oldname + '/'},
								  http_conn=self.http_connection)
            client.delete_object(self.storage_url,
                                 self.auth_token,
                                 self.container,
                                 old_object + '/',
                                 http_conn=self.http_connection)
예제 #48
0
    def test_post_ctype_replicated_when_previous_incomplete_puts(self):
        # primary half                     handoff half
        # ------------                     ------------
        # t0.data: ctype = foo
        #                                  t1.data: ctype = bar
        # t2.meta: ctype = baz
        #
        #              ...run replicator and expect...
        #
        #               t1.data:
        #               t2.meta: ctype = baz
        self.brain.put_container(policy_index=0)

        # incomplete write to primary half
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._put_object(headers={'Content-Type': 'foo'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        # handoff write
        self.brain.stop_primary_half()
        self.container_brain.stop_primary_half()
        self._put_object(headers={'Content-Type': 'bar'})
        self.brain.start_primary_half()
        self.container_brain.start_primary_half()

        # content-type update to primary half
        self.brain.stop_handoff_half()
        self.container_brain.stop_handoff_half()
        self._post_object(headers={'Content-Type': 'baz'})
        self.brain.start_handoff_half()
        self.container_brain.start_handoff_half()

        self.get_to_final_state()

        # check object metadata
        metadata = client.head_object(self.url, self.token,
                                      self.container_name,
                                      self.object_name)

        # check container listing metadata
        container_metadata, objs = client.get_container(self.url, self.token,
                                                        self.container_name)

        for obj in objs:
            if obj['name'] == self.object_name:
                break
        expected = 'baz'
        self.assertEqual(obj['content_type'], expected)
        self._assert_object_metadata_matches_listing(obj, metadata)
        self._assert_consistent_container_dbs()
        self._assert_consistent_object_metadata()
        self._assert_consistent_suffix_hashes()
예제 #49
0
 def run_test(num_locks, catch_503):
     container = 'container-%s' % uuid4()
     client.put_container(self.url, self.token, container)
     # Get the container info into memcache (so no stray
     # get_container_info calls muck up our timings)
     client.get_container(self.url, self.token, container)
     db_files = self.get_container_db_files(container)
     db_conns = []
     for i in range(num_locks):
         db_conn = connect(db_files[i])
         db_conn.execute('begin exclusive transaction')
         db_conns.append(db_conn)
     if catch_503:
         try:
             client.delete_container(self.url, self.token, container)
         except client.ClientException as err:
             self.assertEqual(err.http_status, 503)
         else:
             self.fail("Expected ClientException but didn't get it")
     else:
         client.delete_container(self.url, self.token, container)
예제 #50
0
 def test_create_swift_client(self):
     mock_resp = MagicMock()
     with patch.object(swiftclient.client.Connection, 'get_container',
                       MagicMock(return_value=["text", mock_resp])):
         service_catalog = [{'endpoints': [{'region': 'RegionOne',
                                            'publicURL': 'example.com'}],
                             'type': 'object-store'}]
         client = remote.create_swift_client(TroveContext(
             tenant=uuid.uuid4().hex,
             service_catalog=service_catalog))
         headers, container = client.get_container('bob')
         self.assertIs(headers, "text")
         self.assertIs(container, mock_resp)
예제 #51
0
파일: utils.py 프로젝트: stormers/vault
def get_container_objects(container, storage_url, auth_token):
    object_list = []
    http_conn = client.http_connection(storage_url, insecure=settings.SWIFT_INSECURE)

    _, objects = client.get_container(storage_url,
                                           auth_token,
                                           container,
                                           http_conn=http_conn)

    for object in objects:
        object_list.append(object['name'])

    return object_list
예제 #52
0
 def delete_account_cont(self, account_url, token):
     cnx = sclient.http_connection(account_url)
     al = sclient.get_account(None, token, http_conn=cnx, full_listing=True)
     for container in [c['name'] for c in al[1]]:
         ci = sclient.get_container(None,
                                    token,
                                    container,
                                    http_conn=cnx,
                                    full_listing=True)
         on = [od['name'] for od in ci[1]]
         for obj in on:
             sclient.delete_object('', token, container, obj, http_conn=cnx)
         sclient.delete_container('', token, container, http_conn=cnx)
예제 #53
0
def cloudview(request, cloud, prefix=None):
    """ Returns list of all objects in a cloud. """

    storage_url = request.session.get('storage_url', settings.STORAGE_URL)
    auth_token = request.session.get('auth_token', 'demo')
    request.session['cloud'] = getcloud(request)
    objects = []
    read_acl = []
    buckets = cloud_to_bucket(cloud)
    arrayOfBuckets = buckets.split(",")

    newkey = u"{key}".format(key='bucket')
    for bucket in arrayOfBuckets:
        try:
            meta, objects1 = client.get_container(storage_url, auth_token, bucket, delimiter='/', prefix=prefix)
            for entry in objects1:
                value = u"{value}".format( value=bucket)
                entry.update({newkey:value})
            objects += objects1
            read_acl += meta.get('x-container-read', '').split(',')

        except client.ClientException:
            messages.add_message(request, messages.ERROR, _("No Access to Bucket %s." % bucket))

    #merge all buckets

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    objs.sort(key=lambda x:x['last_modified'],  reverse=True)
    del objs[100:]
    base_url = get_base_url(request)
    account = storage_url.split('/')[-1]
    public = False
    required_acl = ['.r:*', '.rlistings']
    if [x for x in read_acl if x in required_acl]:
        public = True

    return render_to_response("cloudview.html", {
        'container': buckets,
        'cloud': cloud,
        'objects': objs,
        'folders': pseudofolders,
        'session': request.session,
        'prefix': prefix,
        'prefixes': prefixes,
        'base_url': base_url,
        'account': account,
        'public': public},
        context_instance=RequestContext(request))
    def test_async_update_after_PUT(self):
        cpart, cnodes = self.container_ring.get_nodes(self.account, 'c1')
        client.put_container(self.url,
                             self.token,
                             'c1',
                             headers={'X-Storage-Policy': self.policy.name})

        # put an object while one container server is stopped so that we force
        # an async update to it
        kill_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        content = u'stuff'
        client.put_object(self.url,
                          self.token,
                          'c1',
                          'o1',
                          contents=content,
                          content_type='test/ctype')
        meta = client.head_object(self.url, self.token, 'c1', 'o1')

        # re-start the container server and assert that it does not yet know
        # about the object
        start_server((cnodes[0]['ip'], cnodes[0]['port']), self.ipport2server)
        self.assertFalse(
            direct_client.direct_get_container(cnodes[0], cpart, self.account,
                                               'c1')[1])

        # Run the object-updaters to be sure updates are done
        Manager(['object-updater']).once()

        # check the re-started container server got same update as others.
        # we cannot assert the actual etag value because it may be encrypted
        listing_etags = set()
        for cnode in cnodes:
            listing = direct_client.direct_get_container(
                cnode, cpart, self.account, 'c1')[1]
            self.assertEqual(1, len(listing))
            self.assertEqual(len(content), listing[0]['bytes'])
            self.assertEqual('test/ctype', listing[0]['content_type'])
            listing_etags.add(listing[0]['hash'])
        self.assertEqual(1, len(listing_etags))

        # check that listing meta returned to client is consistent with object
        # meta returned to client
        hdrs, listing = client.get_container(self.url, self.token, 'c1')
        self.assertEqual(1, len(listing))
        self.assertEqual('o1', listing[0]['name'])
        self.assertEqual(len(content), listing[0]['bytes'])
        self.assertEqual(meta['etag'], listing[0]['hash'])
        self.assertEqual('test/ctype', listing[0]['content_type'])
예제 #55
0
 def test_create_swift_client(self):
     mock_resp = MagicMock()
     swiftclient.client.Connection.get_container = MagicMock(
         return_value=["text", mock_resp])
     service_catalog = [{
         'endpoints': [{
             'publicURL': 'example.com'
         }],
         'type': 'object-store'
     }]
     client = remote.create_swift_client(
         TroveContext(tenant='123', service_catalog=service_catalog))
     headers, container = client.get_container('bob')
     self.assertIs(headers, "text")
     self.assertIs(container, mock_resp)
예제 #56
0
def delete_container(request, container):
    """ Deletes a container """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    try:
        _m, objects = client.get_container(storage_url, auth_token, container)
        for obj in objects:
            client.delete_object(storage_url, auth_token, container,
                                 obj['name'])
        client.delete_container(storage_url, auth_token, container)
        messages.add_message(request, messages.INFO, _("Container deleted."))
    except client.ClientException:
        messages.add_message(request, messages.ERROR, _("Access denied."))

    return redirect(containerview)
예제 #57
0
def update_container(request, project_id, container_name):

    if request.method == 'PUT':
        sp = JSONParser().parse(request)
        token = get_token_connection(request)
        url = settings.SWIFT_URL + "/AUTH_" + project_id

        headers, obj_list = swift_client.get_container(url, token,
                                                       container_name)
        headers['X-Storage-Policy'] = sp

        path_container = settings.SWIFT_CFG_TMP_DIR + "/" + container_name
        os.mkdir(path_container)

        for obj in obj_list:
            file = open(path_container + "/" + obj["name"], "w")
            obj_headers, obj_body = swift_client.get_object(
                url, token, container_name, obj["name"])
            file.write(obj_body)
            file.close()
            obj["headers"] = obj_headers
            swift_client.delete_object(url, token, container_name, obj["name"])

        swift_client.delete_container(url, token, container_name)
        swift_client.put_container(url, token, container_name, headers)

        for obj in obj_list:
            obj_path = os.path.join(path_container, obj["name"])
            obj_body = open(obj_path, "r")
            content_length = os.stat(obj_path).st_size
            swift_response = {}
            swift_client.put_object(url, token, container_name, obj["name"],
                                    obj_body, content_length, None, None,
                                    obj['content_type'], obj["headers"], None,
                                    None, None, swift_response)
            obj_body.close()
            os.remove(obj_path)

        os.rmdir(path_container)

        return JSONResponse("Container Policy updated correctly",
                            status=status.HTTP_201_CREATED)

    return JSONResponse('Method ' + str(request.method) + ' not allowed.',
                        status=status.HTTP_405_METHOD_NOT_ALLOWED)
예제 #58
0
def get_all(request, container):
    """ Returns List of all Objects present in specified container
    :param container: Name of Swift Container
    :return: meta data and id's of all the objects
    """
    try:
        response_dict = dict()
        storage_url, auth_token = _get_auth_data(request.session)
        data_container = client.get_container(storage_url, auth_token,
                                              container)
        for obj in data_container[1]:
            meta_object = client.head_object(storage_url, auth_token,
                                             container, obj['name'])
            if not meta_object['x-object-meta-deleted']:
                if meta_object['x-object-meta-type'] in [
                        'thumbnail', 'original-thumbnail'
                ]:
                    if meta_object[
                            'x-object-meta-format'] not in response_dict:
                        form = meta_object['x-object-meta-format']
                        response_dict[form] = []
                    if meta_object['x-object-meta-format'] in response_dict:
                        new_obj = {
                            'thumbnail_id': obj['name'],
                            'name': meta_object['x-object-meta-name'],
                            'type': meta_object['x-object-meta-type'],
                            'resolution':
                            meta_object['x-object-meta-resolution']
                        }
                        if meta_object['x-object-meta-format'] is 'thumbnail':
                            new_obj['original_id'] = meta_object[
                                'x-object-meta-original']
                        response_dict[form].append(new_obj)
        return Response(response_dict)
    except swift_exception.ClientException as e:
        print e
        return Response('Please contact your admininstrator',
                        status=status.HTTP_501_NOT_IMPLEMENTED)
    except Exception as e:
        print e
        return Response('Please contact your admininstrator',
                        status=status.HTTP_501_NOT_IMPLEMENTED)
예제 #59
0
def objectview(request, container, prefix=None):
    """ Returns list of all objects in current container. """

    storage_url = request.session.get('storage_url', '')
    auth_token = request.session.get('auth_token', '')

    try:
        meta, objects = client.get_container(storage_url,
                                             auth_token,
                                             container,
                                             delimiter='/',
                                             prefix=prefix)

    except client.ClientException:
        traceback.print_exc()
        messages.add_message(request, messages.ERROR, _("Access denied."))
        return redirect(containerview)

    prefixes = prefix_list(prefix)
    pseudofolders, objs = pseudofolder_object_list(objects, prefix)
    base_url = get_base_url(request)
    account = storage_url.split('/')[-1]

    read_acl = meta.get('x-container-read', '').split(',')
    public = False
    required_acl = ['.r:*', '.rlistings']
    if [x for x in read_acl if x in required_acl]:
        public = True

    return render_to_response("objectview.html", {
        'container': container,
        'objects': objs,
        'folders': pseudofolders,
        'session': request.session,
        'prefix': prefix,
        'prefixes': prefixes,
        'base_url': base_url,
        'account': account,
        'public': public
    },
                              context_instance=RequestContext(request))
예제 #60
0
    def _test_syncing(self, source_container, dest_container):
        # test syncing and versioning
        object_name = 'object-%s' % uuid.uuid4()
        client.put_object(self.url, self.token, source_container, object_name,
                          'version1')

        # cycle container-sync
        Manager(['container-sync']).once()

        # overwrite source
        client.put_object(self.url, self.token, source_container, object_name,
                          'version2')

        # cycle container-sync
        Manager(['container-sync']).once()

        resp_headers, listing = client.get_container(
            self.url, self.token, dest_container,
            query_string='versions')

        self.assertEqual(2, len(listing))