Ejemplo n.º 1
0
def modify_volume(vol_name, meta, new_size):
    meta = {
        set_meta_name(key): value.encode('hex')
        for key, value in meta.iteritems()
    }
    meta['__SXSWIFT__'] = ''
    get_sxcontroller().modifyVolume.call(vol_name,
                                         customVolumeMeta=meta,
                                         size=new_size)
Ejemplo n.º 2
0
def update_account_meta(user_name, remove_list, update_dict):
    user = get_user_object(user_name)
    meta = get_user_meta(user)
    for key in remove_list:
        meta.pop(key, None)

    meta.update(update_dict)
    meta = {set_meta_name(key): value for key, value in meta.iteritems()}
    get_sxcontroller().modifyUser.call(user_name, desc=json.dumps(meta))
Ejemplo n.º 3
0
def update_account_meta(user_name, remove_list, update_dict):
    user = get_user_object(user_name)
    meta = get_user_meta(user)
    for key in remove_list:
        meta.pop(key, None)

    meta.update(update_dict)
    meta = {
        set_meta_name(key): value
        for key, value in meta.iteritems()
    }
    get_sxcontroller().modifyUser.call(
        user_name, desc=json.dumps(meta)
    )
Ejemplo n.º 4
0
    def upload_from_stream(self, stream):
        hash_stream = HashStream(stream)
        sxcontroller = get_sxcontroller()

        def before_flush(context):
            md5 = hash_stream.get_hash()
            etag = self.metadata['etag']
            if etag and etag != md5:
                raise UnprocessableEntity()

            self.metadata['etag'] = md5
            if 'content-length' not in self.metadata:
                content_length = hash_stream.get_length()
                self.metadata['content-length'] = content_length

            meta = self._get_encoded_metadata()
            sxcontroller.initializeAddChunk.call(context.token,
                                                 context.uploaded_blocks, [],
                                                 fileMeta=meta)

        if self.object_path.endswith('/'):
            self.object_path += '.sxnewdir'

        file_uploader = SXFileUploader(sxcontroller)
        file_uploader.upload_stream(
            volume=self.vol_name,
            file_size=self.metadata['content-length'],
            file_name=self.object_path,
            stream=hash_stream,
            before_flush=before_flush,
        )
Ejemplo n.º 5
0
    def copy(self, meta, fresh_meta):
        sxcontroller = get_sxcontroller()
        metadata = get_metadata(self.src_vol, self.src_path)

        clean_meta = {
            'etag':
            metadata.get('etag', ''),
            'content-type':
            metadata.get('content-type', 'application/octet-stream'),
            'last-modified':
            datetime_to_http_iso(datetime.utcnow()),
        }

        if not fresh_meta:
            for key, value in meta:
                clean_meta[key] = value
        meta = clean_meta

        file_info = sxcontroller.getFile.json_call(self.src_vol, self.src_path)

        blocks = [obj.keys()[0] for obj in file_info['fileData']]
        size = file_info['fileSize']

        resp = sxcontroller.initializeFile.call(self.dst_vol,
                                                self.dst_path, size, blocks,
                                                encode_meta(meta))
        token = resp.json()['uploadToken']

        sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
Ejemplo n.º 6
0
    def copy(self, meta, fresh_meta):
        sxcontroller = get_sxcontroller()
        metadata = get_metadata(self.src_vol, self.src_path)

        clean_meta = {
            'etag': metadata.get('etag', ''),
            'content-type': metadata.get('content-type', 'application/octet-stream'),
            'last-modified': datetime_to_http_iso(datetime.utcnow()),
        }

        if not fresh_meta:
            for key, value in meta:
                clean_meta[key] = value
        meta = clean_meta

        file_info = sxcontroller.getFile.json_call(
            self.src_vol, self.src_path
        )

        blocks = [obj.keys()[0] for obj in file_info['fileData']]
        size = file_info['fileSize']

        resp = sxcontroller.initializeFile.call(
            self.dst_vol, self.dst_path, size, blocks,
            encode_meta(meta)
        )
        token = resp.json()['uploadToken']

        sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
Ejemplo n.º 7
0
    def upload_from_stream(self, stream):
        hash_stream = HashStream(stream)
        sxcontroller = get_sxcontroller()

        def before_flush(context):
            md5 = hash_stream.get_hash()
            etag = self.metadata['etag']
            if etag and etag != md5:
                raise UnprocessableEntity()

            self.metadata['etag'] = md5
            if 'content-length' not in self.metadata:
                content_length = hash_stream.get_length()
                self.metadata['content-length'] = content_length

            meta = self._get_encoded_metadata()
            sxcontroller.initializeAddChunk.call(
                context.token,
                context.uploaded_blocks,
                [],
                fileMeta=meta
            )

        if self.object_path.endswith('/'):
            self.object_path += '.sxnewdir'

        file_uploader = SXFileUploader(sxcontroller)
        file_uploader.upload_stream(
            volume=self.vol_name,
            file_size=self.metadata['content-length'],
            file_name=self.object_path,
            stream=hash_stream,
            before_flush=before_flush,
        )
Ejemplo n.º 8
0
def get_downloader():
    global _sxdownloader

    if _sxdownloader is None:
        sxcontroller = get_sxcontroller()
        settings = get_settings()

        kwargs = {}

        threads_no = settings.get("default.downloader.threads")
        if threads_no is not None:
            kwargs["threads_no"] = threads_no
            kwargs["number_of_connections"] = threads_no

        tmp_dir = settings.get("default.downloader.tmp_dir")
        if tmp_dir is not None:
            kwargs["tmp_dir"] = tmp_dir

        cache_files = settings.get("default.downloader.cache_files")
        if cache_files is not None:
            kwargs["cache_files"] = cache_files

        _sxdownloader = SXFileDownloader(sxcontroller, **kwargs)
        _sxdownloader.initialize()

    return _sxdownloader
Ejemplo n.º 9
0
def get_downloader():
    global _sxdownloader

    if _sxdownloader is None:
        sxcontroller = get_sxcontroller()
        settings = get_settings()

        kwargs = {}

        threads_no = settings.get('default.downloader.threads')
        if threads_no is not None:
            kwargs['threads_no'] = threads_no
            kwargs['number_of_connections'] = threads_no

        tmp_dir = settings.get('default.downloader.tmp_dir')
        if tmp_dir is not None:
            kwargs['tmp_dir'] = tmp_dir

        cache_files = settings.get('default.downloader.cache_files')
        if cache_files is not None:
            kwargs['cache_files'] = cache_files

        _sxdownloader = SXFileDownloader(sxcontroller, **kwargs)
        _sxdownloader.initialize()

    return _sxdownloader
Ejemplo n.º 10
0
def get_volume_object(name):
    sxcontroller = get_sxcontroller()
    try:
        volume = sxcontroller.locateVolume.json_call(name, includeCustomMeta=True)
    except SXClusterNotFound:
        raise NotFound

    return volume
Ejemplo n.º 11
0
def delete_volume(vol_name):
    sxcontroller = get_sxcontroller()
    try:
        sxcontroller.deleteVolume.call(vol_name)
    except SXClusterNotFound:
        raise NotFound
    except SXClusterFatalError:
        # TODO: we should have better exception handling in sxclient
        raise Conflict
Ejemplo n.º 12
0
def get_volume_object(name):
    sxcontroller = get_sxcontroller()
    try:
        volume = sxcontroller.locateVolume.json_call(name,
                                                     includeCustomMeta=True)
    except SXClusterNotFound:
        raise NotFound

    return volume
Ejemplo n.º 13
0
def delete_volume(vol_name):
    sxcontroller = get_sxcontroller()
    try:
        sxcontroller.deleteVolume.call(vol_name)
    except SXClusterNotFound:
        raise NotFound
    except SXClusterFatalError:
        # TODO: we should have better exception handling in sxclient
        raise Conflict
Ejemplo n.º 14
0
def create_volume_if_not_exists(vol_name, user_name, size, replica_count, max_revisions, meta):
    try:
        get_volume_object(vol_name)
        return False
    except NotFound:
        pass

    sxcontroller = get_sxcontroller()
    sxcontroller.createVolume.call(vol_name, size, user_name, replica_count, max_revisions, meta)
    return True
Ejemplo n.º 15
0
def create_user(user_name):
    sxcontroller = get_sxcontroller()
    password = ''.join(
        random.choice(string.letters + string.digits) for i in range(16))
    new_user_data = sxclient.UserData.from_userpass_pair(
        user_name, password, sxcontroller.get_cluster_uuid())
    sxcontroller.createUser.json_call(
        userName=user_name,
        userType="normal",
        userKey=new_user_data.secret_key.encode('hex'))
Ejemplo n.º 16
0
def create_volume_if_not_exists(vol_name, user_name, size, replica_count,
                                max_revisions, meta):
    try:
        get_volume_object(vol_name)
        return False
    except NotFound:
        pass

    sxcontroller = get_sxcontroller()
    sxcontroller.createVolume.call(vol_name, size, user_name, replica_count,
                                   max_revisions, meta)
    return True
Ejemplo n.º 17
0
def get_metadata(vol, obj):
    sxcontroller = get_sxcontroller()
    try:
        resp = sxcontroller.getFileMeta.json_call(vol, obj)
    except SXClusterNotFound:
        raise NotFound
    except SXClusterFatalError:
        raise Conflict

    metas = {}
    for key, value in resp['fileMeta'].iteritems():
        key = str(get_meta_name(key))
        metas[key] = value.decode('hex')
    return metas
Ejemplo n.º 18
0
def get_user_object(name):
    if name is None:
        raise NotFound

    sxcontroller = get_sxcontroller()
    users = sxcontroller.listUsers.json_call()

    if name not in users:
        create_user(name)
        users = sxcontroller.listUsers.json_call()
        if name not in users:
            raise NotFound

    return users[name]
Ejemplo n.º 19
0
def get_user_object(name):
    if name is None:
        raise NotFound

    sxcontroller = get_sxcontroller()
    users = sxcontroller.listUsers.json_call()

    if name not in users:
        create_user(name)
        users = sxcontroller.listUsers.json_call()
        if name not in users:
            raise NotFound

    return users[name]
Ejemplo n.º 20
0
def get_metadata(vol, obj):
    sxcontroller = get_sxcontroller()
    try:
        resp = sxcontroller.getFileMeta.json_call(vol, obj)
    except SXClusterNotFound:
        raise NotFound
    except SXClusterFatalError:
        raise Conflict

    metas = {}
    for key, value in resp['fileMeta'].iteritems():
        key = str(get_meta_name(key))
        metas[key] = value.decode('hex')
    return metas
Ejemplo n.º 21
0
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit):
    sxcontroller = get_sxcontroller()
    file_objects = sxcontroller.listFiles.call(vol_name, recursive=True, limit=str(limit)).content
    file_objects = json.loads(file_objects, object_pairs_hook=collections.OrderedDict)
    for file_name, file_object in file_objects["fileList"].iteritems():
        file_name = file_name.lstrip("/")
        file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name)
        file_meta = file_meta.get("fileMeta", {})
        last_modified = datetime.fromtimestamp(file_object["createdAt"])
        last_modified = datetime_to_http_iso(last_modified)
        yield {
            "hash": file_meta.get("sx-etag", "").decode("hex"),
            "content_type": file_meta.get("sx-content-type", "").decode("hex"),
            "last_modified": last_modified,
            "bytes": file_object["fileSize"],
            "name": file_name,
        }
Ejemplo n.º 22
0
 def delete(self):
     sxcontroller = get_sxcontroller()
     if self.object_path.endswith('/'):
         try:
             file_objects = sxcontroller.listFiles.json_call(self.vol_name, recursive=True, filter=self.object_path)
         except SXClusterNotFound:
             raise NotFound
         except SXClusterFatalError:
             raise Conflict
         if(len(file_objects['fileList']) != 1):
             raise Conflict
         self.object_path += '.sxnewdir'
     try:
         sxcontroller.deleteFile.call(self.vol_name, self.object_path)
     except SXClusterNotFound:
         raise NotFound
     except SXClusterFatalError:
         # TODO: we should have better exception handling in sxclient
         raise Conflict
Ejemplo n.º 23
0
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit):
    sxcontroller = get_sxcontroller()
    file_objects = sxcontroller.listFiles.call(vol_name,
                                               recursive=True,
                                               limit=str(limit)).content
    file_objects = json.loads(file_objects,
                              object_pairs_hook=collections.OrderedDict)
    for file_name, file_object in file_objects['fileList'].iteritems():
        file_name = file_name.lstrip('/')
        file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name)
        file_meta = file_meta.get('fileMeta', {})
        last_modified = datetime.fromtimestamp(file_object['createdAt'])
        last_modified = datetime_to_http_iso(last_modified)
        yield {
            'hash': file_meta.get('sx-etag', '').decode('hex'),
            'content_type': file_meta.get('sx-content-type', '').decode('hex'),
            'last_modified': last_modified,
            'bytes': file_object['fileSize'],
            'name': file_name
        }
Ejemplo n.º 24
0
    def save(self, new_meta):
        current_meta = get_metadata(self.vol_name, self.object_path)

        key = 'content-type'
        if key not in new_meta and key in current_meta:
            new_meta[key] = current_meta[key]

        sxcontroller = get_sxcontroller()
        file_info = sxcontroller.getFile.json_call(
            self.vol_name, self.object_path
        )

        blocks = [obj.keys()[0] for obj in file_info['fileData']]
        size = file_info['fileSize']

        # We create new file because this is the only way to alter
        # meta on sx file.
        resp = sxcontroller.initializeFile.call(
            self.vol_name, self.object_path, size, blocks,
            encode_meta(new_meta)
        )
        token = resp.json()['uploadToken']

        sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
Ejemplo n.º 25
0
def get_account_data(
    user_name, prefix, delimiter, limit, start_marker, end_marker
):
    # TODO: at the moment delimiter is ignored since I'm not 100% sure how
    # it works on swift's side
    sxcontroller = get_sxcontroller()
    user = get_user_object(user_name)
    user_meta = get_user_meta(user)

    result = {
        'name': user_name,
        'content': None,
        'timestamp': 0,  # not supported by sx
        'meta': {
            'x-timestamp': 0,
            'x-account-object-count': 0,
            'x-account-container-count': 0,
            'x-account-bytes-used': 0
        },
    }
    result['meta'].update(user_meta)

    content = []

    volumes = sxcontroller.listVolumes.json_call()
    for vol_name, vol_info in volumes['volumeList'].iteritems():
        owner = vol_info['owner']
        if owner != user_name:
            continue

        if start_marker and vol_name <= start_marker:
            continue

        if end_marker and vol_name > end_marker:
            continue

        if not vol_name.startswith(prefix):
            continue

        result['meta']['x-account-container-count'] += 1

        if vol_info.get('filesCount'):
            count = vol_info['filesCount']
            result['meta']['x-account-object-count'] = count
            size = vol_info['filesSize']
            result['meta']['x-account-bytes-used'] = size
        else:
            count = 0
            size = 0
            sxfiles = sxcontroller.listFiles.json_call(vol_name, recursive=True)
            for sxfile in sxfiles['fileList'].itervalues():
                count += 1
                result['meta']['x-account-object-count'] += 1
                file_size = sxfile['fileSize']
                size += file_size
                result['meta']['x-account-bytes-used'] += file_size

        content.append({
            'count': count,
            'bytes': size,
            'name': vol_name
        })

        if len(result) >= limit:
            break

    result['content'] = sorted(content, key=lambda el: el['name'])
    return result
Ejemplo n.º 26
0
def get_account_data(user_name, prefix, delimiter, limit, start_marker,
                     end_marker):
    # TODO: at the moment delimiter is ignored since I'm not 100% sure how
    # it works on swift's side
    sxcontroller = get_sxcontroller()
    user = get_user_object(user_name)
    user_meta = get_user_meta(user)

    result = {
        'name': user_name,
        'content': None,
        'timestamp': 0,  # not supported by sx
        'meta': {
            'x-timestamp': 0,
            'x-account-object-count': 0,
            'x-account-container-count': 0,
            'x-account-bytes-used': 0
        },
    }
    result['meta'].update(user_meta)

    content = []

    volumes = sxcontroller.listVolumes.json_call()
    for vol_name, vol_info in volumes['volumeList'].iteritems():
        owner = vol_info['owner']
        if owner != user_name:
            continue

        if start_marker and vol_name <= start_marker:
            continue

        if end_marker and vol_name > end_marker:
            continue

        if not vol_name.startswith(prefix):
            continue

        result['meta']['x-account-container-count'] += 1

        if vol_info.get('filesCount'):
            count = vol_info['filesCount']
            result['meta']['x-account-object-count'] = count
            size = vol_info['filesSize']
            result['meta']['x-account-bytes-used'] = size
        else:
            count = 0
            size = 0
            sxfiles = sxcontroller.listFiles.json_call(vol_name,
                                                       recursive=True)
            for sxfile in sxfiles['fileList'].itervalues():
                count += 1
                result['meta']['x-account-object-count'] += 1
                file_size = sxfile['fileSize']
                size += file_size
                result['meta']['x-account-bytes-used'] += file_size

        content.append({'count': count, 'bytes': size, 'name': vol_name})

        if len(result) >= limit:
            break

    result['content'] = sorted(content, key=lambda el: el['name'])
    return result
Ejemplo n.º 27
0
def modify_volume(vol_name, meta, new_size):
    meta = {set_meta_name(key): value.encode("hex") for key, value in meta.iteritems()}
    meta["__SXSWIFT__"] = ""
    get_sxcontroller().modifyVolume.call(vol_name, customVolumeMeta=meta, size=new_size)
Ejemplo n.º 28
0
def create_user(user_name):
    sxcontroller = get_sxcontroller()
    password = ''.join(random.choice(string.letters + string.digits) for i in range(16))
    new_user_data = sxclient.UserData.from_userpass_pair(user_name, password, sxcontroller.get_cluster_uuid())
    sxcontroller.createUser.json_call(userName=user_name, userType="normal", userKey=new_user_data.secret_key.encode('hex'))