def modify_volume(vol_name, meta, new_size): meta = { set_meta_name(key): value.encode('hex') for key, value in meta.iteritems() } meta['__SXSWIFT__'] = '' get_sxcontroller().modifyVolume.call(vol_name, customVolumeMeta=meta, size=new_size)
def update_account_meta(user_name, remove_list, update_dict): user = get_user_object(user_name) meta = get_user_meta(user) for key in remove_list: meta.pop(key, None) meta.update(update_dict) meta = {set_meta_name(key): value for key, value in meta.iteritems()} get_sxcontroller().modifyUser.call(user_name, desc=json.dumps(meta))
def update_account_meta(user_name, remove_list, update_dict): user = get_user_object(user_name) meta = get_user_meta(user) for key in remove_list: meta.pop(key, None) meta.update(update_dict) meta = { set_meta_name(key): value for key, value in meta.iteritems() } get_sxcontroller().modifyUser.call( user_name, desc=json.dumps(meta) )
def upload_from_stream(self, stream): hash_stream = HashStream(stream) sxcontroller = get_sxcontroller() def before_flush(context): md5 = hash_stream.get_hash() etag = self.metadata['etag'] if etag and etag != md5: raise UnprocessableEntity() self.metadata['etag'] = md5 if 'content-length' not in self.metadata: content_length = hash_stream.get_length() self.metadata['content-length'] = content_length meta = self._get_encoded_metadata() sxcontroller.initializeAddChunk.call(context.token, context.uploaded_blocks, [], fileMeta=meta) if self.object_path.endswith('/'): self.object_path += '.sxnewdir' file_uploader = SXFileUploader(sxcontroller) file_uploader.upload_stream( volume=self.vol_name, file_size=self.metadata['content-length'], file_name=self.object_path, stream=hash_stream, before_flush=before_flush, )
def copy(self, meta, fresh_meta): sxcontroller = get_sxcontroller() metadata = get_metadata(self.src_vol, self.src_path) clean_meta = { 'etag': metadata.get('etag', ''), 'content-type': metadata.get('content-type', 'application/octet-stream'), 'last-modified': datetime_to_http_iso(datetime.utcnow()), } if not fresh_meta: for key, value in meta: clean_meta[key] = value meta = clean_meta file_info = sxcontroller.getFile.json_call(self.src_vol, self.src_path) blocks = [obj.keys()[0] for obj in file_info['fileData']] size = file_info['fileSize'] resp = sxcontroller.initializeFile.call(self.dst_vol, self.dst_path, size, blocks, encode_meta(meta)) token = resp.json()['uploadToken'] sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def copy(self, meta, fresh_meta): sxcontroller = get_sxcontroller() metadata = get_metadata(self.src_vol, self.src_path) clean_meta = { 'etag': metadata.get('etag', ''), 'content-type': metadata.get('content-type', 'application/octet-stream'), 'last-modified': datetime_to_http_iso(datetime.utcnow()), } if not fresh_meta: for key, value in meta: clean_meta[key] = value meta = clean_meta file_info = sxcontroller.getFile.json_call( self.src_vol, self.src_path ) blocks = [obj.keys()[0] for obj in file_info['fileData']] size = file_info['fileSize'] resp = sxcontroller.initializeFile.call( self.dst_vol, self.dst_path, size, blocks, encode_meta(meta) ) token = resp.json()['uploadToken'] sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def upload_from_stream(self, stream): hash_stream = HashStream(stream) sxcontroller = get_sxcontroller() def before_flush(context): md5 = hash_stream.get_hash() etag = self.metadata['etag'] if etag and etag != md5: raise UnprocessableEntity() self.metadata['etag'] = md5 if 'content-length' not in self.metadata: content_length = hash_stream.get_length() self.metadata['content-length'] = content_length meta = self._get_encoded_metadata() sxcontroller.initializeAddChunk.call( context.token, context.uploaded_blocks, [], fileMeta=meta ) if self.object_path.endswith('/'): self.object_path += '.sxnewdir' file_uploader = SXFileUploader(sxcontroller) file_uploader.upload_stream( volume=self.vol_name, file_size=self.metadata['content-length'], file_name=self.object_path, stream=hash_stream, before_flush=before_flush, )
def get_downloader(): global _sxdownloader if _sxdownloader is None: sxcontroller = get_sxcontroller() settings = get_settings() kwargs = {} threads_no = settings.get("default.downloader.threads") if threads_no is not None: kwargs["threads_no"] = threads_no kwargs["number_of_connections"] = threads_no tmp_dir = settings.get("default.downloader.tmp_dir") if tmp_dir is not None: kwargs["tmp_dir"] = tmp_dir cache_files = settings.get("default.downloader.cache_files") if cache_files is not None: kwargs["cache_files"] = cache_files _sxdownloader = SXFileDownloader(sxcontroller, **kwargs) _sxdownloader.initialize() return _sxdownloader
def get_downloader(): global _sxdownloader if _sxdownloader is None: sxcontroller = get_sxcontroller() settings = get_settings() kwargs = {} threads_no = settings.get('default.downloader.threads') if threads_no is not None: kwargs['threads_no'] = threads_no kwargs['number_of_connections'] = threads_no tmp_dir = settings.get('default.downloader.tmp_dir') if tmp_dir is not None: kwargs['tmp_dir'] = tmp_dir cache_files = settings.get('default.downloader.cache_files') if cache_files is not None: kwargs['cache_files'] = cache_files _sxdownloader = SXFileDownloader(sxcontroller, **kwargs) _sxdownloader.initialize() return _sxdownloader
def get_volume_object(name): sxcontroller = get_sxcontroller() try: volume = sxcontroller.locateVolume.json_call(name, includeCustomMeta=True) except SXClusterNotFound: raise NotFound return volume
def delete_volume(vol_name): sxcontroller = get_sxcontroller() try: sxcontroller.deleteVolume.call(vol_name) except SXClusterNotFound: raise NotFound except SXClusterFatalError: # TODO: we should have better exception handling in sxclient raise Conflict
def create_volume_if_not_exists(vol_name, user_name, size, replica_count, max_revisions, meta): try: get_volume_object(vol_name) return False except NotFound: pass sxcontroller = get_sxcontroller() sxcontroller.createVolume.call(vol_name, size, user_name, replica_count, max_revisions, meta) return True
def create_user(user_name): sxcontroller = get_sxcontroller() password = ''.join( random.choice(string.letters + string.digits) for i in range(16)) new_user_data = sxclient.UserData.from_userpass_pair( user_name, password, sxcontroller.get_cluster_uuid()) sxcontroller.createUser.json_call( userName=user_name, userType="normal", userKey=new_user_data.secret_key.encode('hex'))
def get_metadata(vol, obj): sxcontroller = get_sxcontroller() try: resp = sxcontroller.getFileMeta.json_call(vol, obj) except SXClusterNotFound: raise NotFound except SXClusterFatalError: raise Conflict metas = {} for key, value in resp['fileMeta'].iteritems(): key = str(get_meta_name(key)) metas[key] = value.decode('hex') return metas
def get_user_object(name): if name is None: raise NotFound sxcontroller = get_sxcontroller() users = sxcontroller.listUsers.json_call() if name not in users: create_user(name) users = sxcontroller.listUsers.json_call() if name not in users: raise NotFound return users[name]
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit): sxcontroller = get_sxcontroller() file_objects = sxcontroller.listFiles.call(vol_name, recursive=True, limit=str(limit)).content file_objects = json.loads(file_objects, object_pairs_hook=collections.OrderedDict) for file_name, file_object in file_objects["fileList"].iteritems(): file_name = file_name.lstrip("/") file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name) file_meta = file_meta.get("fileMeta", {}) last_modified = datetime.fromtimestamp(file_object["createdAt"]) last_modified = datetime_to_http_iso(last_modified) yield { "hash": file_meta.get("sx-etag", "").decode("hex"), "content_type": file_meta.get("sx-content-type", "").decode("hex"), "last_modified": last_modified, "bytes": file_object["fileSize"], "name": file_name, }
def delete(self): sxcontroller = get_sxcontroller() if self.object_path.endswith('/'): try: file_objects = sxcontroller.listFiles.json_call(self.vol_name, recursive=True, filter=self.object_path) except SXClusterNotFound: raise NotFound except SXClusterFatalError: raise Conflict if(len(file_objects['fileList']) != 1): raise Conflict self.object_path += '.sxnewdir' try: sxcontroller.deleteFile.call(self.vol_name, self.object_path) except SXClusterNotFound: raise NotFound except SXClusterFatalError: # TODO: we should have better exception handling in sxclient raise Conflict
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit): sxcontroller = get_sxcontroller() file_objects = sxcontroller.listFiles.call(vol_name, recursive=True, limit=str(limit)).content file_objects = json.loads(file_objects, object_pairs_hook=collections.OrderedDict) for file_name, file_object in file_objects['fileList'].iteritems(): file_name = file_name.lstrip('/') file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name) file_meta = file_meta.get('fileMeta', {}) last_modified = datetime.fromtimestamp(file_object['createdAt']) last_modified = datetime_to_http_iso(last_modified) yield { 'hash': file_meta.get('sx-etag', '').decode('hex'), 'content_type': file_meta.get('sx-content-type', '').decode('hex'), 'last_modified': last_modified, 'bytes': file_object['fileSize'], 'name': file_name }
def save(self, new_meta): current_meta = get_metadata(self.vol_name, self.object_path) key = 'content-type' if key not in new_meta and key in current_meta: new_meta[key] = current_meta[key] sxcontroller = get_sxcontroller() file_info = sxcontroller.getFile.json_call( self.vol_name, self.object_path ) blocks = [obj.keys()[0] for obj in file_info['fileData']] size = file_info['fileSize'] # We create new file because this is the only way to alter # meta on sx file. resp = sxcontroller.initializeFile.call( self.vol_name, self.object_path, size, blocks, encode_meta(new_meta) ) token = resp.json()['uploadToken'] sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def get_account_data( user_name, prefix, delimiter, limit, start_marker, end_marker ): # TODO: at the moment delimiter is ignored since I'm not 100% sure how # it works on swift's side sxcontroller = get_sxcontroller() user = get_user_object(user_name) user_meta = get_user_meta(user) result = { 'name': user_name, 'content': None, 'timestamp': 0, # not supported by sx 'meta': { 'x-timestamp': 0, 'x-account-object-count': 0, 'x-account-container-count': 0, 'x-account-bytes-used': 0 }, } result['meta'].update(user_meta) content = [] volumes = sxcontroller.listVolumes.json_call() for vol_name, vol_info in volumes['volumeList'].iteritems(): owner = vol_info['owner'] if owner != user_name: continue if start_marker and vol_name <= start_marker: continue if end_marker and vol_name > end_marker: continue if not vol_name.startswith(prefix): continue result['meta']['x-account-container-count'] += 1 if vol_info.get('filesCount'): count = vol_info['filesCount'] result['meta']['x-account-object-count'] = count size = vol_info['filesSize'] result['meta']['x-account-bytes-used'] = size else: count = 0 size = 0 sxfiles = sxcontroller.listFiles.json_call(vol_name, recursive=True) for sxfile in sxfiles['fileList'].itervalues(): count += 1 result['meta']['x-account-object-count'] += 1 file_size = sxfile['fileSize'] size += file_size result['meta']['x-account-bytes-used'] += file_size content.append({ 'count': count, 'bytes': size, 'name': vol_name }) if len(result) >= limit: break result['content'] = sorted(content, key=lambda el: el['name']) return result
def get_account_data(user_name, prefix, delimiter, limit, start_marker, end_marker): # TODO: at the moment delimiter is ignored since I'm not 100% sure how # it works on swift's side sxcontroller = get_sxcontroller() user = get_user_object(user_name) user_meta = get_user_meta(user) result = { 'name': user_name, 'content': None, 'timestamp': 0, # not supported by sx 'meta': { 'x-timestamp': 0, 'x-account-object-count': 0, 'x-account-container-count': 0, 'x-account-bytes-used': 0 }, } result['meta'].update(user_meta) content = [] volumes = sxcontroller.listVolumes.json_call() for vol_name, vol_info in volumes['volumeList'].iteritems(): owner = vol_info['owner'] if owner != user_name: continue if start_marker and vol_name <= start_marker: continue if end_marker and vol_name > end_marker: continue if not vol_name.startswith(prefix): continue result['meta']['x-account-container-count'] += 1 if vol_info.get('filesCount'): count = vol_info['filesCount'] result['meta']['x-account-object-count'] = count size = vol_info['filesSize'] result['meta']['x-account-bytes-used'] = size else: count = 0 size = 0 sxfiles = sxcontroller.listFiles.json_call(vol_name, recursive=True) for sxfile in sxfiles['fileList'].itervalues(): count += 1 result['meta']['x-account-object-count'] += 1 file_size = sxfile['fileSize'] size += file_size result['meta']['x-account-bytes-used'] += file_size content.append({'count': count, 'bytes': size, 'name': vol_name}) if len(result) >= limit: break result['content'] = sorted(content, key=lambda el: el['name']) return result
def modify_volume(vol_name, meta, new_size): meta = {set_meta_name(key): value.encode("hex") for key, value in meta.iteritems()} meta["__SXSWIFT__"] = "" get_sxcontroller().modifyVolume.call(vol_name, customVolumeMeta=meta, size=new_size)
def create_user(user_name): sxcontroller = get_sxcontroller() password = ''.join(random.choice(string.letters + string.digits) for i in range(16)) new_user_data = sxclient.UserData.from_userpass_pair(user_name, password, sxcontroller.get_cluster_uuid()) sxcontroller.createUser.json_call(userName=user_name, userType="normal", userKey=new_user_data.secret_key.encode('hex'))