def modify_volume(vol_name, meta, new_size):
meta = {
set_meta_name(key): value.encode('hex')
for key, value in meta.iteritems()
}
meta['__SXSWIFT__'] = ''
get_sxcontroller().modifyVolume.call(vol_name,
customVolumeMeta=meta,
size=new_size)
def update_account_meta(user_name, remove_list, update_dict):
user = get_user_object(user_name)
meta = get_user_meta(user)
for key in remove_list:
meta.pop(key, None)
meta.update(update_dict)
meta = {set_meta_name(key): value for key, value in meta.iteritems()}
get_sxcontroller().modifyUser.call(user_name, desc=json.dumps(meta))
def update_account_meta(user_name, remove_list, update_dict):
user = get_user_object(user_name)
meta = get_user_meta(user)
for key in remove_list:
meta.pop(key, None)
meta.update(update_dict)
meta = {
set_meta_name(key): value
for key, value in meta.iteritems()
}
get_sxcontroller().modifyUser.call(
user_name, desc=json.dumps(meta)
)
def upload_from_stream(self, stream):
hash_stream = HashStream(stream)
sxcontroller = get_sxcontroller()
def before_flush(context):
md5 = hash_stream.get_hash()
etag = self.metadata['etag']
if etag and etag != md5:
raise UnprocessableEntity()
self.metadata['etag'] = md5
if 'content-length' not in self.metadata:
content_length = hash_stream.get_length()
self.metadata['content-length'] = content_length
meta = self._get_encoded_metadata()
sxcontroller.initializeAddChunk.call(context.token,
context.uploaded_blocks, [],
fileMeta=meta)
if self.object_path.endswith('/'):
self.object_path += '.sxnewdir'
file_uploader = SXFileUploader(sxcontroller)
file_uploader.upload_stream(
volume=self.vol_name,
file_size=self.metadata['content-length'],
file_name=self.object_path,
stream=hash_stream,
before_flush=before_flush,
)
def copy(self, meta, fresh_meta):
sxcontroller = get_sxcontroller()
metadata = get_metadata(self.src_vol, self.src_path)
clean_meta = {
'etag':
metadata.get('etag', ''),
'content-type':
metadata.get('content-type', 'application/octet-stream'),
'last-modified':
datetime_to_http_iso(datetime.utcnow()),
}
if not fresh_meta:
for key, value in meta:
clean_meta[key] = value
meta = clean_meta
file_info = sxcontroller.getFile.json_call(self.src_vol, self.src_path)
blocks = [obj.keys()[0] for obj in file_info['fileData']]
size = file_info['fileSize']
resp = sxcontroller.initializeFile.call(self.dst_vol,
self.dst_path, size, blocks,
encode_meta(meta))
token = resp.json()['uploadToken']
sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def copy(self, meta, fresh_meta):
sxcontroller = get_sxcontroller()
metadata = get_metadata(self.src_vol, self.src_path)
clean_meta = {
'etag': metadata.get('etag', ''),
'content-type': metadata.get('content-type', 'application/octet-stream'),
'last-modified': datetime_to_http_iso(datetime.utcnow()),
}
if not fresh_meta:
for key, value in meta:
clean_meta[key] = value
meta = clean_meta
file_info = sxcontroller.getFile.json_call(
self.src_vol, self.src_path
)
blocks = [obj.keys()[0] for obj in file_info['fileData']]
size = file_info['fileSize']
resp = sxcontroller.initializeFile.call(
self.dst_vol, self.dst_path, size, blocks,
encode_meta(meta)
)
token = resp.json()['uploadToken']
sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def upload_from_stream(self, stream):
hash_stream = HashStream(stream)
sxcontroller = get_sxcontroller()
def before_flush(context):
md5 = hash_stream.get_hash()
etag = self.metadata['etag']
if etag and etag != md5:
raise UnprocessableEntity()
self.metadata['etag'] = md5
if 'content-length' not in self.metadata:
content_length = hash_stream.get_length()
self.metadata['content-length'] = content_length
meta = self._get_encoded_metadata()
sxcontroller.initializeAddChunk.call(
context.token,
context.uploaded_blocks,
[],
fileMeta=meta
)
if self.object_path.endswith('/'):
self.object_path += '.sxnewdir'
file_uploader = SXFileUploader(sxcontroller)
file_uploader.upload_stream(
volume=self.vol_name,
file_size=self.metadata['content-length'],
file_name=self.object_path,
stream=hash_stream,
before_flush=before_flush,
)
def get_downloader():
global _sxdownloader
if _sxdownloader is None:
sxcontroller = get_sxcontroller()
settings = get_settings()
kwargs = {}
threads_no = settings.get("default.downloader.threads")
if threads_no is not None:
kwargs["threads_no"] = threads_no
kwargs["number_of_connections"] = threads_no
tmp_dir = settings.get("default.downloader.tmp_dir")
if tmp_dir is not None:
kwargs["tmp_dir"] = tmp_dir
cache_files = settings.get("default.downloader.cache_files")
if cache_files is not None:
kwargs["cache_files"] = cache_files
_sxdownloader = SXFileDownloader(sxcontroller, **kwargs)
_sxdownloader.initialize()
return _sxdownloader
def get_downloader():
global _sxdownloader
if _sxdownloader is None:
sxcontroller = get_sxcontroller()
settings = get_settings()
kwargs = {}
threads_no = settings.get('default.downloader.threads')
if threads_no is not None:
kwargs['threads_no'] = threads_no
kwargs['number_of_connections'] = threads_no
tmp_dir = settings.get('default.downloader.tmp_dir')
if tmp_dir is not None:
kwargs['tmp_dir'] = tmp_dir
cache_files = settings.get('default.downloader.cache_files')
if cache_files is not None:
kwargs['cache_files'] = cache_files
_sxdownloader = SXFileDownloader(sxcontroller, **kwargs)
_sxdownloader.initialize()
return _sxdownloader
def get_volume_object(name):
sxcontroller = get_sxcontroller()
try:
volume = sxcontroller.locateVolume.json_call(name, includeCustomMeta=True)
except SXClusterNotFound:
raise NotFound
return volume
def delete_volume(vol_name):
sxcontroller = get_sxcontroller()
try:
sxcontroller.deleteVolume.call(vol_name)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
# TODO: we should have better exception handling in sxclient
raise Conflict
def get_volume_object(name):
sxcontroller = get_sxcontroller()
try:
volume = sxcontroller.locateVolume.json_call(name,
includeCustomMeta=True)
except SXClusterNotFound:
raise NotFound
return volume
def delete_volume(vol_name):
sxcontroller = get_sxcontroller()
try:
sxcontroller.deleteVolume.call(vol_name)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
# TODO: we should have better exception handling in sxclient
raise Conflict
def create_volume_if_not_exists(vol_name, user_name, size, replica_count, max_revisions, meta):
try:
get_volume_object(vol_name)
return False
except NotFound:
pass
sxcontroller = get_sxcontroller()
sxcontroller.createVolume.call(vol_name, size, user_name, replica_count, max_revisions, meta)
return True
def create_user(user_name):
sxcontroller = get_sxcontroller()
password = ''.join(
random.choice(string.letters + string.digits) for i in range(16))
new_user_data = sxclient.UserData.from_userpass_pair(
user_name, password, sxcontroller.get_cluster_uuid())
sxcontroller.createUser.json_call(
userName=user_name,
userType="normal",
userKey=new_user_data.secret_key.encode('hex'))
def create_volume_if_not_exists(vol_name, user_name, size, replica_count,
max_revisions, meta):
try:
get_volume_object(vol_name)
return False
except NotFound:
pass
sxcontroller = get_sxcontroller()
sxcontroller.createVolume.call(vol_name, size, user_name, replica_count,
max_revisions, meta)
return True
def get_metadata(vol, obj):
sxcontroller = get_sxcontroller()
try:
resp = sxcontroller.getFileMeta.json_call(vol, obj)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
raise Conflict
metas = {}
for key, value in resp['fileMeta'].iteritems():
key = str(get_meta_name(key))
metas[key] = value.decode('hex')
return metas
def get_user_object(name):
if name is None:
raise NotFound
sxcontroller = get_sxcontroller()
users = sxcontroller.listUsers.json_call()
if name not in users:
create_user(name)
users = sxcontroller.listUsers.json_call()
if name not in users:
raise NotFound
return users[name]
def get_user_object(name):
if name is None:
raise NotFound
sxcontroller = get_sxcontroller()
users = sxcontroller.listUsers.json_call()
if name not in users:
create_user(name)
users = sxcontroller.listUsers.json_call()
if name not in users:
raise NotFound
return users[name]
def get_metadata(vol, obj):
sxcontroller = get_sxcontroller()
try:
resp = sxcontroller.getFileMeta.json_call(vol, obj)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
raise Conflict
metas = {}
for key, value in resp['fileMeta'].iteritems():
key = str(get_meta_name(key))
metas[key] = value.decode('hex')
return metas
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit):
sxcontroller = get_sxcontroller()
file_objects = sxcontroller.listFiles.call(vol_name, recursive=True, limit=str(limit)).content
file_objects = json.loads(file_objects, object_pairs_hook=collections.OrderedDict)
for file_name, file_object in file_objects["fileList"].iteritems():
file_name = file_name.lstrip("/")
file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name)
file_meta = file_meta.get("fileMeta", {})
last_modified = datetime.fromtimestamp(file_object["createdAt"])
last_modified = datetime_to_http_iso(last_modified)
yield {
"hash": file_meta.get("sx-etag", "").decode("hex"),
"content_type": file_meta.get("sx-content-type", "").decode("hex"),
"last_modified": last_modified,
"bytes": file_object["fileSize"],
"name": file_name,
}
def delete(self):
sxcontroller = get_sxcontroller()
if self.object_path.endswith('/'):
try:
file_objects = sxcontroller.listFiles.json_call(self.vol_name, recursive=True, filter=self.object_path)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
raise Conflict
if(len(file_objects['fileList']) != 1):
raise Conflict
self.object_path += '.sxnewdir'
try:
sxcontroller.deleteFile.call(self.vol_name, self.object_path)
except SXClusterNotFound:
raise NotFound
except SXClusterFatalError:
# TODO: we should have better exception handling in sxclient
raise Conflict
def list_files(vol_name, prefix, delimiter, start_marker, end_marker, limit):
sxcontroller = get_sxcontroller()
file_objects = sxcontroller.listFiles.call(vol_name,
recursive=True,
limit=str(limit)).content
file_objects = json.loads(file_objects,
object_pairs_hook=collections.OrderedDict)
for file_name, file_object in file_objects['fileList'].iteritems():
file_name = file_name.lstrip('/')
file_meta = sxcontroller.getFileMeta.json_call(vol_name, file_name)
file_meta = file_meta.get('fileMeta', {})
last_modified = datetime.fromtimestamp(file_object['createdAt'])
last_modified = datetime_to_http_iso(last_modified)
yield {
'hash': file_meta.get('sx-etag', '').decode('hex'),
'content_type': file_meta.get('sx-content-type', '').decode('hex'),
'last_modified': last_modified,
'bytes': file_object['fileSize'],
'name': file_name
}
def save(self, new_meta):
current_meta = get_metadata(self.vol_name, self.object_path)
key = 'content-type'
if key not in new_meta and key in current_meta:
new_meta[key] = current_meta[key]
sxcontroller = get_sxcontroller()
file_info = sxcontroller.getFile.json_call(
self.vol_name, self.object_path
)
blocks = [obj.keys()[0] for obj in file_info['fileData']]
size = file_info['fileSize']
# We create new file because this is the only way to alter
# meta on sx file.
resp = sxcontroller.initializeFile.call(
self.vol_name, self.object_path, size, blocks,
encode_meta(new_meta)
)
token = resp.json()['uploadToken']
sxcontroller.flushUploadedFile.call_on_node(resp.node_address, token)
def get_account_data(
user_name, prefix, delimiter, limit, start_marker, end_marker
):
# TODO: at the moment delimiter is ignored since I'm not 100% sure how
# it works on swift's side
sxcontroller = get_sxcontroller()
user = get_user_object(user_name)
user_meta = get_user_meta(user)
result = {
'name': user_name,
'content': None,
'timestamp': 0, # not supported by sx
'meta': {
'x-timestamp': 0,
'x-account-object-count': 0,
'x-account-container-count': 0,
'x-account-bytes-used': 0
},
}
result['meta'].update(user_meta)
content = []
volumes = sxcontroller.listVolumes.json_call()
for vol_name, vol_info in volumes['volumeList'].iteritems():
owner = vol_info['owner']
if owner != user_name:
continue
if start_marker and vol_name <= start_marker:
continue
if end_marker and vol_name > end_marker:
continue
if not vol_name.startswith(prefix):
continue
result['meta']['x-account-container-count'] += 1
if vol_info.get('filesCount'):
count = vol_info['filesCount']
result['meta']['x-account-object-count'] = count
size = vol_info['filesSize']
result['meta']['x-account-bytes-used'] = size
else:
count = 0
size = 0
sxfiles = sxcontroller.listFiles.json_call(vol_name, recursive=True)
for sxfile in sxfiles['fileList'].itervalues():
count += 1
result['meta']['x-account-object-count'] += 1
file_size = sxfile['fileSize']
size += file_size
result['meta']['x-account-bytes-used'] += file_size
content.append({
'count': count,
'bytes': size,
'name': vol_name
})
if len(result) >= limit:
break
result['content'] = sorted(content, key=lambda el: el['name'])
return result
def get_account_data(user_name, prefix, delimiter, limit, start_marker,
end_marker):
# TODO: at the moment delimiter is ignored since I'm not 100% sure how
# it works on swift's side
sxcontroller = get_sxcontroller()
user = get_user_object(user_name)
user_meta = get_user_meta(user)
result = {
'name': user_name,
'content': None,
'timestamp': 0, # not supported by sx
'meta': {
'x-timestamp': 0,
'x-account-object-count': 0,
'x-account-container-count': 0,
'x-account-bytes-used': 0
},
}
result['meta'].update(user_meta)
content = []
volumes = sxcontroller.listVolumes.json_call()
for vol_name, vol_info in volumes['volumeList'].iteritems():
owner = vol_info['owner']
if owner != user_name:
continue
if start_marker and vol_name <= start_marker:
continue
if end_marker and vol_name > end_marker:
continue
if not vol_name.startswith(prefix):
continue
result['meta']['x-account-container-count'] += 1
if vol_info.get('filesCount'):
count = vol_info['filesCount']
result['meta']['x-account-object-count'] = count
size = vol_info['filesSize']
result['meta']['x-account-bytes-used'] = size
else:
count = 0
size = 0
sxfiles = sxcontroller.listFiles.json_call(vol_name,
recursive=True)
for sxfile in sxfiles['fileList'].itervalues():
count += 1
result['meta']['x-account-object-count'] += 1
file_size = sxfile['fileSize']
size += file_size
result['meta']['x-account-bytes-used'] += file_size
content.append({'count': count, 'bytes': size, 'name': vol_name})
if len(result) >= limit:
break
result['content'] = sorted(content, key=lambda el: el['name'])
return result
def modify_volume(vol_name, meta, new_size):
meta = {set_meta_name(key): value.encode("hex") for key, value in meta.iteritems()}
meta["__SXSWIFT__"] = ""
get_sxcontroller().modifyVolume.call(vol_name, customVolumeMeta=meta, size=new_size)
def create_user(user_name):
sxcontroller = get_sxcontroller()
password = ''.join(random.choice(string.letters + string.digits) for i in range(16))
new_user_data = sxclient.UserData.from_userpass_pair(user_name, password, sxcontroller.get_cluster_uuid())
sxcontroller.createUser.json_call(userName=user_name, userType="normal", userKey=new_user_data.secret_key.encode('hex'))