def _do_write(self, path, content):
# check parent directory exists
self.checkParentDirExists(path)
type = self.guess_type(path)
things = []
if type == "directory":
self.log.debug("SwiftFS._do_write create directory")
things.append(SwiftUploadObject(None, object_name=path))
else:
self.log.debug("SwiftFS._do_write create file/notebook from '%s'",
content)
output = io.BytesIO(content.encode('utf-8'))
things.append(SwiftUploadObject(output, object_name=path))
# Now do the upload
path = self.clean_path(path)
try:
response = self.swift.upload(self.container, things)
except SwiftError as e:
self.log.error("SwiftFS._do_write swift-error: %s", e.value)
raise
except ClientException as e:
self.log.error("SwiftFS._do_write client-error: %s", e.value)
raise
for r in response:
self.log.debug("SwiftFS._do_write action: '%s', response: '%s'",
r['action'], r['success'])
def mock_import_analysis(self, name='', description='', details='', filepath='', params='', inputs='', outputs=''):
filename = filepath
created = 'mm-dd-yyyy 00:00:00.000'
user = '******'
sessionQuery = self.session.query(Analysis).from_statement(text("SELECT * FROM analysis where name=:name")).\
params(name=name).first().side_effect = mod_se
checkMod = sessionQuery(name=name)
if(checkMod is None):
analysisMod = Mock(spec=Analysis(name=name, filepath=filename, description=description, details=details, created=created, user=user, parameters=params, inputs=inputs, outputs=outputs))
options = {'os_auth_url': self.config['SWIFT_AUTH_URL'], 'os_username': self.config['SWIFT_USERNAME'], 'os_password': self.config['SWIFT_PASSWORD'], 'os_tenant_id': self.config['SWIFT_TENANT_ID'], 'os_tenant_name': self.config['SWIFT_TENANT_NAME']}
swiftService = Mock(spec=SwiftService(options=options))
objects = []
objects.append(SwiftUploadObject(self.config['DB_LOCATION'], object_name='sqlite.db'))
objects.append(SwiftUploadObject(filepath, object_name=filename))
swiftUpload = swiftService.upload(container='containerModules', objects=objects).return_value = ('Metadata', 'Module')
for uploaded in swiftUpload:
print(uploaded)
self.session.add(analysisMod)
self.session.commit()
return "import_success"
else:
raise RuntimeError("Analysis " + name + " already exists")
def handle(self, *args, **options):
_opts = {'object_uu_threads': 20}
dir = 'media' # settings.MEDIA_ROOT
container = settings.SWIFT_CONTAINER_NAME
with SwiftService(
options=_opts) as swift, OutputManager() as out_manager:
try:
# Collect all the files and folders in the given directory
objs = []
dir_markers = []
# change to directory so it isn't uploaded as part of all the
# file object names
os.chdir(dir)
for (_dir, _ds, _fs) in os.walk('.'):
if not (_ds + _fs):
dir_markers.append(_dir)
else:
objs.extend([os.path.join(_dir, _f) for _f in _fs])
# Now that we've collected all the required files and dir markers
# build the ``SwiftUploadObject``s for the call to upload
objs = [
SwiftUploadObject(o, object_name=o.replace(dir, dir, 1))
for o in objs
]
dir_markers = [
SwiftUploadObject(None,
object_name=d.replace(dir, dir, 1),
options={'dir_marker': True})
for d in dir_markers
]
# Schedule uploads on the SwiftService thread pool and iterate
# over the results
for r in swift.upload(container, objs + dir_markers):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s"
% (container, r['object'], error))
else:
logger.error("%s" % error)
os.chdir('..')
except SwiftError as e:
logger.error(e.value)
def main(dp_env: str, container_prefix: str, schema_base_url: str) -> None:
ROOT_PKG_NAME = "amsterdam_schema"
with TemporaryDirectory() as temp_dir:
files_root = Path(temp_dir)
# First fetch the datasets and schemas
schema_pub_paths = fetch_local_as_publishable(ROOT_PKG_NAME, files_root)
if schema_base_url is not None:
replace_schema_base_url(files_root, schema_base_url)
index_file_obj = get_index_file_obj(schema_pub_paths, files_root)
# Then fetch the documentation
doc_pub_paths = fetch_local_as_publishable(
ROOT_PKG_NAME, files_root, ("docs",), "**/*.html"
)
with SwiftService() as swift:
container = f"{container_prefix}{dp_env}"
# Delete old objects in datasets
deletes = swift.delete(container, options={"prefix": "datasets"})
for r in deletes:
if not r["success"]:
logger.warn(
f"Warning: Remote object {container}/{r['object']} could not be removed."
)
# Add new objects
upload_objects = [SwiftUploadObject(index_file_obj, object_name="datasets/index.json")]
for schema_path_parts in schema_pub_paths:
object_name = create_object_name(schema_path_parts)
upload_objects.append(
SwiftUploadObject(
str(files_root / "/".join(schema_path_parts)),
object_name=object_name,
options={"header": ["content-type:application/json"]},
)
)
for doc_path_parts in doc_pub_paths:
object_name = "/".join(doc_path_parts[1:])
upload_objects.append(
SwiftUploadObject(
str(files_root / "/".join(doc_path_parts)),
object_name=object_name,
options={"header": ["content-type:text/html"]},
)
)
uploads = swift.upload(container, upload_objects)
errors = False
for r in uploads:
if not r["success"]:
errors = True
logger.error(r["error"])
if errors:
raise Exception("Failed to publish schemas")
def saveObjsBackend(objs, backend, config):
if(backend == 'hdfs'):
for obj in objs:
try:
# obj[0] is hdfs path and obj[1] is local filesystem path
if '.py' in obj[1]: # If you are uploading a module do not allow it to be overwritten
subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]])
else: # If it is the metadata then it has to be overwritten everytime
subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]])
except Exception as e:
shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
raise RuntimeError(e)
elif(backend == 'swift'):
options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os .environ['OS_TENANT_NAME']}
swiftService = SwiftService(options=options)
objects = []
for obj in objs:
objects.append(SwiftUploadObject(obj[1], object_name=obj[0]))
swiftUpload = swiftService.upload(container='containerModules', objects=objects)
for uploaded in swiftUpload:
if("error" in uploaded.keys()):
shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH'])
raise RuntimeError(uploaded['error'])
elif(backend == 'nfs'):
for obj in objs:
shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0])
print('Metadata/Module changed and uploaded')
def upload(bucket,
name,
from_folder,
ttl=None,
segment_size="400M",
buffer_size=0):
global upload_buffer
from_path = os.path.join(from_folder, name)
headers = []
if ttl:
delete_after_header = 'x-delete-after:%i' % ttl
headers.append(delete_after_header)
options = {
'header': headers,
'segment_size': __normalized_segment_size(segment_size)
}
upload_object = SwiftUploadObject(from_path,
object_name=name,
options=options)
if bucket not in upload_buffer:
upload_buffer[bucket] = []
upload_buffer[bucket].append(upload_object)
if len(upload_buffer[bucket]) <= buffer_size:
return
flush_upload_buffer(bucket)
def swift_file_commit(_opts, container, obj_name, obj_source=None):
with SwiftService(options=_opts) as swift, OutputManager() as out_manager:
try:
if obj_source is None:
obj_source = obj_name # file path
#check if file exists
if (not os.path.exists(obj_source)):
print('File %s does not exist' % obj_source)
exit()
objs = [SwiftUploadObject(obj_source, obj_name)
] # (source, object name) --> (value,key)
for r in swift.upload(container, objs):
if r['success']:
if 'object' in r:
print(r['object'])
elif 'for_object' in r:
print('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
logger.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
logger.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error))
else:
logger.error("%s" % error)
except SwiftError as e:
logger.error(e.value)
def uploadfiles(container, options, objs):
# first argument is the container to upload file to
# second argument is a dictionary of the options to supply to SwiftUploadObject
# third argument is a list of the files to upload
with SwiftService() as swift:
try:
# create the SwiftUploadObject list of objects to upload
#objs = [SwiftUploadObject(obj, options=options) for obj in objs]
objs = [SwiftUploadObject(obj) for obj in objs]
# Schedule uploads on the SwiftService thread pool and iterate over the results
#for result in swift.upload(container, objs):
for result in swift.upload(container, objs):
if result['success']:
if 'object' in result:
print("Successfully uploaded %s." % result['object'])
elif 'for_object' in result:
print('%s segment %s' %
(result['for_object'], result['segment_index']))
else:
error = result['error']
if result['action'] == "create_container":
print('Warning: failed to create container '
"'%s'%s", container, error)
elif result['action'] == "upload_object":
print(
"Failed to upload object %s to container %s: %s" %
(container, result['object'], error))
else:
print("%s" % error)
except SwiftError as e:
print("SwiftError: %s" % e)
def upload_from_file(self, path):
"""
Stores the contents of the file pointed by the ``path`` variable.
If the file is bigger than 5 Gig, it will be broken into segments.
:type path: ``str``
:param path: Absolute path to the file to be uploaded to Swift.
:rtype: ``bool``
:return: ``True`` if successful, ``False`` if not.
.. note::
* The size of the segments chosen (or any of the other upload
options) is not under user control.
* If called this method will remap the
``swiftclient.service.get_conn`` factory method to
``self._provider._connect_swift``
.. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa
"""
upload_options = {}
if 'segment_size' not in upload_options:
if os.path.getsize(path) >= FIVE_GIG:
upload_options['segment_size'] = FIVE_GIG
# remap the swift service's connection factory method
swiftclient.service.get_conn = self._provider._connect_swift
result = True
with SwiftService() as swift:
upload_object = SwiftUploadObject(path, object_name=self.name)
for up_res in swift.upload(self.cbcontainer.name,
[upload_object, ],
options=upload_options):
result = result and up_res['success']
return result
def _put(self, source_path, remote_filename):
lp = util.fsdecode(source_path.name)
if config.mp_segment_size > 0:
from swiftclient.service import SwiftUploadObject
st = os.stat(lp)
# only upload using Dynamic Large Object if mpvolsize is triggered
if st.st_size >= config.mp_segment_size:
log.Debug(u"Uploading Dynamic Large Object")
mp = self.svc.upload(
self.container, [
SwiftUploadObject(lp,
object_name=self.prefix +
util.fsdecode(remote_filename))
],
options={u'segment_size': config.mp_segment_size})
uploads = [a for a in mp if u'container' not in a[u'action']]
for upload in uploads:
if not upload[u'success']:
raise BackendException(upload[u'traceback'])
return
rp = self.prefix + util.fsdecode(remote_filename)
log.Debug(u"Uploading '%s' to '%s' in remote container '%s'" %
(lp, rp, self.container))
self.conn.put_object(container=self.container,
obj=self.prefix + util.fsdecode(remote_filename),
contents=open(lp, u'rb'))
def migrate_SLO(container_name, object_name, src_head, src_srvclient,
tgt_srvclient):
"""Migrate static large object.
Note that static large object (slo) does not actually work with RGW as of
0.9.4.x, so hopefully not too many of these. We migrate them anyway.
It's a little difficult to upload the SLO manifest file before we are sure
all the segments are uploaded to Swift. When the PUT operation sees the
?multipart-manifest=put query parameter, it reads the request body and
verifies that each segment object exists and that the sizes and ETags
match. If there is a mismatch, the PUT operation fails.
"""
with tempfile.NamedTemporaryFile() as temp_file:
down_res_iter = src_srvclient.download(
container=container_name,
objects=[object_name],
options={'out_file': temp_file.name, 'checksum': False}
)
for down_res in down_res_iter:
if down_res['success']:
headers = ['x-static-large-object:True']
upload_iter = tgt_srvclient.upload(
container_name,
[SwiftUploadObject(temp_file.name,
object_name=object_name)],
options={'header': headers}
)
for r in upload_iter:
if not r['success']:
raise Exception(r['error'])
else:
raise Exception(down_res['error'])
def upload(self, input_filename, path_to_file, **kwargs):
# fp = open(filename, "rb")
# self.con.put_object(self.container, keyname, fp)
import swiftclient.service
from swiftclient.service import SwiftService
from swiftclient.service import SwiftUploadObject
swiftclient.service._default_global_options['os_auth_url'] = self.conf[
"auth_url"]
swiftclient.service._default_global_options['os_username'] = self.conf[
"access_key"]
swiftclient.service._default_global_options['os_password'] = self.conf[
"secret_key"]
swiftclient.service._default_global_options[
'os_tenant_name'] = self.conf["tenant_name"]
swiftclient.service._default_global_options[
'auth_version'] = self.conf["auth_version"]
swiftclient.service._default_global_options['insecure'] = self.conf[
"insecure"]
if 'segment_size' not in self.conf:
self.conf["segment_size"] = '1073741824'
if 'chunk_size' not in self.conf:
self.conf["chunk_size"] = '20480'
with SwiftService() as ss:
for result in ss.upload(
self.conf["s3_bucket"],
[SwiftUploadObject(path_to_file, input_filename)],
{'segment_size': self.conf["segment_size"]}):
if not str(result['success']):
self.print_upload_failure(path_to_file, result)
raise Exception('Upload Failure of File: ' + path_to_file)
def upload_file(self,
user_id,
path,
name,
file,
content_type='application/octet-stream',
content_length=-1):
container, object_name = self.build_object_name(user_id, path, name)
# prepend account and container to path
headers = {'Content-Type': content_type or 'application/octet-stream'}
# if content_length >= 0:
# headers['Content-Length'] = str(content_length)
upload_obj = SwiftUploadObject(source=LengthWrapper(
file, content_length, True),
object_name=object_name,
options={'header': headers})
log = logging.getLogger(__name__)
log.info('Tool Upload %s', upload_obj)
for res in self.swift.upload(container, [upload_obj]):
if res['action'] == 'create_container':
res = self._create_container(container)
# Getting a funny response iterator here
# 1. action: create_container
# 2. action: upload_object
log.info('Tool Result %s', res)
if res.get('error', None):
# res['error'].http_status == 413:
# -> Request Entity Too Large
# res['error'].http_resonse_content == 'Upload exceeds quota'
raise res['error']
def upload_files_to_swift(self, file_paths, dest_path=None):
"""
Bulk upload of a list of files to current SWIFT object storage container under the same destination path
"""
objs = []
# file object
for file_path in file_paths:
file_name = os.path.basename(file_path)
object_name = file_name
if dest_path != None:
object_name = dest_path + "/" + file_name
obj = SwiftUploadObject(file_path, object_name=object_name)
objs.append(obj)
try:
for result in self.swift.upload(self.config["swift_container"],
objs):
if not result['success']:
error = result['error']
if result['action'] == "upload_object":
logging.error(
"Failed to upload object %s to container %s: %s" %
(self.config["swift_container"], result['object'],
error))
else:
logging.error("%s" % error)
except SwiftError:
logging.exception("error uploading file to SWIFT container")
def create_and_upload_tarball(swiftservice,
tmp_dir,
container,
tarball_name,
tarball_options='-czf',
delete_after=3600,
segment_size=1048576000,
use_slo=True,
segment_container=None,
leave_segments=False,
changed=None,
skip_identical=False,
fail_fast=True,
dir_marker=False):
"""Create a tarball containing the tmp_dir and upload it to Swift.
This method allows to upload files bigger than 5GB.
It will create 2 swift containers to store the segments and
one container to reference the manifest with the segment pointers
"""
try:
with tempfile.NamedTemporaryFile() as tmp_tarball:
tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options)
objs = [SwiftUploadObject(tmp_tarball, object_name=tarball_name)]
options = {
'meta': [],
'header': ['X-Delete-After: ' + str(delete_after)],
'segment_size': segment_size,
'use_slo': use_slo,
'segment_container': segment_container,
'leave_segments': leave_segments,
'changed': changed,
'skip_identical': skip_identical,
'fail_fast': fail_fast,
'dir_marker': dir_marker
}
for r in swiftservice.upload(container, objs, options=options):
if r['success']:
if 'object' in r:
LOG.info(r['object'])
elif 'for_object' in r:
LOG.info('%s segment %s' %
(r['for_object'], r['segment_index']))
else:
error = r['error']
if r['action'] == "create_container":
LOG.warning(
'Warning: failed to create container '
"'%s'%s", container, error)
elif r['action'] == "upload_object":
LOG.error(
"Failed to upload object %s to container %s: %s" %
(container, r['object'], error))
else:
LOG.error("%s" % error)
except SwiftError as e:
LOG.error(e.value)
def _upload_to_blobstore(self, blob_to_upload_path, blob_target_name):
log_prefix = '[SWIFT] [UPLOAD]'
segment_size = (1 << 30) # 1 GiB segment size
if self.container:
self.logger.info(
'{} Started to upload the tarball to the object storage.'.
format(log_prefix))
upload_success = True
try:
blob_to_upload_object = SwiftUploadObject(
blob_to_upload_path, object_name=blob_target_name)
options = {
'segment_size': segment_size,
'segment_container': self.CONTAINER
}
for response in self.swift.service.upload(
self.CONTAINER, [blob_to_upload_object], options):
# Swift client will try to create the container in case it is not existing - we want to
# hide the error which may occur due to missing priviledges for those operations
if response[
'action'] != 'create_container' and not response[
'success']:
upload_success = False
self.logger.error(
'{} ERROR: blob_to_upload={}, blob_target_name={}, container={}, {}, segment_size={}, action={}\n{}'
.format(log_prefix, blob_to_upload_path,
blob_target_name, self.CONTAINER,
segment_size, response['action'],
response['error']))
else:
self.logger.info(
'{} SUCCESS: blob_to_upload={}, blob_target_name={}, container={}, segment_size={}, action={}'
.format(log_prefix, blob_to_upload_path,
blob_target_name, self.CONTAINER,
segment_size, response['action']))
self.logger.info(
'{} Waiting for next swift operation to finish...'.
format(log_prefix))
except Exception as error:
upload_success = False
self.logger.error(
'{} ERROR: blob_to_upload={}, blob_target_name={}, container={}, error={}'
.format(log_prefix, blob_to_upload_path, blob_target_name,
self.CONTAINER, error))
if upload_success:
self.logger.info(
'{} SUCCESS: blob_to_upload={}, blob_target_name={}, container={}'
.format(log_prefix, blob_to_upload_path, blob_target_name,
self.CONTAINER))
return True
message = '{} ERROR: blob={} could not be uploaded to container={}.'.format(
log_prefix, blob_to_upload_path, self.CONTAINER)
self.logger.error(message)
raise Exception(message)
def upload_results(results_folder, destination_container):
SETTINGS_LOCATION = os.environ.get('SWIFT_SETTINGS')
if not SETTINGS_LOCATION:
raise RuntimeError("No OpenStack settings provided for the upload")
with open(SETTINGS_LOCATION, 'r') as sf:
settings = json.load(sf)
# generate filelist from results folder
files = [(os.path.join(dp, f)) for dp, dn, fn in os.walk(results_folder)
for f in fn]
labels = convert_filenames_to_labels(results_folder, files)
options = {
'object_uu_threads': 20,
'os_auth_token': get_keystone_token(settings)
}
# Remove password from the OpenStack settings, otherwise the SwiftService
# connection will be built ignoring the token and failing authentication
del (settings['os_password'])
options.update(**settings)
def generate_options(label):
if label in ['/info', '/info_fullres.json', '/transform.json']:
return {}
else:
return {'header': ['Content-Encoding:gzip']}
with SwiftService(
options=options) as swift, OutputManager() as out_manager:
try:
logger.info(F'Making public container {destination_container}')
swift.post(container=destination_container,
options={
'read_acl':
'.r:*,.rlistings',
'header':
['X-Container-Meta-Access-Control-Allow-Origin: *']
})
logger.info(
f'Uploading {len(files)} objects to {destination_container}')
start = perf_counter()
objects = [
SwiftUploadObject(file,
object_name=labels[index],
options=generate_options(labels[index]))
for index, file in enumerate(files)
]
for result in swift.upload(destination_container, objects):
if not result['success']:
logger.error(f"Failed to upload object")
raise RuntimeError("Failed to upload object")
finish = perf_counter()
logger.info(f'Completed in {timedelta(seconds=finish-start)}')
except SwiftError as e:
logger.exception(e.value)
raise RuntimeError("Failed to upload objects")
def upload(self, container, source, object_name):
obj = SwiftUploadObject(source, object_name)
with SwiftService(options=self.config) as swift:
try:
for result in swift.upload(container=container, objects={obj}):
if not result["success"]:
raise result["error"]
except SwiftError as e:
self.logger.error(e.value)
def put(self):
"""Put an object"""
try:
upload_object = SwiftUploadObject(self.name, object_name=self.name)
response = self._object_store._service.upload(
self.container.name, [upload_object])
for item in response:
continue
except SwiftError as e:
print(e)
raise (e)
def make_upload_object(stamp_dict):
"""Given a stamp dict that follows the cutout schema,
create a corresponding SwiftUploadObject.
"""
try:
obj = SwiftUploadObject(
source=BytesIO(stamp_dict['stampData']),
object_name=stamp_dict['fileName'])
except TypeError:
print('%% Cannot get stamp\n')
return obj
def push_file(self, local_path, remote_path):
(local_dir, basename) = os.path.split(local_path)
obj = SwiftUploadObject(local_path, object_name=remote_path)
results = self._client.upload(self._container, [obj])
has_results = False
for r in results:
has_results = True
if not r["success"]:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, r["error"]))
if not has_results:
raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, "NO RESULTS"))
def upload(self, container: str, local_path: str,
object_id: str) -> None: # noqa C901
"""Upload file to given folder (container)
Args:
container: Name of container of the objectstore to download to
local_path: Path to upload to
object_id: File to upload
Raises:
AirflowException: Upload cannot be executed
"""
filename = ntpath.basename(local_path)
with self.connection() as swift:
try:
for r in swift.upload(
container,
[
SwiftUploadObject(
local_path,
object_name=object_id,
options={
"header": [
f'content-disposition: attachment; filename="{filename}"'
] # noqa
},
)
],
):
if r["success"]:
if "object" in r:
self.log.info(f"uploaded: {r['object']}")
elif "for_object" in r:
self.log.info(
f"{r['for_object']} segment {r['segment_index']}"
)
else:
error = r["error"]
if r["action"] == "create_container":
self.log.warning(
"Warning: failed to create container "
"'%s'%s", container, error)
elif r["action"] == "upload_object":
self.log.error(
"Failed to upload object %s to container %s: %s"
% (container, r["object"], error))
else:
self.log.error("%s" % error)
raise AirflowException(
f"Failed to upload file: {r['object']} ({error})")
except SwiftError as e:
self.log.error(e.value)
raise AirflowException("Failed to upload file") from e
def swift_create_default_retention_policy(swift, info):
container = info["swift"]["container"]
objects = [
SwiftUploadObject(
io.BytesIO(json.dumps(DEFAULT_RETENTION_POLICY).encode()),
object_name="retention_policy.json",
)
]
for res in swift.upload(container, objects):
if not res["success"] and res["action"] != "create_container":
raise ValueError(res)
return retention_from_conf(DEFAULT_RETENTION_POLICY)
def migrate_DLO(src_container_name, tgt_container_name, object_name, src_head,
src_srvclient, tgt_srvclient):
"""Migrate dynamic large object."""
headers = ['x-object-manifest:%s' % src_head['x-object-manifest']]
headers.append('%s:%s' % (OLD_TIMESTAMP_HEADER, src_head['x-timestamp']))
upload_iter = tgt_srvclient.upload(
tgt_container_name, [SwiftUploadObject(None, object_name=object_name)],
options={'header': headers})
for r in upload_iter:
if not r['success']:
raise Exception(r['error'])
def upload_object(swift, args):
"""Upload an object to swift
:param swift: Instance of swift service
:type swift: swiftclient.service.SwiftService
:param args: Parsed arguments object from argparse
:type args: obj
"""
o_name = args.format.replace('__DATE__', args.date)
obj = SwiftUploadObject(args.object, object_name=o_name)
resp = swift.upload(args.container, [obj])
for r in resp:
check_swift_error(r)
def main(dp_env, github_url, schema_base_url, use_local):
# We extract the zip, because otherwise we need a big set
# of open file handles during upload, now we can use file-paths
with TemporaryDirectory() as temp_dir:
if use_local:
repo_root = fetch_repo_root()
files_root = repo_root.parent
publishable_paths = fetch_local_as_publishable(repo_root)
else:
files_root = Path(temp_dir)
publishable_paths = extract_and_fetch_paths(github_url, temp_dir)
if schema_base_url is not None:
replace_schema_base_url(temp_dir, schema_base_url)
index_file_obj = get_index_file_obj(publishable_paths)
with SwiftService() as swift:
try:
uploads = [
SwiftUploadObject(index_file_obj,
object_name="datasets/index.json")
]
for path_parts in publishable_paths:
object_name = create_object_name(path_parts)
uploads.append(
SwiftUploadObject(
str(files_root / "/".join(path_parts)),
object_name=object_name,
options={
"header": ["content-type:application/json"]
},
))
for r in swift.upload(f"schemas-{dp_env}", uploads):
if not r["success"]:
logger.error(r["error"])
except SwiftError as e:
logger.exception(e.value)
def put(self, userId, bucket, key, data):
try:
uri = self.uri_for(userId, bucket, key)
swift_bucket, swift_key = self._parse_uri(uri)
obj = SwiftUploadObject(object_name=swift_key, source=io.BytesIO(data))
resp = self.client.upload(container=swift_bucket, objects=[obj])
for upload in resp:
if upload["action"] == "upload_object" and upload["success"]:
return uri
else:
raise Exception("Failed uploading object to swift")
except Exception as e:
raise e
def put(self, userId, bucket, key, data):
try:
uri = self.uri_for(userId, bucket, key)
swift_bucket, swift_key = self._parse_uri(uri)
obj = SwiftUploadObject(object_name=swift_key,
source=StringIO(data))
resp = self.client.upload(container=swift_bucket, objects=[obj])
for upload in resp:
if upload['action'] == 'upload_object' and upload['success']:
return uri
else:
raise Exception('Failed uploading object to swift')
except Exception as e:
raise e
def dump(dump_location, dbconfig_path):
"""Dump Mysql and MongoDB databases relative to the given edX instance"""
info = json.load(click.open_file(dbconfig_path))
now = datetime.datetime.utcnow().isoformat()
output_dir = os.path.join(dump_location, f"{now}")
click.echo(f"Creating dumps in {output_dir}")
os.mkdir(output_dir)
click.echo("Dumping mongodb")
output_path = os.path.join(output_dir, "mongodb_dump.gz")
mongo_host = info["mongo"]["host"]
mongo_port = info["mongo"]["port"]
cmd = f"mongodump -h {mongo_host}:{mongo_port} " f"--gzip --archive={output_path}"
print(f"Running:\n{cmd}")
if os.system(cmd) != 0:
click.echo("Error dumping mongo")
click.echo("Dumping mysql")
output_path = os.path.join(output_dir, "mysql_dump")
for mysql_info in info["mysql"]:
cmd = f"mydumper --compress {mysql_options(mysql_info)} -o {output_path}"
print(f"Running:\n{cmd.replace(mysql_info['password'], '')}")
if os.system(cmd) != 0:
click.echo(f'Error dumping mysql db {mysql_info.get("dbname")}')
if "swift" in info:
to_upload = []
for filepath in iglob(f"{output_dir}/**", recursive=True):
if not os.path.isfile(filepath):
continue
to_upload.append(
SwiftUploadObject(
filepath,
object_name=f"{'/'.join(filepath.split('/')[2:])}"))
if "container" not in info["swift"]:
click.echo("No container specified. Aborting")
click.get_current_context().fail()
container = info["swift"]["container"]
print(f"Uploading via SWIFT to container {container}")
with getSwiftService(info) as swift:
# Consume the return value of swift.upload
problems = [
el for el in swift.upload(container, to_upload)
if not el["success"] and el["action"] != "create_container"
]
if problems:
print("There were problems uploading the dump via swift")
print(problems)
sys.exit(-1)
def create_folder(self, user_id, path='', description=None):
container, object_path = self.build_object_name(user_id, path)
# create upload object
object_path = SwiftUploadObject(None,
object_name=object_path,
options={
'dir_marker': True,
'meta': {
'description': description
or '',
},
})
folders = []
for res in self.swift.upload(container, [object_path]):
if not res['success']:
raise res['error']
if res['action'] == 'create_container':
# if res['response_dict']['reason'] == 'Created'
# status will be 202 if container already existed
if res['response_dict']['status'] == 201:
# set up metadata for user container
res = self._create_container(container)
# TODO: project only:
if res['action'] == 'create_dir_marker':
meta = {}
if description:
meta['description'] = description
folder = SwiftPostObject(object_name=res['object'],
options={
'header': res['headers'],
'meta': meta,
})
folders.append(folder)
# TODO: check whether we should use post above instead of upload
# maybe we can avoid calling swift twice?
# also woke sure container get's created in case of post
ret = []
for res in self.swift.post(container, folders):
if not res['success']:
raise res['error']
ret.append(res)
return ret