def _do_write(self, path, content): # check parent directory exists self.checkParentDirExists(path) type = self.guess_type(path) things = [] if type == "directory": self.log.debug("SwiftFS._do_write create directory") things.append(SwiftUploadObject(None, object_name=path)) else: self.log.debug("SwiftFS._do_write create file/notebook from '%s'", content) output = io.BytesIO(content.encode('utf-8')) things.append(SwiftUploadObject(output, object_name=path)) # Now do the upload path = self.clean_path(path) try: response = self.swift.upload(self.container, things) except SwiftError as e: self.log.error("SwiftFS._do_write swift-error: %s", e.value) raise except ClientException as e: self.log.error("SwiftFS._do_write client-error: %s", e.value) raise for r in response: self.log.debug("SwiftFS._do_write action: '%s', response: '%s'", r['action'], r['success'])
def mock_import_analysis(self, name='', description='', details='', filepath='', params='', inputs='', outputs=''): filename = filepath created = 'mm-dd-yyyy 00:00:00.000' user = '******' sessionQuery = self.session.query(Analysis).from_statement(text("SELECT * FROM analysis where name=:name")).\ params(name=name).first().side_effect = mod_se checkMod = sessionQuery(name=name) if(checkMod is None): analysisMod = Mock(spec=Analysis(name=name, filepath=filename, description=description, details=details, created=created, user=user, parameters=params, inputs=inputs, outputs=outputs)) options = {'os_auth_url': self.config['SWIFT_AUTH_URL'], 'os_username': self.config['SWIFT_USERNAME'], 'os_password': self.config['SWIFT_PASSWORD'], 'os_tenant_id': self.config['SWIFT_TENANT_ID'], 'os_tenant_name': self.config['SWIFT_TENANT_NAME']} swiftService = Mock(spec=SwiftService(options=options)) objects = [] objects.append(SwiftUploadObject(self.config['DB_LOCATION'], object_name='sqlite.db')) objects.append(SwiftUploadObject(filepath, object_name=filename)) swiftUpload = swiftService.upload(container='containerModules', objects=objects).return_value = ('Metadata', 'Module') for uploaded in swiftUpload: print(uploaded) self.session.add(analysisMod) self.session.commit() return "import_success" else: raise RuntimeError("Analysis " + name + " already exists")
def handle(self, *args, **options): _opts = {'object_uu_threads': 20} dir = 'media' # settings.MEDIA_ROOT container = settings.SWIFT_CONTAINER_NAME with SwiftService( options=_opts) as swift, OutputManager() as out_manager: try: # Collect all the files and folders in the given directory objs = [] dir_markers = [] # change to directory so it isn't uploaded as part of all the # file object names os.chdir(dir) for (_dir, _ds, _fs) in os.walk('.'): if not (_ds + _fs): dir_markers.append(_dir) else: objs.extend([os.path.join(_dir, _f) for _f in _fs]) # Now that we've collected all the required files and dir markers # build the ``SwiftUploadObject``s for the call to upload objs = [ SwiftUploadObject(o, object_name=o.replace(dir, dir, 1)) for o in objs ] dir_markers = [ SwiftUploadObject(None, object_name=d.replace(dir, dir, 1), options={'dir_marker': True}) for d in dir_markers ] # Schedule uploads on the SwiftService thread pool and iterate # over the results for r in swift.upload(container, objs + dir_markers): if r['success']: if 'object' in r: print(r['object']) elif 'for_object' in r: print('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": logger.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": logger.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: logger.error("%s" % error) os.chdir('..') except SwiftError as e: logger.error(e.value)
def main(dp_env: str, container_prefix: str, schema_base_url: str) -> None: ROOT_PKG_NAME = "amsterdam_schema" with TemporaryDirectory() as temp_dir: files_root = Path(temp_dir) # First fetch the datasets and schemas schema_pub_paths = fetch_local_as_publishable(ROOT_PKG_NAME, files_root) if schema_base_url is not None: replace_schema_base_url(files_root, schema_base_url) index_file_obj = get_index_file_obj(schema_pub_paths, files_root) # Then fetch the documentation doc_pub_paths = fetch_local_as_publishable( ROOT_PKG_NAME, files_root, ("docs",), "**/*.html" ) with SwiftService() as swift: container = f"{container_prefix}{dp_env}" # Delete old objects in datasets deletes = swift.delete(container, options={"prefix": "datasets"}) for r in deletes: if not r["success"]: logger.warn( f"Warning: Remote object {container}/{r['object']} could not be removed." ) # Add new objects upload_objects = [SwiftUploadObject(index_file_obj, object_name="datasets/index.json")] for schema_path_parts in schema_pub_paths: object_name = create_object_name(schema_path_parts) upload_objects.append( SwiftUploadObject( str(files_root / "/".join(schema_path_parts)), object_name=object_name, options={"header": ["content-type:application/json"]}, ) ) for doc_path_parts in doc_pub_paths: object_name = "/".join(doc_path_parts[1:]) upload_objects.append( SwiftUploadObject( str(files_root / "/".join(doc_path_parts)), object_name=object_name, options={"header": ["content-type:text/html"]}, ) ) uploads = swift.upload(container, upload_objects) errors = False for r in uploads: if not r["success"]: errors = True logger.error(r["error"]) if errors: raise Exception("Failed to publish schemas")
def saveObjsBackend(objs, backend, config): if(backend == 'hdfs'): for obj in objs: try: # obj[0] is hdfs path and obj[1] is local filesystem path if '.py' in obj[1]: # If you are uploading a module do not allow it to be overwritten subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', obj[1], obj[0]]) else: # If it is the metadata then it has to be overwritten everytime subprocess.check_call(['hdfs', 'dfs', '-copyFromLocal', '-f', obj[1], obj[0]]) except Exception as e: shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH']) raise RuntimeError(e) elif(backend == 'swift'): options = {'os_auth_url': os.environ['OS_AUTH_URL'], 'os_username': os.environ['OS_USERNAME'], 'os_password': os.environ['OS_PASSWORD'], 'os_tenant_id': os.environ['OS_TENANT_ID'], 'os_tenant_name': os .environ['OS_TENANT_NAME']} swiftService = SwiftService(options=options) objects = [] for obj in objs: objects.append(SwiftUploadObject(obj[1], object_name=obj[0])) swiftUpload = swiftService.upload(container='containerModules', objects=objects) for uploaded in swiftUpload: if("error" in uploaded.keys()): shutil.copyfile(config['BACKUP_METADATA_LOCAL_PATH'], config['METADATA_LOCAL_PATH']) raise RuntimeError(uploaded['error']) elif(backend == 'nfs'): for obj in objs: shutil.copyfile(obj[1], config['MODULES_DIR'] + obj[0]) print('Metadata/Module changed and uploaded')
def upload(bucket, name, from_folder, ttl=None, segment_size="400M", buffer_size=0): global upload_buffer from_path = os.path.join(from_folder, name) headers = [] if ttl: delete_after_header = 'x-delete-after:%i' % ttl headers.append(delete_after_header) options = { 'header': headers, 'segment_size': __normalized_segment_size(segment_size) } upload_object = SwiftUploadObject(from_path, object_name=name, options=options) if bucket not in upload_buffer: upload_buffer[bucket] = [] upload_buffer[bucket].append(upload_object) if len(upload_buffer[bucket]) <= buffer_size: return flush_upload_buffer(bucket)
def swift_file_commit(_opts, container, obj_name, obj_source=None): with SwiftService(options=_opts) as swift, OutputManager() as out_manager: try: if obj_source is None: obj_source = obj_name # file path #check if file exists if (not os.path.exists(obj_source)): print('File %s does not exist' % obj_source) exit() objs = [SwiftUploadObject(obj_source, obj_name) ] # (source, object name) --> (value,key) for r in swift.upload(container, objs): if r['success']: if 'object' in r: print(r['object']) elif 'for_object' in r: print('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": logger.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": logger.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: logger.error("%s" % error) except SwiftError as e: logger.error(e.value)
def uploadfiles(container, options, objs): # first argument is the container to upload file to # second argument is a dictionary of the options to supply to SwiftUploadObject # third argument is a list of the files to upload with SwiftService() as swift: try: # create the SwiftUploadObject list of objects to upload #objs = [SwiftUploadObject(obj, options=options) for obj in objs] objs = [SwiftUploadObject(obj) for obj in objs] # Schedule uploads on the SwiftService thread pool and iterate over the results #for result in swift.upload(container, objs): for result in swift.upload(container, objs): if result['success']: if 'object' in result: print("Successfully uploaded %s." % result['object']) elif 'for_object' in result: print('%s segment %s' % (result['for_object'], result['segment_index'])) else: error = result['error'] if result['action'] == "create_container": print('Warning: failed to create container ' "'%s'%s", container, error) elif result['action'] == "upload_object": print( "Failed to upload object %s to container %s: %s" % (container, result['object'], error)) else: print("%s" % error) except SwiftError as e: print("SwiftError: %s" % e)
def upload_from_file(self, path): """ Stores the contents of the file pointed by the ``path`` variable. If the file is bigger than 5 Gig, it will be broken into segments. :type path: ``str`` :param path: Absolute path to the file to be uploaded to Swift. :rtype: ``bool`` :return: ``True`` if successful, ``False`` if not. .. note:: * The size of the segments chosen (or any of the other upload options) is not under user control. * If called this method will remap the ``swiftclient.service.get_conn`` factory method to ``self._provider._connect_swift`` .. seealso:: https://github.com/gvlproject/cloudbridge/issues/35#issuecomment-297629661 # noqa """ upload_options = {} if 'segment_size' not in upload_options: if os.path.getsize(path) >= FIVE_GIG: upload_options['segment_size'] = FIVE_GIG # remap the swift service's connection factory method swiftclient.service.get_conn = self._provider._connect_swift result = True with SwiftService() as swift: upload_object = SwiftUploadObject(path, object_name=self.name) for up_res in swift.upload(self.cbcontainer.name, [upload_object, ], options=upload_options): result = result and up_res['success'] return result
def _put(self, source_path, remote_filename): lp = util.fsdecode(source_path.name) if config.mp_segment_size > 0: from swiftclient.service import SwiftUploadObject st = os.stat(lp) # only upload using Dynamic Large Object if mpvolsize is triggered if st.st_size >= config.mp_segment_size: log.Debug(u"Uploading Dynamic Large Object") mp = self.svc.upload( self.container, [ SwiftUploadObject(lp, object_name=self.prefix + util.fsdecode(remote_filename)) ], options={u'segment_size': config.mp_segment_size}) uploads = [a for a in mp if u'container' not in a[u'action']] for upload in uploads: if not upload[u'success']: raise BackendException(upload[u'traceback']) return rp = self.prefix + util.fsdecode(remote_filename) log.Debug(u"Uploading '%s' to '%s' in remote container '%s'" % (lp, rp, self.container)) self.conn.put_object(container=self.container, obj=self.prefix + util.fsdecode(remote_filename), contents=open(lp, u'rb'))
def migrate_SLO(container_name, object_name, src_head, src_srvclient, tgt_srvclient): """Migrate static large object. Note that static large object (slo) does not actually work with RGW as of 0.9.4.x, so hopefully not too many of these. We migrate them anyway. It's a little difficult to upload the SLO manifest file before we are sure all the segments are uploaded to Swift. When the PUT operation sees the ?multipart-manifest=put query parameter, it reads the request body and verifies that each segment object exists and that the sizes and ETags match. If there is a mismatch, the PUT operation fails. """ with tempfile.NamedTemporaryFile() as temp_file: down_res_iter = src_srvclient.download( container=container_name, objects=[object_name], options={'out_file': temp_file.name, 'checksum': False} ) for down_res in down_res_iter: if down_res['success']: headers = ['x-static-large-object:True'] upload_iter = tgt_srvclient.upload( container_name, [SwiftUploadObject(temp_file.name, object_name=object_name)], options={'header': headers} ) for r in upload_iter: if not r['success']: raise Exception(r['error']) else: raise Exception(down_res['error'])
def upload(self, input_filename, path_to_file, **kwargs): # fp = open(filename, "rb") # self.con.put_object(self.container, keyname, fp) import swiftclient.service from swiftclient.service import SwiftService from swiftclient.service import SwiftUploadObject swiftclient.service._default_global_options['os_auth_url'] = self.conf[ "auth_url"] swiftclient.service._default_global_options['os_username'] = self.conf[ "access_key"] swiftclient.service._default_global_options['os_password'] = self.conf[ "secret_key"] swiftclient.service._default_global_options[ 'os_tenant_name'] = self.conf["tenant_name"] swiftclient.service._default_global_options[ 'auth_version'] = self.conf["auth_version"] swiftclient.service._default_global_options['insecure'] = self.conf[ "insecure"] if 'segment_size' not in self.conf: self.conf["segment_size"] = '1073741824' if 'chunk_size' not in self.conf: self.conf["chunk_size"] = '20480' with SwiftService() as ss: for result in ss.upload( self.conf["s3_bucket"], [SwiftUploadObject(path_to_file, input_filename)], {'segment_size': self.conf["segment_size"]}): if not str(result['success']): self.print_upload_failure(path_to_file, result) raise Exception('Upload Failure of File: ' + path_to_file)
def upload_file(self, user_id, path, name, file, content_type='application/octet-stream', content_length=-1): container, object_name = self.build_object_name(user_id, path, name) # prepend account and container to path headers = {'Content-Type': content_type or 'application/octet-stream'} # if content_length >= 0: # headers['Content-Length'] = str(content_length) upload_obj = SwiftUploadObject(source=LengthWrapper( file, content_length, True), object_name=object_name, options={'header': headers}) log = logging.getLogger(__name__) log.info('Tool Upload %s', upload_obj) for res in self.swift.upload(container, [upload_obj]): if res['action'] == 'create_container': res = self._create_container(container) # Getting a funny response iterator here # 1. action: create_container # 2. action: upload_object log.info('Tool Result %s', res) if res.get('error', None): # res['error'].http_status == 413: # -> Request Entity Too Large # res['error'].http_resonse_content == 'Upload exceeds quota' raise res['error']
def upload_files_to_swift(self, file_paths, dest_path=None): """ Bulk upload of a list of files to current SWIFT object storage container under the same destination path """ objs = [] # file object for file_path in file_paths: file_name = os.path.basename(file_path) object_name = file_name if dest_path != None: object_name = dest_path + "/" + file_name obj = SwiftUploadObject(file_path, object_name=object_name) objs.append(obj) try: for result in self.swift.upload(self.config["swift_container"], objs): if not result['success']: error = result['error'] if result['action'] == "upload_object": logging.error( "Failed to upload object %s to container %s: %s" % (self.config["swift_container"], result['object'], error)) else: logging.error("%s" % error) except SwiftError: logging.exception("error uploading file to SWIFT container")
def create_and_upload_tarball(swiftservice, tmp_dir, container, tarball_name, tarball_options='-czf', delete_after=3600, segment_size=1048576000, use_slo=True, segment_container=None, leave_segments=False, changed=None, skip_identical=False, fail_fast=True, dir_marker=False): """Create a tarball containing the tmp_dir and upload it to Swift. This method allows to upload files bigger than 5GB. It will create 2 swift containers to store the segments and one container to reference the manifest with the segment pointers """ try: with tempfile.NamedTemporaryFile() as tmp_tarball: tarball.create_tarball(tmp_dir, tmp_tarball.name, tarball_options) objs = [SwiftUploadObject(tmp_tarball, object_name=tarball_name)] options = { 'meta': [], 'header': ['X-Delete-After: ' + str(delete_after)], 'segment_size': segment_size, 'use_slo': use_slo, 'segment_container': segment_container, 'leave_segments': leave_segments, 'changed': changed, 'skip_identical': skip_identical, 'fail_fast': fail_fast, 'dir_marker': dir_marker } for r in swiftservice.upload(container, objs, options=options): if r['success']: if 'object' in r: LOG.info(r['object']) elif 'for_object' in r: LOG.info('%s segment %s' % (r['for_object'], r['segment_index'])) else: error = r['error'] if r['action'] == "create_container": LOG.warning( 'Warning: failed to create container ' "'%s'%s", container, error) elif r['action'] == "upload_object": LOG.error( "Failed to upload object %s to container %s: %s" % (container, r['object'], error)) else: LOG.error("%s" % error) except SwiftError as e: LOG.error(e.value)
def _upload_to_blobstore(self, blob_to_upload_path, blob_target_name): log_prefix = '[SWIFT] [UPLOAD]' segment_size = (1 << 30) # 1 GiB segment size if self.container: self.logger.info( '{} Started to upload the tarball to the object storage.'. format(log_prefix)) upload_success = True try: blob_to_upload_object = SwiftUploadObject( blob_to_upload_path, object_name=blob_target_name) options = { 'segment_size': segment_size, 'segment_container': self.CONTAINER } for response in self.swift.service.upload( self.CONTAINER, [blob_to_upload_object], options): # Swift client will try to create the container in case it is not existing - we want to # hide the error which may occur due to missing priviledges for those operations if response[ 'action'] != 'create_container' and not response[ 'success']: upload_success = False self.logger.error( '{} ERROR: blob_to_upload={}, blob_target_name={}, container={}, {}, segment_size={}, action={}\n{}' .format(log_prefix, blob_to_upload_path, blob_target_name, self.CONTAINER, segment_size, response['action'], response['error'])) else: self.logger.info( '{} SUCCESS: blob_to_upload={}, blob_target_name={}, container={}, segment_size={}, action={}' .format(log_prefix, blob_to_upload_path, blob_target_name, self.CONTAINER, segment_size, response['action'])) self.logger.info( '{} Waiting for next swift operation to finish...'. format(log_prefix)) except Exception as error: upload_success = False self.logger.error( '{} ERROR: blob_to_upload={}, blob_target_name={}, container={}, error={}' .format(log_prefix, blob_to_upload_path, blob_target_name, self.CONTAINER, error)) if upload_success: self.logger.info( '{} SUCCESS: blob_to_upload={}, blob_target_name={}, container={}' .format(log_prefix, blob_to_upload_path, blob_target_name, self.CONTAINER)) return True message = '{} ERROR: blob={} could not be uploaded to container={}.'.format( log_prefix, blob_to_upload_path, self.CONTAINER) self.logger.error(message) raise Exception(message)
def upload_results(results_folder, destination_container): SETTINGS_LOCATION = os.environ.get('SWIFT_SETTINGS') if not SETTINGS_LOCATION: raise RuntimeError("No OpenStack settings provided for the upload") with open(SETTINGS_LOCATION, 'r') as sf: settings = json.load(sf) # generate filelist from results folder files = [(os.path.join(dp, f)) for dp, dn, fn in os.walk(results_folder) for f in fn] labels = convert_filenames_to_labels(results_folder, files) options = { 'object_uu_threads': 20, 'os_auth_token': get_keystone_token(settings) } # Remove password from the OpenStack settings, otherwise the SwiftService # connection will be built ignoring the token and failing authentication del (settings['os_password']) options.update(**settings) def generate_options(label): if label in ['/info', '/info_fullres.json', '/transform.json']: return {} else: return {'header': ['Content-Encoding:gzip']} with SwiftService( options=options) as swift, OutputManager() as out_manager: try: logger.info(F'Making public container {destination_container}') swift.post(container=destination_container, options={ 'read_acl': '.r:*,.rlistings', 'header': ['X-Container-Meta-Access-Control-Allow-Origin: *'] }) logger.info( f'Uploading {len(files)} objects to {destination_container}') start = perf_counter() objects = [ SwiftUploadObject(file, object_name=labels[index], options=generate_options(labels[index])) for index, file in enumerate(files) ] for result in swift.upload(destination_container, objects): if not result['success']: logger.error(f"Failed to upload object") raise RuntimeError("Failed to upload object") finish = perf_counter() logger.info(f'Completed in {timedelta(seconds=finish-start)}') except SwiftError as e: logger.exception(e.value) raise RuntimeError("Failed to upload objects")
def upload(self, container, source, object_name): obj = SwiftUploadObject(source, object_name) with SwiftService(options=self.config) as swift: try: for result in swift.upload(container=container, objects={obj}): if not result["success"]: raise result["error"] except SwiftError as e: self.logger.error(e.value)
def put(self): """Put an object""" try: upload_object = SwiftUploadObject(self.name, object_name=self.name) response = self._object_store._service.upload( self.container.name, [upload_object]) for item in response: continue except SwiftError as e: print(e) raise (e)
def make_upload_object(stamp_dict): """Given a stamp dict that follows the cutout schema, create a corresponding SwiftUploadObject. """ try: obj = SwiftUploadObject( source=BytesIO(stamp_dict['stampData']), object_name=stamp_dict['fileName']) except TypeError: print('%% Cannot get stamp\n') return obj
def push_file(self, local_path, remote_path): (local_dir, basename) = os.path.split(local_path) obj = SwiftUploadObject(local_path, object_name=remote_path) results = self._client.upload(self._container, [obj]) has_results = False for r in results: has_results = True if not r["success"]: raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, r["error"])) if not has_results: raise RuntimeError("Cannot push file [%s]>[%s]: %s" % (local_path, remote_path, "NO RESULTS"))
def upload(self, container: str, local_path: str, object_id: str) -> None: # noqa C901 """Upload file to given folder (container) Args: container: Name of container of the objectstore to download to local_path: Path to upload to object_id: File to upload Raises: AirflowException: Upload cannot be executed """ filename = ntpath.basename(local_path) with self.connection() as swift: try: for r in swift.upload( container, [ SwiftUploadObject( local_path, object_name=object_id, options={ "header": [ f'content-disposition: attachment; filename="{filename}"' ] # noqa }, ) ], ): if r["success"]: if "object" in r: self.log.info(f"uploaded: {r['object']}") elif "for_object" in r: self.log.info( f"{r['for_object']} segment {r['segment_index']}" ) else: error = r["error"] if r["action"] == "create_container": self.log.warning( "Warning: failed to create container " "'%s'%s", container, error) elif r["action"] == "upload_object": self.log.error( "Failed to upload object %s to container %s: %s" % (container, r["object"], error)) else: self.log.error("%s" % error) raise AirflowException( f"Failed to upload file: {r['object']} ({error})") except SwiftError as e: self.log.error(e.value) raise AirflowException("Failed to upload file") from e
def swift_create_default_retention_policy(swift, info): container = info["swift"]["container"] objects = [ SwiftUploadObject( io.BytesIO(json.dumps(DEFAULT_RETENTION_POLICY).encode()), object_name="retention_policy.json", ) ] for res in swift.upload(container, objects): if not res["success"] and res["action"] != "create_container": raise ValueError(res) return retention_from_conf(DEFAULT_RETENTION_POLICY)
def migrate_DLO(src_container_name, tgt_container_name, object_name, src_head, src_srvclient, tgt_srvclient): """Migrate dynamic large object.""" headers = ['x-object-manifest:%s' % src_head['x-object-manifest']] headers.append('%s:%s' % (OLD_TIMESTAMP_HEADER, src_head['x-timestamp'])) upload_iter = tgt_srvclient.upload( tgt_container_name, [SwiftUploadObject(None, object_name=object_name)], options={'header': headers}) for r in upload_iter: if not r['success']: raise Exception(r['error'])
def upload_object(swift, args): """Upload an object to swift :param swift: Instance of swift service :type swift: swiftclient.service.SwiftService :param args: Parsed arguments object from argparse :type args: obj """ o_name = args.format.replace('__DATE__', args.date) obj = SwiftUploadObject(args.object, object_name=o_name) resp = swift.upload(args.container, [obj]) for r in resp: check_swift_error(r)
def main(dp_env, github_url, schema_base_url, use_local): # We extract the zip, because otherwise we need a big set # of open file handles during upload, now we can use file-paths with TemporaryDirectory() as temp_dir: if use_local: repo_root = fetch_repo_root() files_root = repo_root.parent publishable_paths = fetch_local_as_publishable(repo_root) else: files_root = Path(temp_dir) publishable_paths = extract_and_fetch_paths(github_url, temp_dir) if schema_base_url is not None: replace_schema_base_url(temp_dir, schema_base_url) index_file_obj = get_index_file_obj(publishable_paths) with SwiftService() as swift: try: uploads = [ SwiftUploadObject(index_file_obj, object_name="datasets/index.json") ] for path_parts in publishable_paths: object_name = create_object_name(path_parts) uploads.append( SwiftUploadObject( str(files_root / "/".join(path_parts)), object_name=object_name, options={ "header": ["content-type:application/json"] }, )) for r in swift.upload(f"schemas-{dp_env}", uploads): if not r["success"]: logger.error(r["error"]) except SwiftError as e: logger.exception(e.value)
def put(self, userId, bucket, key, data): try: uri = self.uri_for(userId, bucket, key) swift_bucket, swift_key = self._parse_uri(uri) obj = SwiftUploadObject(object_name=swift_key, source=io.BytesIO(data)) resp = self.client.upload(container=swift_bucket, objects=[obj]) for upload in resp: if upload["action"] == "upload_object" and upload["success"]: return uri else: raise Exception("Failed uploading object to swift") except Exception as e: raise e
def put(self, userId, bucket, key, data): try: uri = self.uri_for(userId, bucket, key) swift_bucket, swift_key = self._parse_uri(uri) obj = SwiftUploadObject(object_name=swift_key, source=StringIO(data)) resp = self.client.upload(container=swift_bucket, objects=[obj]) for upload in resp: if upload['action'] == 'upload_object' and upload['success']: return uri else: raise Exception('Failed uploading object to swift') except Exception as e: raise e
def dump(dump_location, dbconfig_path): """Dump Mysql and MongoDB databases relative to the given edX instance""" info = json.load(click.open_file(dbconfig_path)) now = datetime.datetime.utcnow().isoformat() output_dir = os.path.join(dump_location, f"{now}") click.echo(f"Creating dumps in {output_dir}") os.mkdir(output_dir) click.echo("Dumping mongodb") output_path = os.path.join(output_dir, "mongodb_dump.gz") mongo_host = info["mongo"]["host"] mongo_port = info["mongo"]["port"] cmd = f"mongodump -h {mongo_host}:{mongo_port} " f"--gzip --archive={output_path}" print(f"Running:\n{cmd}") if os.system(cmd) != 0: click.echo("Error dumping mongo") click.echo("Dumping mysql") output_path = os.path.join(output_dir, "mysql_dump") for mysql_info in info["mysql"]: cmd = f"mydumper --compress {mysql_options(mysql_info)} -o {output_path}" print(f"Running:\n{cmd.replace(mysql_info['password'], '')}") if os.system(cmd) != 0: click.echo(f'Error dumping mysql db {mysql_info.get("dbname")}') if "swift" in info: to_upload = [] for filepath in iglob(f"{output_dir}/**", recursive=True): if not os.path.isfile(filepath): continue to_upload.append( SwiftUploadObject( filepath, object_name=f"{'/'.join(filepath.split('/')[2:])}")) if "container" not in info["swift"]: click.echo("No container specified. Aborting") click.get_current_context().fail() container = info["swift"]["container"] print(f"Uploading via SWIFT to container {container}") with getSwiftService(info) as swift: # Consume the return value of swift.upload problems = [ el for el in swift.upload(container, to_upload) if not el["success"] and el["action"] != "create_container" ] if problems: print("There were problems uploading the dump via swift") print(problems) sys.exit(-1)
def create_folder(self, user_id, path='', description=None): container, object_path = self.build_object_name(user_id, path) # create upload object object_path = SwiftUploadObject(None, object_name=object_path, options={ 'dir_marker': True, 'meta': { 'description': description or '', }, }) folders = [] for res in self.swift.upload(container, [object_path]): if not res['success']: raise res['error'] if res['action'] == 'create_container': # if res['response_dict']['reason'] == 'Created' # status will be 202 if container already existed if res['response_dict']['status'] == 201: # set up metadata for user container res = self._create_container(container) # TODO: project only: if res['action'] == 'create_dir_marker': meta = {} if description: meta['description'] = description folder = SwiftPostObject(object_name=res['object'], options={ 'header': res['headers'], 'meta': meta, }) folders.append(folder) # TODO: check whether we should use post above instead of upload # maybe we can avoid calling swift twice? # also woke sure container get's created in case of post ret = [] for res in self.swift.post(container, folders): if not res['success']: raise res['error'] ret.append(res) return ret