def __init__(self, connection): log.debug("class: %s" % self.__class__.__name__) self.connection = connection self.add_io_info = AddIOInfo() self.test_op_code = "create"
def test_exec(config): test_info = AddTestInfo("create users") add_io_info = AddIOInfo() add_io_info.initialize() try: test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count, config.cluster_name) # dump the list of users into a file with open("user_details", "w") as fout: json.dump(all_user_details, fout) test_info.success_status("user creation completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("user creation failed: %s" % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo('create m buckets, n keys and download') try: # test case starts test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.upload(buckets) rgw.download_keys() test_info.success_status('test completed') sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec_read(config): grants = {'permission': 'READ', 'user_id': None, 'recursive': True} test_info = AddTestInfo( 'Test with read permission on buckets for all users') add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) user1 = all_user_details[0] log.info('user1: %s' % user1) all_user_details.pop(0) u1 = ObjectOps(config, user1) for each_user in all_user_details: u2 = ObjectOps(config, each_user) u2_canonical_id = u2.canonical_id log.info('canonical id of u2: %s' % u2_canonical_id) grants['user_id'] = u2_canonical_id u1.grants = None u1.create_bucket() u1.set_bucket_properties() u2.bucket_names = u1.bucket_names u2.buckets_created = u1.buckets_created u2.grants = None u2.set_bucket_properties() # set permissions and read u1.grants = grants u1.set_bucket_properties() u2.bucket_names = u1.bucket_names u2.buckets_created = u1.buckets_created u2.grants = None u2.set_bucket_properties() test_info.success_status('test completed') except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def create_admin_user(self, username, displayname, cluster_name="ceph"): try: add_io_info = AddIOInfo() cmd = ( 'radosgw-admin user create --uid="%s" --display-name="%s" --cluster %s' % (username, displayname, cluster_name)) log.info("cmd: %s" % cmd) variable = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) v = variable.stdout.read() v_as_json = json.loads(v) # log.info(v_as_json) user_details = {} user_details["user_id"] = v_as_json["user_id"] user_details["display_name"] = v_as_json["display_name"] user_details["access_key"] = v_as_json["keys"][0]["access_key"] user_details["secret_key"] = v_as_json["keys"][0]["secret_key"] add_io_info.add_user_info( **{ "user_id": user_details["user_id"], "access_key": user_details["access_key"], "secret_key": user_details["secret_key"], }) return user_details except subprocess.CalledProcessError as e: error = e.output + str(e.returncode) log.error(error) return False
def test_exec(config): test_info = AddTestInfo("multipart Upload with cancel and download") add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) log.info("multipart upload enabled") for each_user in all_user_details: config.objects_count = 1 rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.break_upload_at_part_no = config.break_at_part_no rgw.multipart_upload(buckets) log.info("starting at part no: %s" % config.break_at_part_no) log.info("--------------------------------------------------") rgw.break_upload_at_part_no = 0 rgw.multipart_upload(buckets) rgw.download_keys() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec(config): test_info = AddTestInfo('create m buckets') add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) for each_user in all_user_details: rgw = ObjectOps(config, each_user) assert rgw.create_bucket() test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): test_info = AddTestInfo( 'enable versioning on a bucket and upload keys and its versions') add_io_info = AddIOInfo() add_io_info.initialize() try: test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) for each_user in all_user_details: rgw = ObjectOps(config, each_user) rgw.enable_versioning = True rgw.version_count = config.version_count buckets = rgw.create_bucket() rgw.set_bucket_properties() rgw.upload(buckets) test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) sys.exit(1)
def create_admin_user(self, username, displayname, cluster_name='ceph'): try: add_io_info = AddIOInfo() cmd = 'radosgw-admin user create --uid=%s --display-name=%s --cluster %s' % ( username, displayname, cluster_name) log.info('cmd: %s' % cmd) variable = subprocess.Popen(cmd, stdout=subprocess.PIPE, shell=True) v = variable.stdout.read() v_as_json = json.loads(v) # log.info(v_as_json) user_details = {} user_details['user_id'] = v_as_json['user_id'] user_details['display_name'] = v_as_json['display_name'] user_details['access_key'] = v_as_json['keys'][0]['access_key'] user_details['secret_key'] = v_as_json['keys'][0]['secret_key'] add_io_info.add_user_info( **{ 'user_id': user_details['user_id'], 'access_key': user_details['access_key'], 'secret_key': user_details['secret_key'] }) return user_details except subprocess.CalledProcessError as e: error = e.output + str(e.returncode) log.error(error) return False
def test_exec(config): test_info = AddTestInfo('multipart Upload') add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) log.info('multipart upload enabled') for each_user in all_user_details: config.objects_count = 1 rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.multipart_upload(buckets) test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec_read(config): test_info = AddTestInfo("Test with read permission on buckets") try: # test case starts add_io_info = AddIOInfo() add_io_info.initialize() test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) user1 = all_user_details[0] log.info("user1: %s" % user1) user2 = all_user_details[1] log.info("user2: %s" % user2) u1 = ObjectOps(config, user1) u2 = ObjectOps(config, user2) u1_grants = { "permission": "READ", "user_id": u2.canonical_id, "recursive": True, } u2_grants = { "permission": "FULL_CONTROL", "user_id": u1.canonical_id, "recursive": True, } u1.grants = u1_grants u1.create_bucket() u1_buckets = u1.set_bucket_properties() u1.upload(u1_buckets, object_base_name=u1.canonical_id + ".key") all_keys = u1_buckets[0].get_all_keys() for key in all_keys: log.info("all keys from user 1--------------") log.info("name: %s" % key.name) u2.grants = u2_grants u2.create_bucket() u2_buckets = u2.set_bucket_properties() bu2 = u1.connection["conn"].get_bucket(u2_buckets[0].name) log.info("copying the objects from u1 to u2") for each in all_keys: bu2.copy_key(each.key, u1_buckets[0].name, each.key) all_keys2 = bu2.get_all_keys() for key in all_keys2: log.info("all keys from user 2--------------") log.info("name: %s" % key.name) log.info("verifying copied objects--------") u2.grants = None u2_buckets = u2.set_bucket_properties() all_keys3 = u2_buckets[0].get_all_keys() for key in all_keys3: log.info("all keys from user 2--------------") log.info("name: %s" % key.name) test_info.success_status("test completed") except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec_read(config): test_info = AddTestInfo('Test with read permission on buckets') try: # test case starts add_io_info = AddIOInfo() add_io_info.initialize() test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) user1 = all_user_details[0] log.info('user1: %s' % user1) user2 = all_user_details[1] log.info('user2: %s' % user2) u1 = ObjectOps(config, user1) u2 = ObjectOps(config, user2) u1_grants = { 'permission': 'READ', 'user_id': u2.canonical_id, 'recursive': True } u2_grants = { 'permission': 'FULL_CONTROL', 'user_id': u1.canonical_id, 'recursive': True } u1.grants = u1_grants u1.create_bucket() u1_buckets = u1.set_bucket_properties() u1.upload(u1_buckets, object_base_name=u1.canonical_id + '.key') all_keys = u1_buckets[0].get_all_keys() for key in all_keys: log.info('all keys from user 1--------------') log.info('name: %s' % key.name) u2.grants = u2_grants u2.create_bucket() u2_buckets = u2.set_bucket_properties() bu2 = u1.connection['conn'].get_bucket(u2_buckets[0].name) log.info('copying the objects from u1 to u2') for each in all_keys: bu2.copy_key(each.key, u1_buckets[0].name, each.key) all_keys2 = bu2.get_all_keys() for key in all_keys2: log.info('all keys from user 2--------------') log.info('name: %s' % key.name) log.info('verifying copied objects--------') u2.grants = None u2_buckets = u2.set_bucket_properties() all_keys3 = u2_buckets[0].get_all_keys() for key in all_keys3: log.info('all keys from user 2--------------') log.info('name: %s' % key.name) test_info.success_status('test completed') except AssertionError as e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def __init__(self, bucket, filename): log.debug('class: %s' % self.__class__.__name__) self.bucket = bucket self.split_files_list = [] self.filename = filename self.json_ops = None self.cancel_multpart = False self.mp = None self.md5 = None self.break_at_part_no = 0 self.add_io_info = AddIOInfo()
def test_exec(config): test_info = AddTestInfo("create m buckets, n objects and delete") add_io_info = AddIOInfo() add_io_info.initialize() try: test_info.started_info() rgw_service = RGWService() quota_mgmt = QuotaMgmt() test_config = AddToCephConf() if config.shards: test_config.set_to_ceph_conf( "global", ConfigOpts.rgw_override_bucket_index_max_shards, config.shards) log.info( "test to continue after service restart, sleept time 120 seconds" ) no_of_shards_for_each_bucket = int(config.shards) * int( config.bucket_count) if config.dynamic_sharding is True: test_config.set_to_ceph_conf( "global", ConfigOpts.rgw_max_objs_per_shard, config.max_objects_per_shard, ) test_config.set_to_ceph_conf("global", ConfigOpts.rgw_dynamic_resharding, True) num_shards_expected = config.objects_count / config.max_objects_per_shard log.info("num_shards_expected: %s" % num_shards_expected) log.info( "test to continue after service restart, sleept time 120 seconds" ) rgw_service.restart() time.sleep(120) all_user_details = rgw_lib.create_users(config.user_count) for each_user in all_user_details: if config.max_objects: quota_mgmt.set_bucket_quota(each_user["user_id"], config.max_objects) quota_mgmt.enable_bucket_quota(each_user["user_id"]) rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.upload(buckets) test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
class Bucket(object): def __init__(self, connection): log.debug('class: %s' % self.__class__.__name__) self.connection = connection self.add_io_info = AddIOInfo() self.test_op_code = 'create' def create(self, bucket_name, json_file): log.debug('function: %s' % self.create.__name__) log.info('in create bucket') """ :param bucket_name: string :rtype: dict :return: create_bucket_stack: args: 1.status: True, bucket created or False if bucket creation failed 2.bucket: bucket objects 3.msgs: error messages """ try: bucket = self.connection.create_bucket(bucket_name) create_bucket_stack = {'status': True, 'bucket': bucket} add_bucket_to_json = JBucket(json_file) add_bucket_to_json.add(bucket_name) self.add_io_info.add_bucket_info(self.connection.access_key, **{'bucket_name': bucket_name, 'test_op_code': self.test_op_code}) except (exception.AWSConnectionError, exception.BotoClientError, exception.S3ResponseError, exception.S3CreateError, IOError), e: log.error(e) create_bucket_stack = {'status': False, 'msgs': e} return create_bucket_stack
def __init__(self, key, json_file): log.debug('class: %s' % self.__class__.__name__) self.json_file = json_file self.jkey = JKeys(self.json_file) self.key = key self.add_io_info = AddIOInfo()
def test_exec(config): test_info = AddTestInfo("create m buckets, n keys and download") add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) for each_user in all_user_details: rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.upload(buckets) rgw.download_keys() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec_read(config): """ config.bucket_count = 3 config.objects_count = 3 config.objects_size_range = {'min': 50, 'max': 100} """ grants = {"permission": "READ", "user_id": None, "recursive": True} test_info = AddTestInfo("Test with read permission on buckets") add_io_info = AddIOInfo() add_io_info.initialize() try: # test case starts test_info.started_info() all_user_details = rgw_lib.create_users(config.user_count) user1 = all_user_details[0] log.info("user1: %s" % user1) user2 = all_user_details[1] log.info("user2: %s" % user2) u1 = ObjectOps(config, user1) u2 = ObjectOps(config, user2) u2_canonical_id = u2.canonical_id log.info("canonical id of u2: %s" % u2_canonical_id) grants["user_id"] = u2_canonical_id u1.grants = None u1.create_bucket() u1.set_bucket_properties() u2.bucket_names = u1.bucket_names u2.buckets_created = u1.buckets_created u2.grants = None u2.set_bucket_properties() # set permissions and read u1.grants = grants u1.set_bucket_properties() u2.bucket_names = u1.bucket_names u2.buckets_created = u1.buckets_created u2.grants = None u2.set_bucket_properties() test_info.success_status("test completed") except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec(config): test_info = AddTestInfo('create m buckets, n objects and delete') try: test_info.started_info() add_io_info = AddIOInfo() read_io_info = ReadIOInfo() add_io_info.initialize() rgw_service = RGWService() quota_mgmt = QuotaMgmt() test_config = AddToCephConf() if config.shards: test_config.set_to_ceph_conf( 'global', ConfigOpts.rgw_override_bucket_index_max_shards, config.shards) rgw_service.restart() no_of_shards_for_each_bucket = int(config.shards) * int( config.bucket_count) with open('user_details') as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ 'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key'] }) for each_user in all_user_details: if config.max_objects: quota_mgmt.set_bucket_quota(each_user['user_id'], config.max_objects) quota_mgmt.enable_bucket_quota(each_user['user_id']) rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.upload(buckets) #read_io_info.verify_io() test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo("multipart Upload with cancel and download") try: # test case starts test_info.started_info() with open("user_details") as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ "user_id": each_user["user_id"], "access_key": each_user["access_key"], "secret_key": each_user["secret_key"], }) log.info("multipart upload enabled") for each_user in all_user_details: config.objects_count = 2 rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.break_upload_at_part_no = config.break_at_part_no rgw.multipart_upload(buckets) log.info("starting at part no: %s" % config.break_at_part_no) log.info("--------------------------------------------------") rgw.break_upload_at_part_no = 0 rgw.multipart_upload(buckets) rgw.download_keys() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo('multipart Upload with cancel and download') try: # test case starts test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info(**{'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key']}) log.info('multipart upload enabled') for each_user in all_user_details: config.objects_count = 2 rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.break_upload_at_part_no = config.break_at_part_no rgw.multipart_upload(buckets) log.info('starting at part no: %s' % config.break_at_part_no) log.info('--------------------------------------------------') rgw.break_upload_at_part_no = 0 rgw.multipart_upload(buckets) rgw.download_keys() test_info.success_status('test completed') sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo( "enable versioning on a bucket and upload keys and its versions " "and suspend version on the same bucket" ) try: test_info.started_info() with open("user_details") as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ "user_id": each_user["user_id"], "access_key": each_user["access_key"], "secret_key": each_user["secret_key"], } ) for each_user in all_user_details: rgw = ObjectOps(config, each_user) rgw.enable_versioning = True rgw.version_count = config.version_count buckets = rgw.create_bucket() rgw.set_bucket_properties() rgw.upload(buckets) rgw.enable_versioning = False rgw.set_bucket_properties() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo("create m buckets, n objects and delete") try: # test case starts test_info.started_info() with open("user_details") as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ "user_id": each_user["user_id"], "access_key": each_user["access_key"], "secret_key": each_user["secret_key"], } ) for each_user in all_user_details: rgw = ObjectOps(config, each_user) rgw.bucket_ops.test_op_code = "delete" buckets = rgw.create_bucket() rgw.upload(buckets, test_op_code="delete") rgw.delete_keys() time.sleep(15) rgw.delete_bucket() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo('create m buckets, n objects and delete') try: # test case starts test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ 'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key'] }) for each_user in all_user_details: rgw = ObjectOps(config, each_user) rgw.bucket_ops.test_op_code = 'delete' buckets = rgw.create_bucket() rgw.upload(buckets, test_op_code='delete') rgw.delete_keys() time.sleep(15) rgw.delete_bucket() test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() read_io_info = ReadIOInfo() test_info = AddTestInfo("create m buckets") try: # test case starts test_info.started_info() with open("user_details") as fout: all_user_details = simplejson.load(fout) # write all users details to yaml file. for each_user in all_user_details: add_io_info.add_user_info( **{ "user_id": each_user["user_id"], "access_key": each_user["access_key"], "secret_key": each_user["secret_key"], }) for each_user in all_user_details: log.info("User credentials: %s" % each_user) rgw = ObjectOps(config, each_user) assert rgw.create_bucket() # read_io_info.verify_io() test_info.success_status("test completed") sys.exit(0) except AssertionError as e: log.error(e) test_info.failed_status("test failed: %s" % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() read_io_info = ReadIOInfo() test_info = AddTestInfo('create m buckets') try: # test case starts test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) # write all users details to yaml file. for each_user in all_user_details: add_io_info.add_user_info( **{ 'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key'] }) for each_user in all_user_details: log.info('User credentials: %s' % each_user) rgw = ObjectOps(config, each_user) assert rgw.create_bucket() #read_io_info.verify_io() test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() read_io = ReadIOInfo() test_info = AddTestInfo('multipart Upload') try: # test case starts test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) log.info('multipart upload enabled') for each_user in all_user_details: add_io_info.add_user_info( **{ 'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key'] }) for each_user in all_user_details: config.objects_count = 1 rgw = ObjectOps(config, each_user) buckets = rgw.create_bucket() rgw.multipart_upload(buckets) test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
def test_exec(config): add_io_info = AddIOInfo() add_io_info.initialize() test_info = AddTestInfo('enable versioning on a bucket and upload ' 'keys and its versions and delete its versions') try: test_info.started_info() with open('user_details') as fout: all_user_details = simplejson.load(fout) for each_user in all_user_details: add_io_info.add_user_info( **{ 'user_id': each_user['user_id'], 'access_key': each_user['access_key'], 'secret_key': each_user['secret_key'] }) for each_user in all_user_details: rgw = ObjectOps(config, each_user) rgw.enable_versioning = True rgw.version_count = config.version_count buckets = rgw.create_bucket() rgw.set_bucket_properties() rgw.upload(buckets) rgw.delete_key_version() test_info.success_status('test completed') sys.exit(0) except AssertionError, e: log.error(e) test_info.failed_status('test failed: %s' % e) sys.exit(1)
class MultipartPut(object): def __init__(self, bucket, filename): log.debug('class: %s' % self.__class__.__name__) self.bucket = bucket self.split_files_list = [] self.filename = filename self.json_ops = None self.cancel_multpart = False self.mp = None self.md5 = None self.break_at_part_no = 0 self.add_io_info = AddIOInfo() def iniate_multipart(self, json_file): try: self.json_ops = JMulpipart(json_file) log.info('initaiting multipart upload') file_path = os.path.dirname(self.filename) key_name = os.path.basename(self.filename) if not os.path.exists(json_file): log.info('fresh multipart upload') log.info('got filename: %s\ngot filepath: %s' % (self.filename, file_path)) utils.split_file(self.filename) self.split_files_list = sorted( glob.glob(file_path + '/' + 'x*')) # log.info('split files list: %s' % self.split_files_list) self.json_ops.total_parts_count = len(self.split_files_list) self.json_ops.bucket_name = self.bucket.name log.info('total file parts %s' % self.json_ops.total_parts_count) remaining_file_parts = [] for each_file in self.split_files_list: remaining_file_parts.append( (each_file, (self.split_files_list.index(each_file) + 1))) # log.info('remainig file parts structure :%s' % remaining_file_parts) self.json_ops.remaining_file_parts = remaining_file_parts self.mp = self.bucket.initiate_multipart_upload(key_name) self.json_ops.mp_id = self.mp.id self.json_ops.key_name = self.mp.key_name log.info('multipart_id :%s' % self.mp.id) log.info('key_name %s' % self.mp.key_name) self.json_ops.create_update_json_file() else: log.info('not fresh mulitpart') self.json_ops.refresh_json_data() self.mp = boto.s3.multipart.MultiPartUpload(self.bucket) self.mp.key_name = self.json_ops.key_name self.mp.id = self.json_ops.mp_id log.info('multipart_id :%s' % self.mp.id) log.info('key_name %s' % self.mp.key_name) except (exception.BotoClientError, exception.S3ResponseError) as e: log.error(e) return False def put(self): try: log.info('loading the json data') self.json_ops.refresh_json_data() self.json_ops.refresh_json_data() log.debug('remaining parts assigning') log.debug('making a copy of list of remaining parts') remaining_file_parts_copy = list( self.json_ops.remaining_file_parts) log.debug('starting the loop') for each_file_part in self.json_ops.remaining_file_parts: log.info('file part to upload: %s\nfile part number: %s' % (each_file_part[0], int(each_file_part[1]))) log.info('entering iteration') if self.break_at_part_no != 0 and self.break_at_part_no == int( each_file_part[1]): log.info('upload stopped at partno : %s' % each_file_part[1]) break fp = open(each_file_part[0], 'rb') self.mp.upload_part_from_file(fp, int(each_file_part[1])) fp.close() log.info('part of file uploaded') remaining_file_parts_copy.remove(each_file_part) self.json_ops.remaining_file_parts = remaining_file_parts_copy log.info('updating json file') self.json_ops.create_update_json_file() log.info('printing all the uploaded parts') for part in self.mp: log.info('%s: %s' % (part.part_number, part.size)) if self.break_at_part_no == 0: # if self.cancel_multpart: # log.info('cancelling upload') # # self.mp.cancel_upload() # # if not self.mp: # upload_status = {'status': False} # log.info('completing upload') self.mp.complete_upload() mp_object = self.bucket.get_key(self.mp.key_name) log.info('adding io info to yaml') self.add_io_info.add_keys_info( self.bucket.connection.access_key, self.bucket.name, **{ 'key_name': os.path.basename(self.filename), 'md5_on_s3': mp_object.etag.replace('"', ''), 'size': mp_object.size, 'upload_type': 'multipart', 'test_op_code': 'create' }) upload_status = {'status': True} """ # the following code is better than splitting the file, # but commenting this for now and going ahead with splting the files chunk_count = int(math.ceil(filename / float(chunk_size))) # Send the file parts, using FileChunkIO to create a file-like object # that points to a certain byte range within the original file. We # set bytes to never exceed the original file size for i in range(chunk_count): offset = chunk_size * i bytes = min(chunk_size, file_size - offset) with FileChunkIO(filename, 'r', offset=offset, bytes=bytes) as fp: mp.upload_part_from_file(fp, part_num=i + 1) # Finish the upload """ except (exception.BotoClientError, exception.S3ResponseError) as e: log.error(e) upload_status = {'status': False, 'msg': e} return upload_status
class PutContentsFromFile(object): def __init__(self, key, json_file): log.debug('class: %s' % self.__class__.__name__) self.json_file = json_file self.jkey = JKeys(self.json_file) self.key = key self.add_io_info = AddIOInfo() def set_metadata(self, **metadata): log.debug('function: %s' % self.set_metadata.__name__) log.info('setting metadata %s' % metadata) metadata_name = list(metadata.keys())[0] metadata_value = list(metadata.values())[0] try: self.key.set_metadata(metadata_name, metadata_value) return True except (exception.BotoClientError, exception.S3ResponseError) as e: log.error(e) return False def put(self, filename, test_op_code='create'): log.debug('function: %s' % self.put.__name__) log.info('upload of file: %s' % filename) """ :param filename: filename i.e along with location :return: dictionary, args: 1. status: True for successful upload or False for failed upload, 2. msgs : error messages """ try: self.key.set_contents_from_filename(filename) md5_on_s3 = self.key.etag.replace('"', '') key_details = { 'key_name': self.key.key, 'size': os.stat(filename).st_size, 'md5_local': utils.get_md5(filename), 'md5_on_s3': md5_on_s3, 'opcode': { "edit": { "new_md5": None }, "move": { "new_name": None }, "delete": { "deleted": None } } } self.jkey.add(self.key.bucket.name, **key_details) self.add_io_info.add_keys_info( self.key.bucket.connection.access_key, self.key.bucket.name, **{ 'key_name': self.key.key, 'size': os.stat(filename).st_size, 'md5_on_s3': md5_on_s3, 'upload_type': 'normal', 'test_op_code': test_op_code }) upload_status = {'status': True} except (exception.BotoClientError, exception.S3ResponseError) as e: log.error(e) upload_status = {'status': False, 'msgs': e} return upload_status def get(self, filename): log.debug('function: %s' % self.get.__name__) log.info('getting the contents of file %s:' % self.key) log.info('download or get the file to filename: %s' % filename) """ :param: filename: mention the filename which will be used to get the contents from s3 to this file. can be different from the original filename :return: dictionary, args: 1. status: True for successful download or False for failed download, 2. msgs : error messages """ try: self.key.get_contents_to_filename(filename) md5_on_s3 = self.key.etag.replace('"', '') md5_local = utils.get_md5(filename) if md5_on_s3 == md5_local: md5_match = "match" else: md5_match = "no match" key_details = { 'key_name': os.path.basename(filename), 'key_name_os_s3': self.key.name, 'size': os.stat(filename).st_size, 'md5_local': md5_local, 'md5_on_s3': md5_on_s3, 'md5_match': md5_match, 'opcode': { "edit": { "new_md5": None }, "move": { "new_name": None }, "delete": { "deleted": None } } } self.jkey.add(self.key.bucket.name, **key_details) download_status = {'status': True} except (exception.BotoClientError, exception.S3ResponseError, Exception) as e: log.error(e) download_status = {'status': False, 'msgs': e} return download_status