def _check_download_record(self, record): """ first check checkpoint file base info than check complete parts size :param record: :return: """ if not (os.path.exists(self._tmp_file) and os.path.getsize(self._tmp_file) == self.size): return False try: if not operator.eq([record['bucketName'], record['objectKey'], record['versionId'], record['downloadFile']], [self.bucket, self.objkey, self.versionid, self.filename]): logger.warning('the bucketName or objectKey or downloadFile was changed. clear the record') return False object_meta_resp = self.obscmdutil.get_object_metadata_nocheck(self.bucket, self.objkey, self.versionid) check_resp(object_meta_resp) if not operator.eq(record['objectStatus'], [self.objkey, object_meta_resp.body.contentLength, object_meta_resp.body.lastModified, self.versionid]): logger.warning('the versionId or lastModified was changed. clear the record') return False except: return False return True
def __init__(self, bucket, objkey, src_obspath, partsize, tasknum, enable_checkpoint, checkpoint_file, checksum, metadata, obsclient): super(UploadOperation, self).__init__(bucket, objkey, src_obspath, partsize, tasknum, enable_checkpoint, checkpoint_file, obsclient) self.checksum = checksum self.metadata = metadata self.cmdtype = 'copy' self.copy_source = self.filename.replace('obs://', '') srcb, srck = split_bucket_key(src_obspath) metedata_resp = self.obscmdutil.get_object_metadata_nocheck(srcb, srck) if metedata_resp.status < 300: self.lastModified = metedata_resp.body.lastModified self.size = metedata_resp.body.contentLength else: logger.warning( 'touch the objetc {0} error. ErrorCode:{1}, ErrorMessage:{2}'.format( self.objkey, metedata_resp.errorCode, metedata_resp.errorMessage)) self._delete_record() raise Exception( 'touch the objetc {0} error. ErrorCode:{1}, ErrorMessage:{2}'.format( self.objkey, metedata_resp.status, metedata_resp.errorMessage)) resp_for_check_bucket = self.obscmdutil.head_bucket_nocheck(self.bucket) check_resp(resp_for_check_bucket) self._lock = compat.Lock() self._exception = [] self._record = None
def _get_bucket_acl(self, bucket): resp = self.client.getBucketAcl(bucket) check_resp(resp) self._outprint('owner_id: %s\n' % resp.body.owner.owner_id) self._outprint('owner_name: %s\n' % resp.body.owner.owner_name) acls = {} for grant in resp.body.grants: group = grant.grantee.group if group not in acls: acls[group] = {} grantee_id = grant.grantee.grantee_id if grantee_id not in acls[group]: acls[group][grantee_id] = {} acls[group][grantee_id]['name'] = grant.grantee.grantee_name if 'premission' not in acls[group][grantee_id]: acls[group][grantee_id]['premission'] = [] acls[group][grantee_id]['premission'].append(grant.permission) for group, grantees in acls.items(): self._outprint('\n############## group: %s' % group) for grantee_id, item in grantees.items(): self._outprint('grant_id: %s\n' % grantee_id) self._outprint('grant_name: %s\n' % item['name']) self._outprint('permission: %s\n\n' % ', '.join(item['premission']))
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths bucket, key = split_bucket_key(path) self._warning_prompt('Are you sure to delete bucket website?') resp = self.client.deleteBucketWebsite(bucket) check_resp(resp) self._outprint("delete bucket website complete.")
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths policy = parsed_args.policy bucket, key = split_bucket_key(path) resp = self.client.setBucketPolicy(bucket, policy) check_resp(resp) self._outprint("create policy complete: %s\n" % path)
def _get_bucket_location(self, bucket): """ get bucket location info :param bucket: :return: """ resp = self.client.getBucketLocation(bucket) check_resp(resp) self._outprint("location: %s\n" % resp.body.location)
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths rule_str = parsed_args.lifecycle bucket, key = split_bucket_key(path) rule = self._check_lifecycle(bucket, rule_str) lifecycle = json.loads(rule) resp = self.client.setBucketLifecycle(bucket, lifecycle) check_resp(resp) self._outprint("create bucket lifecycle complete.")
def _get_bucket_storage(self, bucket): """ get bucket storage info, include size and object number :param bucket: :return: """ resp = self.client.getBucketStorageInfo(bucket) check_resp(resp) self._outprint("size: %s\n" % bytes_to_unitstr(resp.body.size)) self._outprint("objectNumber: %s\n" % resp.body.objectNumber)
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths log_str = parsed_args.log bucket, key = split_bucket_key(path) logstatus = None if log_str is not None: logstatus = self._build_log(log_str=log_str) resp = self.client.setBucketLogging(bucket, logstatus=logstatus) check_resp(resp) self._outprint("set bucket log complete.")
def _get_bucket_metadata(self, bucket): """ get bucket metadata info :param bucket: :return: """ resp = self.client.getBucketMetadata(bucket) check_resp(resp) self._outprint('obs version: %s' % resp.body.obsVersion) self._outprint("storage type: %s" % resp.body.storageClass) self._outprint('location: %s' % resp.body.location)
def _get_bucket_policy(self, bucket): try: resp = self.client.getBucketPolicy(bucket) check_resp(resp) except Exception as e: if 'NoSuchBucketPolicy' not in str(e): raise e else: self._outprint("no bucket policy.") return self._outprint("policyJSON: %s\n" % resp.body.policyJSON)
def create_bucket(client, name=None, region=None): """ Creates a bucket :returns: the name of the bucket created """ if name: bucket_name = name else: bucket_name = random_string() resp = client.createBucket(name) check_resp(resp) return bucket_name
def _get_bucket_logging(self, bucket): resp = self.client.getBucketLoggingConfiguration(bucket) check_resp(resp) self._outprint('targetBucket: %s\n' % resp.body.targetBucket) self._outprint('targetPrefix: %s\n' % resp.body.targetPrefix) index = 1 for grant in resp.body.targetGrants: self._outprint('\n############## grant [' + str(index) + ']') self._outprint('grant_id: %s\n' % grant.grantee.grantee_id) self._outprint('grant_name: %s\n' % grant.grantee.grantee_name) self._outprint('group: %s\n' % grant.grantee.group) self._outprint('permission: %s\n' % grant.permission) index += 1
def _get_bucket_tagging(self, bucket): try: resp = self.client.getBucketTagging(bucket) check_resp(resp) except Exception as e: if 'NoSuchTagSet' not in str(e): raise e else: self._outprint("no bucket tag.") return for tag in resp.body.tagSet: self._outprint('tag {key}: {value}\n'.format(key=tag.key, value=tag.value))
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths rule_id = parsed_args.id bucket, key = split_bucket_key(path) self._warning_prompt('Are you sure to delete bucket life circle?') if rule_id is None: # delete all lifecycles resp = self.client.deleteBucketLifecycle(bucket) check_resp(resp) self._outprint("delete all lifecycle complete.") else: # delete one lifecycle rule_str = self._check_lifecycle(bucket, rule_id) if rule_str is not None and rule_str != 'NOTIN': resp = self.client.deleteBucketLifecycle(bucket) check_resp(resp) lifecycle = json.loads(rule_str) resp = self.client.setBucketLifecycle(bucket, lifecycle) check_resp(resp) self._outprint("delete lifecycle " + rule_id + " complete.") elif rule_str is None: resp = self.client.deleteBucketLifecycle(bucket) check_resp(resp) self._outprint("delete lifecycle " + rule_id + " complete.") elif rule_str == 'NOTIN': self._outprint("there is no lifecycle named: " + rule_id)
def _list_all_buckets(self): resp = self.client.listBuckets() check_resp(resp) if not resp.body.buckets: self._outprint("no bucket found.") bucket_infos = [] for bucket in resp.body.buckets: storage = ObsCmdUtil(self.client).get_bucket_storage(bucket.name) bucket_infos.append((bucket, storage)) for bucket, storage in bucket_infos: self._outprint("%-19s\t%-8s\t%-8s\t%-8s\n" % (bucket.create_date, bucket.location, storage, join_bucket_key(bucket.name)))
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths acl_control = parsed_args.acl_control acl_str = parsed_args.acl bucket, key = split_bucket_key(path) acl = None if not acl_control and acl_str: acl = self._build_acl(acl_str) resp = self.client.setBucketAcl(bucket, acl=acl, aclControl=acl_control) # resp = self.client.setBucketAcl(bucket, acl=acl, aclControl=HeadPermission.PUBLIC_READ_WRITE_DELIVERED) check_resp(resp) self._outprint("create acl complete.")
def _get_bucket_website(self, bucket): try: resp = self.client.getBucketWebsiteConfiguration(bucket) check_resp(resp) except Exception as e: if 'NoSuchWebsiteConfiguration' not in str(e): raise e else: self._outprint("no bucket website configuration.") return if resp.body.redirectAllRequestTo: self._outprint('redirectAllRequestTo.hostName: %s' % resp.body.redirectAllRequestTo.hostName) self._outprint('redirectAllRequestTo.protocol: %s' % resp.body.redirectAllRequestTo.protocol) if resp.body.indexDocument: self._outprint('indexDocument.suffix: %s' % resp.body.indexDocument.suffix) if resp.body.errorDocument: self._outprint('errorDocument.key: %s' % resp.body.errorDocument.key) self._outprint('') if resp.body.routingRules: index = 1 for rout in resp.body.routingRules: self._outprint('##### routingRule[%d]:' % index) index += 1 self._outprint('condition.keyPrefixEquals: %s ' % rout.condition.keyPrefixEquals) self._outprint('condition.httpErrorCodeReturnedEquals: %s' % rout.condition.httpErrorCodeReturnedEquals) self._outprint('redirect.protocol: %s ' % rout.redirect.protocol) self._outprint('redirect.hostName: %s' % rout.redirect.hostName) self._outprint('redirect.replaceKeyPrefixWith: %s' % rout.redirect.replaceKeyPrefixWith) self._outprint('redirect.replaceKeyWith: %s' % rout.redirect.replaceKeyWith) self._outprint('redirect.httpRedirectCode: %s' % rout.redirect.httpRedirectCode) self._outprint('')
def __init__(self, bucket, objkey, upload_file, partsize, tasknum, enable_checkpoint, checkpoint_file, checksum, metadata, obsclient): super(UploadOperation, self).__init__(bucket, objkey, upload_file, partsize, tasknum, enable_checkpoint, checkpoint_file, obsclient) self.checksum = checksum self.metadata = metadata self.cmdtype = 'upload' try: self.size = os.path.getsize(self.filename) self.lastModified = os.path.getmtime(self.filename) except Exception: logger.warning('obtain uploadFile infomation error. Please check') self._delete_record() raise Exception('obtain uploadFile infomation error. Please check') resp_for_check_bucket = self.obscmdutil.head_bucket_nocheck(self.bucket) check_resp(resp_for_check_bucket) self._lock = compat.Lock() self._exception = [] self._record = None
def _get_bucket_lifecycle(self, bucket): try: resp = self.client.getBucketLifecycleConfiguration(bucket) check_resp(resp) except Exception as e: if 'NoSuchLifecycleConfiguration' not in str(e): raise e else: self._outprint("no bucket lifecycle.") return index = 1 for rule in resp.body.lifecycleConfig.rule: self._outprint('\n############## rule [' + str(index) + ']\n') self._outprint('id: %s\n' % rule.id) self._outprint('rule.prefix: %s\n' % rule.prefix) self._outprint('rule.status: %s\n' % rule.status) if rule.noncurrentVersionExpiration: self._outprint('rule.noncurrentVersionExpiration: %s\n' % rule.noncurrentVersionExpiration.noncurrentDays) if rule.expiration: self._outprint('rule.expiration.days: %s\n' % rule.expiration.days) self._outprint('') if rule.noncurrentVersionTransition: self._outprint('### rule.noncurrentVersionExpiration') for item in rule.noncurrentVersionTransition: self._outprint( 'storage: %s\tdays: %d' % (item['storageClass'], item['noncurrentDays'])) self._outprint('') if rule.transition: self._outprint('### rule.transition') for item in rule.transition: self._outprint('storage: %s\tdays: %d' % (item['storageClass'], item['days'])) index += 1
def _run(self, parsed_args, parsed_globals): path = parsed_args.paths website_str = parsed_args.website website_obj = json.loads(website_str) bucket, key = split_bucket_key(path) index_suff = website_obj['indexDocument']['suffix'] error_page = None if website_obj.get('errorDocument') is not None: error_page = website_obj['errorDocument']['key'] if self._check_index_page(index_suff): if self._check_error_page(error_page): resp = self.client.setBucketWebsite(bucket, website=website_obj) check_resp(resp) self._outprint("create bucket website complete.") else: self._outprint( "input error, errorDocument[key] must be html/jpg/png/bmp/webp file in the root directory, and filename can be only consist of English letter, number and '-'." ) else: self._outprint( "input error, indexDocument[suffix] must be html file in the root directory, and filename can be only consist of English letter, number and '-'." )
def test_check_resp_with_no_exception(self): resp = DotDict({'status': 200, 'code': 'ok', 'msg': "ok"}) self.assertEqual(check_resp(resp), None)
def _get_bucket_quota(self, bucket): resp = self.client.getBucketStorageInfo(bucket) check_resp(resp) self._outprint("quota: %s\n" % resp.body.quota)