def _check_download_record(self, record):
"""
first check checkpoint file base info
than check complete parts size
:param record:
:return:
"""
if not (os.path.exists(self._tmp_file) and os.path.getsize(self._tmp_file) == self.size):
return False
try:
if not operator.eq([record['bucketName'], record['objectKey'], record['versionId'],
record['downloadFile']],
[self.bucket, self.objkey, self.versionid,
self.filename]):
logger.warning('the bucketName or objectKey or downloadFile was changed. clear the record')
return False
object_meta_resp = self.obscmdutil.get_object_metadata_nocheck(self.bucket, self.objkey,
self.versionid)
check_resp(object_meta_resp)
if not operator.eq(record['objectStatus'],
[self.objkey, object_meta_resp.body.contentLength,
object_meta_resp.body.lastModified, self.versionid]):
logger.warning('the versionId or lastModified was changed. clear the record')
return False
except:
return False
return True
def __init__(self, bucket, objkey, src_obspath, partsize, tasknum, enable_checkpoint, checkpoint_file,
checksum, metadata, obsclient):
super(UploadOperation, self).__init__(bucket, objkey, src_obspath, partsize, tasknum, enable_checkpoint,
checkpoint_file, obsclient)
self.checksum = checksum
self.metadata = metadata
self.cmdtype = 'copy'
self.copy_source = self.filename.replace('obs://', '')
srcb, srck = split_bucket_key(src_obspath)
metedata_resp = self.obscmdutil.get_object_metadata_nocheck(srcb, srck)
if metedata_resp.status < 300:
self.lastModified = metedata_resp.body.lastModified
self.size = metedata_resp.body.contentLength
else:
logger.warning(
'touch the objetc {0} error. ErrorCode:{1}, ErrorMessage:{2}'.format(
self.objkey, metedata_resp.errorCode, metedata_resp.errorMessage))
self._delete_record()
raise Exception(
'touch the objetc {0} error. ErrorCode:{1}, ErrorMessage:{2}'.format(
self.objkey, metedata_resp.status, metedata_resp.errorMessage))
resp_for_check_bucket = self.obscmdutil.head_bucket_nocheck(self.bucket)
check_resp(resp_for_check_bucket)
self._lock = compat.Lock()
self._exception = []
self._record = None
def _get_bucket_acl(self, bucket):
resp = self.client.getBucketAcl(bucket)
check_resp(resp)
self._outprint('owner_id: %s\n' % resp.body.owner.owner_id)
self._outprint('owner_name: %s\n' % resp.body.owner.owner_name)
acls = {}
for grant in resp.body.grants:
group = grant.grantee.group
if group not in acls:
acls[group] = {}
grantee_id = grant.grantee.grantee_id
if grantee_id not in acls[group]:
acls[group][grantee_id] = {}
acls[group][grantee_id]['name'] = grant.grantee.grantee_name
if 'premission' not in acls[group][grantee_id]:
acls[group][grantee_id]['premission'] = []
acls[group][grantee_id]['premission'].append(grant.permission)
for group, grantees in acls.items():
self._outprint('\n############## group: %s' % group)
for grantee_id, item in grantees.items():
self._outprint('grant_id: %s\n' % grantee_id)
self._outprint('grant_name: %s\n' % item['name'])
self._outprint('permission: %s\n\n' %
', '.join(item['premission']))
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
bucket, key = split_bucket_key(path)
self._warning_prompt('Are you sure to delete bucket website?')
resp = self.client.deleteBucketWebsite(bucket)
check_resp(resp)
self._outprint("delete bucket website complete.")
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
policy = parsed_args.policy
bucket, key = split_bucket_key(path)
resp = self.client.setBucketPolicy(bucket, policy)
check_resp(resp)
self._outprint("create policy complete: %s\n" % path)
def _get_bucket_location(self, bucket):
"""
get bucket location info
:param bucket:
:return:
"""
resp = self.client.getBucketLocation(bucket)
check_resp(resp)
self._outprint("location: %s\n" % resp.body.location)
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
rule_str = parsed_args.lifecycle
bucket, key = split_bucket_key(path)
rule = self._check_lifecycle(bucket, rule_str)
lifecycle = json.loads(rule)
resp = self.client.setBucketLifecycle(bucket, lifecycle)
check_resp(resp)
self._outprint("create bucket lifecycle complete.")
def _get_bucket_storage(self, bucket):
"""
get bucket storage info, include size and object number
:param bucket:
:return:
"""
resp = self.client.getBucketStorageInfo(bucket)
check_resp(resp)
self._outprint("size: %s\n" % bytes_to_unitstr(resp.body.size))
self._outprint("objectNumber: %s\n" % resp.body.objectNumber)
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
log_str = parsed_args.log
bucket, key = split_bucket_key(path)
logstatus = None
if log_str is not None:
logstatus = self._build_log(log_str=log_str)
resp = self.client.setBucketLogging(bucket, logstatus=logstatus)
check_resp(resp)
self._outprint("set bucket log complete.")
def _get_bucket_metadata(self, bucket):
"""
get bucket metadata info
:param bucket:
:return:
"""
resp = self.client.getBucketMetadata(bucket)
check_resp(resp)
self._outprint('obs version: %s' % resp.body.obsVersion)
self._outprint("storage type: %s" % resp.body.storageClass)
self._outprint('location: %s' % resp.body.location)
def _get_bucket_policy(self, bucket):
try:
resp = self.client.getBucketPolicy(bucket)
check_resp(resp)
except Exception as e:
if 'NoSuchBucketPolicy' not in str(e):
raise e
else:
self._outprint("no bucket policy.")
return
self._outprint("policyJSON: %s\n" % resp.body.policyJSON)
def create_bucket(client, name=None, region=None):
"""
Creates a bucket
:returns: the name of the bucket created
"""
if name:
bucket_name = name
else:
bucket_name = random_string()
resp = client.createBucket(name)
check_resp(resp)
return bucket_name
def _get_bucket_logging(self, bucket):
resp = self.client.getBucketLoggingConfiguration(bucket)
check_resp(resp)
self._outprint('targetBucket: %s\n' % resp.body.targetBucket)
self._outprint('targetPrefix: %s\n' % resp.body.targetPrefix)
index = 1
for grant in resp.body.targetGrants:
self._outprint('\n############## grant [' + str(index) + ']')
self._outprint('grant_id: %s\n' % grant.grantee.grantee_id)
self._outprint('grant_name: %s\n' % grant.grantee.grantee_name)
self._outprint('group: %s\n' % grant.grantee.group)
self._outprint('permission: %s\n' % grant.permission)
index += 1
def _get_bucket_tagging(self, bucket):
try:
resp = self.client.getBucketTagging(bucket)
check_resp(resp)
except Exception as e:
if 'NoSuchTagSet' not in str(e):
raise e
else:
self._outprint("no bucket tag.")
return
for tag in resp.body.tagSet:
self._outprint('tag {key}: {value}\n'.format(key=tag.key,
value=tag.value))
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
rule_id = parsed_args.id
bucket, key = split_bucket_key(path)
self._warning_prompt('Are you sure to delete bucket life circle?')
if rule_id is None:
# delete all lifecycles
resp = self.client.deleteBucketLifecycle(bucket)
check_resp(resp)
self._outprint("delete all lifecycle complete.")
else:
# delete one lifecycle
rule_str = self._check_lifecycle(bucket, rule_id)
if rule_str is not None and rule_str != 'NOTIN':
resp = self.client.deleteBucketLifecycle(bucket)
check_resp(resp)
lifecycle = json.loads(rule_str)
resp = self.client.setBucketLifecycle(bucket, lifecycle)
check_resp(resp)
self._outprint("delete lifecycle " + rule_id + " complete.")
elif rule_str is None:
resp = self.client.deleteBucketLifecycle(bucket)
check_resp(resp)
self._outprint("delete lifecycle " + rule_id + " complete.")
elif rule_str == 'NOTIN':
self._outprint("there is no lifecycle named: " + rule_id)
def _list_all_buckets(self):
resp = self.client.listBuckets()
check_resp(resp)
if not resp.body.buckets:
self._outprint("no bucket found.")
bucket_infos = []
for bucket in resp.body.buckets:
storage = ObsCmdUtil(self.client).get_bucket_storage(bucket.name)
bucket_infos.append((bucket, storage))
for bucket, storage in bucket_infos:
self._outprint("%-19s\t%-8s\t%-8s\t%-8s\n" % (bucket.create_date, bucket.location, storage, join_bucket_key(bucket.name)))
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
acl_control = parsed_args.acl_control
acl_str = parsed_args.acl
bucket, key = split_bucket_key(path)
acl = None
if not acl_control and acl_str:
acl = self._build_acl(acl_str)
resp = self.client.setBucketAcl(bucket,
acl=acl,
aclControl=acl_control)
# resp = self.client.setBucketAcl(bucket, acl=acl, aclControl=HeadPermission.PUBLIC_READ_WRITE_DELIVERED)
check_resp(resp)
self._outprint("create acl complete.")
def _get_bucket_website(self, bucket):
try:
resp = self.client.getBucketWebsiteConfiguration(bucket)
check_resp(resp)
except Exception as e:
if 'NoSuchWebsiteConfiguration' not in str(e):
raise e
else:
self._outprint("no bucket website configuration.")
return
if resp.body.redirectAllRequestTo:
self._outprint('redirectAllRequestTo.hostName: %s' %
resp.body.redirectAllRequestTo.hostName)
self._outprint('redirectAllRequestTo.protocol: %s' %
resp.body.redirectAllRequestTo.protocol)
if resp.body.indexDocument:
self._outprint('indexDocument.suffix: %s' %
resp.body.indexDocument.suffix)
if resp.body.errorDocument:
self._outprint('errorDocument.key: %s' %
resp.body.errorDocument.key)
self._outprint('')
if resp.body.routingRules:
index = 1
for rout in resp.body.routingRules:
self._outprint('##### routingRule[%d]:' % index)
index += 1
self._outprint('condition.keyPrefixEquals: %s ' %
rout.condition.keyPrefixEquals)
self._outprint('condition.httpErrorCodeReturnedEquals: %s' %
rout.condition.httpErrorCodeReturnedEquals)
self._outprint('redirect.protocol: %s ' %
rout.redirect.protocol)
self._outprint('redirect.hostName: %s' %
rout.redirect.hostName)
self._outprint('redirect.replaceKeyPrefixWith: %s' %
rout.redirect.replaceKeyPrefixWith)
self._outprint('redirect.replaceKeyWith: %s' %
rout.redirect.replaceKeyWith)
self._outprint('redirect.httpRedirectCode: %s' %
rout.redirect.httpRedirectCode)
self._outprint('')
def __init__(self, bucket, objkey, upload_file, partsize, tasknum, enable_checkpoint, checkpoint_file,
checksum, metadata, obsclient):
super(UploadOperation, self).__init__(bucket, objkey, upload_file, partsize, tasknum, enable_checkpoint,
checkpoint_file, obsclient)
self.checksum = checksum
self.metadata = metadata
self.cmdtype = 'upload'
try:
self.size = os.path.getsize(self.filename)
self.lastModified = os.path.getmtime(self.filename)
except Exception:
logger.warning('obtain uploadFile infomation error. Please check')
self._delete_record()
raise Exception('obtain uploadFile infomation error. Please check')
resp_for_check_bucket = self.obscmdutil.head_bucket_nocheck(self.bucket)
check_resp(resp_for_check_bucket)
self._lock = compat.Lock()
self._exception = []
self._record = None
def _get_bucket_lifecycle(self, bucket):
try:
resp = self.client.getBucketLifecycleConfiguration(bucket)
check_resp(resp)
except Exception as e:
if 'NoSuchLifecycleConfiguration' not in str(e):
raise e
else:
self._outprint("no bucket lifecycle.")
return
index = 1
for rule in resp.body.lifecycleConfig.rule:
self._outprint('\n############## rule [' + str(index) + ']\n')
self._outprint('id: %s\n' % rule.id)
self._outprint('rule.prefix: %s\n' % rule.prefix)
self._outprint('rule.status: %s\n' % rule.status)
if rule.noncurrentVersionExpiration:
self._outprint('rule.noncurrentVersionExpiration: %s\n' %
rule.noncurrentVersionExpiration.noncurrentDays)
if rule.expiration:
self._outprint('rule.expiration.days: %s\n' %
rule.expiration.days)
self._outprint('')
if rule.noncurrentVersionTransition:
self._outprint('### rule.noncurrentVersionExpiration')
for item in rule.noncurrentVersionTransition:
self._outprint(
'storage: %s\tdays: %d' %
(item['storageClass'], item['noncurrentDays']))
self._outprint('')
if rule.transition:
self._outprint('### rule.transition')
for item in rule.transition:
self._outprint('storage: %s\tdays: %d' %
(item['storageClass'], item['days']))
index += 1
def _run(self, parsed_args, parsed_globals):
path = parsed_args.paths
website_str = parsed_args.website
website_obj = json.loads(website_str)
bucket, key = split_bucket_key(path)
index_suff = website_obj['indexDocument']['suffix']
error_page = None
if website_obj.get('errorDocument') is not None:
error_page = website_obj['errorDocument']['key']
if self._check_index_page(index_suff):
if self._check_error_page(error_page):
resp = self.client.setBucketWebsite(bucket,
website=website_obj)
check_resp(resp)
self._outprint("create bucket website complete.")
else:
self._outprint(
"input error, errorDocument[key] must be html/jpg/png/bmp/webp file in the root directory, and filename can be only consist of English letter, number and '-'."
)
else:
self._outprint(
"input error, indexDocument[suffix] must be html file in the root directory, and filename can be only consist of English letter, number and '-'."
)
def test_check_resp_with_no_exception(self):
resp = DotDict({'status': 200, 'code': 'ok', 'msg': "ok"})
self.assertEqual(check_resp(resp), None)
def _get_bucket_quota(self, bucket):
resp = self.client.getBucketStorageInfo(bucket)
check_resp(resp)
self._outprint("quota: %s\n" % resp.body.quota)