def get_object(session=None, bucket=None, key=None, gettype=None): """ Get object function include several variable session = session from where it called from bucket = your bucket name key = your object file name gettype = your type of object result None|acl|tagging|torrent """ if not session: session = login_lib.get_client_session() try: if gettype is None: detail_object = session.get_object(Bucket=bucket, Key=key) elif gettype == 'acl': detail_object = session.get_object_acl(Bucket=bucket, Key=key) elif gettype == 'tagging': detail_object = session.get_object_tagging(Bucket=bucket, Key=key) elif gettype == 'torrent': detail_object = session.get_object_torrent(Bucket=bucket, Key=key) return detail_object except Exception as e: log_utils.log_err(e) exit()
def do_create(initialize): try: create_data = {} stack = list(initialize.keys())[0] stack_name = list(initialize[stack])[0] if stack == 'cloudian': if initialize[stack][stack_name]['template'] == 'user': parameters = initialize[stack][stack_name]['parameters'] create_data = user.create(data=None, json=parameters) if initialize[stack][stack_name]['template'] == 'credentials': parameters = initialize[stack][stack_name]['parameters'] method = 'POST' if 'accessKey' not in parameters: method = 'PUT' create_data = credential.create(data=parameters, json=None, method=method) return create_data except Exception as e: log_utils.log_err(e) raise else: pass finally: pass
def delete_bucket(session=None, name=None): if not session: session = login_lib.get_client_session() try: bucket = session.delete_bucket(Bucket=name) return bucket except Exception as e: log_utils.log_err(e) exit()
def list_object(session=None, bucket=None): if not session: session = login_lib.get_client_session() list_object = session.list_objects(Bucket=bucket) try: return list_object.get('Contents') except Exception as e: log_utils.log_err(e) exit()
def group_list(data=None, json=None, method='GET'): url = base_url + '/list' groups = requestors.request(url=url, data=data, json=json, method=method) try: groups_data = groups['data'] except Exception: log_utils.log_err(groups['status_message']) else: return [group['groupId'] for group in list(groups_data)]
def bucket_list(session=None): if not session: session = login_lib.get_client_session() bucket = session.list_buckets() try: list_bucket = bucket.__getitem__('Buckets') return list_bucket except Exception as e: log_utils.log_err(e) exit()
def get_usage(data=None, json=None, method='GET'): usage = requestors.request(url=base_url, data=data, json=json, method=method) try: usage_data = usage['data'] except Exception: log_utils.log_err(usage['status_message']) else: return usage_data
def delete(data=None, json=None, method='DELETE'): user_credentials = requestors.request(url=user.base_url + '/credentials', data=data, json=json, method=method) try: user_credentials_data = user_credentials['data'] except Exception: log_utils.log_err(user_credentials['status_message']) else: return user_credentials_data
def get_permission(data=None, json=None, method='GET'): permission = requestors.request(url=base_url, data=data, json=json, method=method) try: permission_data = permission['data'] except Exception: log_utils.log_err(permission['status_message']) else: return permission_data
def buckets_policy(data=None, json=None, method='GET'): bucket_policy = requestors.request(url=base_url + '/bucketsperpolicy', data=data, json=json, method=method) try: bucket_policy_data = bucket_policy['data'] except Exception: log_utils.log_err(bucket_policy['status_message']) else: return bucket_policy_data
def delete(data=None, json=None, method='DELETE'): delete_user = requestors.request(url=base_url, data=data, json=json, method=method) try: data_user = delete_user['data'] except Exception: log_utils.log_err(delete_user['status_message']) return None else: return data_user
def create(data=None, json=None, method='PUT'): create_user = requestors.request(url=base_url, data=data, json=json, method=method) try: data_user = create_user['data'] except Exception: log_utils.log_err(create_user['status_message']) raise else: return data_user
def detail(data=None, json=None, method='GET'): get_user = requestors.request(url=base_url, data=data, json=json, method=method) try: data_user = get_user['data'] except Exception: log_utils.log_err(get_user['status_message']) return None else: return data_user
def get_list(data=None, json=None, method='GET'): base_url_list = base_url + '/list' list_user = requestors.request(url=base_url_list, data=data, json=json, method=method) try: data_user = list_user['data'] except Exception: log_utils.log_err(list_user['status_message']) return None else: return data_user
def get(data=None, json=None, method='GET'): qos_details = requestors.request(url=base_url, data=data, json=json, method=method) try: qos_data = qos_details['data'] except Exception as e: log_utils.log_err(qos_details['status_message']) return None else: return qos_data
def get_client_session(): try: if check_session(): sess = None with open('/tmp/s3_session.pkl', 'rb') as f: sess = dill.load(f) return sess.client('s3', endpoint_url=S3_ENDPOINT) else: log_utils.log_err("Loading Session Failed") log_utils.log_err("Please login first") except Exception as e: log_utils.log_err("Loading Session Failed") log_utils.log_err("Please login first") log_utils.log_err(e)
def post_bucket(session=None, name=None, json=None): if not session: session = login_lib.get_client_session() try: if json is None: bucket = session.create_bucket(Bucket=name) else: bucket = session.create_bucket(Bucket=name, **json) return bucket except Exception as e: log_utils.log_err(e) exit()
def get_policy(data=None, json=None, method='GET'): url = base_url if data is None: url = base_url + '/listpolicy' elif 'policyId' in data: url = base_url policy = requestors.request(url=url, data=data, json=json, method=method) try: policy_data = policy['data'] except Exception: log_utils.log_err(policy['status_message']) else: return policy_data
def put_bucket_acl(session=None, name=None, aclParams=None): if not session: session = login_lib.get_client_session() try: bucket = None if aclParams.get('acp') is None: bucket = session.put_bucket_acl(Bucket=name, ACL=aclParams['acl']) else: bucket = session.put_bucket_acl( Bucket=name, ACL=aclParams['acl'], AccessControlPolicy=aclParams['acp']) return bucket except Exception as e: log_utils.log_err(e) exit()
def custom_post_bucket(session=None, name=None, policy=None): if not session: session = login_lib.get_client_session() try: url = session.generate_presigned_url('create_bucket', Params={ 'Bucket': name, }, HttpMethod='PUT') headers = dict() headers['x-gmt-policyid'] = policy create_bucket = requests.put(url, headers=headers) return create_bucket except Exception as e: log_utils.log_err(e) exit()
def put_object_multipart(session=None, bucket=None, key=None): config = TransferConfig(multipart_threshold=1024 * 25, max_concurrency=10, multipart_chunksize=1024 * 25, use_threads=True) if not session: session = login_lib.get_client_session() try: with open(key, 'rb') as data: object = session.upload_file(Filename=data.name, Bucket=bucket, Key=os.path.basename(data.name), Config=config) return object except Exception as e: log_utils.log_err(e) exit()
def get_credential(data=None, json=None, method='GET'): url = user.base_url if 'accessKey' in data: url = url + '/credentials' elif 'userId' in data and 'groupId' in data: url = url + '/credentials/list' if 'active' in data: url = url + '/active' credentials = requestors.request(url=url, data=data, json=json, method=method) try: credentials_data = credentials['data'] except Exception: log_utils.log_err(credentials['status_message']) else: return credentials_data
def do_update(initialize): try: update_data = {} stack = list(initialize.keys())[0] stack_name = list(initialize[stack])[0] if stack == 'cloudian': if initialize[stack][stack_name]['template'] == 'user': parameters = initialize[stack][stack_name]['parameters'] create_data = user.update(data=None, json=parameters) if initialize[stack][stack_name]['template'] == 'credentials': parameters = initialize[stack][stack_name]['parameters'] create_data = credential.update(data=None, json=parameters) return create_data except Exception as e: log_utils.log_err(e) raise else: pass finally: pass
def put_object(session=None, bucket=None, key=None, acl=None, tagging=None): if not session: session = login_lib.get_client_session() try: if acl is None and tagging is None: # put_object with open(key, 'rb') as data: object = session.put_object( Bucket=bucket, Key=key, ContentType=mimetypes.MimeTypes().guess_type(key)[0], Body=data) elif acl is not None and tagging is None: # put object acl object = session.put_object_acl(Bucket=bucket, Key=key, ACL=acl) elif acl is None and tagging is None: # put object tagging object = session.put_object_tagging(Bucket=bucket, Key=key, Tagging=tagging) else: # put object acl tagging with open(key, 'rb') as data: object = session.put_object( Bucket=bucket, Key=key, ContentType=mimetypes.MimeTypes().guess_type(key)[0], Body=data, ACL=acl, Tagging=tagging) return object except Exception as e: log_utils.log_err(e) exit()
def execute(self): set_file = self.args["--file"] default_file = orchestration.check_manifest_file() if set_file: if os.path.exists(set_file): default_file = set_file else: log_utils.log_err("{} file is not exists!".format(set_file)) exit() if not default_file: log_utils.log_err("Can't find obs.yml manifest file!") q_stack = cli_utils.question( "Do you want to generate obs.yml manifest? ") if q_stack: if self.args["--interactive"]: print(ncurses.init()) else: print(prompt.init()) q_deploy = cli_utils.question("Continue to deploy? ") if q_deploy: default_file = "obs.yml" else: exit() else: exit() else: q_deploy = cli_utils.question("Continue to deploy? ") if q_deploy: default_file = "obs.yml" else: exit() deploy_init = yaml_utils.file_parser(default_file) try: orchestration.do_create(deploy_init) except Exception as e: log_utils.log_err("Deploying Stack failed...") exit() stack = list(deploy_init.keys())[0] stack_name = list(deploy_init[stack])[0] parameters = deploy_init[stack][stack_name]['parameters'] project_list = list() data = {} for key in parameters: if parameters[key] != '': data[key] = parameters[key] project_list.append(data) if len(project_list) > 0: print(tabulate(project_list, headers='keys', tablefmt="grid"))
def execute(self): set_file = self.args["--file"] default_file = orchestration.check_manifest_file() if set_file: if os.path.exists(set_file): default_file = set_file else: log_utils.log_err("{} file is not exists!".format(set_file)) exit() if not default_file: log_utils.log_err("Can't find obs.yml manifest file!") exit() deploy_init = yaml_utils.file_parser(default_file) try: orchestration.do_update(deploy_init) except Exception as e: log_utils.log_err("Deploying Stack failed...") exit()
def dump_session(sess): try: with open('/tmp/s3_session.pkl', 'wb') as f: dill.dump(sess, f) except Exception: log_utils.log_err("Dump session failed")
def execute(self): if self.args['user']: if self.args['--group_id'] and self.args['--id']: tab_data = list() data = { 'groupId': self.args['--group_id'], 'userId': self.args['--id'], } user_data = user.detail(data=data) if user_data is None: log_utils.log_err(user_data) exit() else: user_data_fix = { 'UserType': user_data['userType'], 'Name': user_data['fullName'], 'EmailAddr': user_data['emailAddr'], 'Address': user_data['address1'] + user_data['address2'], 'City': user_data['city'], 'Status': user_data['active'] } tab_data.append(user_data_fix) print(tabulate(tab_data, headers='keys', tablefmt='grid')) else: data = { 'groupId': 'testing', 'userType': 'all', 'userStatus': 'active' } data_user = list() all_user = user.get_list(data=data) number = 1 for items in all_user: data_item = { 'No.': number, 'User': items['userId'], 'Name': items['fullName'], 'EmailAddr': items['emailAddr'], 'Address': items['address1'], 'City': items['city'], 'Status': items['active'] } number = number + 1 data_user.append(data_item) print(tabulate(data_user, headers='keys', tablefmt='grid')) if self.args['qos']: if self.args['--group_id'] and self.args['--id']: tab_data = list() list_params = { 'userId': self.args['--id'], 'groupId': self.args['--group_id'] } try: qos_data = qos.get(data=list_params) limitList = qos_data['qosLimitList'] except Exception as e: log_utils.log_err(e) exit() else: quota = 0 for i in limitList: if i['type'] == 'STORAGE_QUOTA_KBYTES': quota = i['value'] user_qos = { 'User': qos_data['userId'], 'Quota': '{} GB'.format(quota / 1024**2) } tab_data.append(user_qos) print(tabulate(tab_data, headers='keys', tablefmt='grid')) exit() else: log_utils.log_err('Missing parameter.') exit() if self.args['credential']: if self.args['--group_id'] and self.args['--id']: list_params = { 'userId': self.args['--id'], 'groupId': self.args['--group_id'] } credential_list = credential.get_credential(data=list_params) if credential_list is None: log_utils.log_err(credential_list) exit() else: number = 1 data_credential = list() for item in credential_list: cTime = time.strftime( "%d-%m-%Y, %H:%M:%S", time.localtime(int(str(item['createDate'])[:-3]))) if item['expireDate'] is not None: dTime = time.strftime( "%d-%m-%Y, %H:%M:%S", time.localtime( int(str(item['expireDate'])[:-3]))) else: dTime = item['expireDate'] data_item = { 'No.': number, 'Access Key': item['accessKey'], 'Secret Key': item['secretKey'], 'Create': cTime, 'Expire': dTime, 'Status': item['active'] } number = number + 1 data_credential.append(data_item) print( tabulate(data_credential, headers='keys', tablefmt='grid')) exit() elif self.args['--key']: tab_data = list() list_params = {'accessKey': self.args['--key']} credential_data = credential.get_credential(data=list_params) if credential_data is None: log_utils.log_err(credential_list) exit() else: cTime = time.strftime( "%d-%m-%Y, %H:%M:%S", time.localtime( int(str(credential_data['createDate'])[:-3]))) if credential_data['expireDate'] is not None: dTime = time.strftime( "%d-%m-%Y, %H:%M:%deaS", time.localtime( int(str(credential_data['expireDate'])[:-3]))) else: dTime = credential_data['expireDate'] cred_detail = { 'Access Key': credential_data['accessKey'], 'Secret Key': credential_data['secretKey'], 'Create': cTime, 'Expire': dTime, 'Status': credential_data['active'] } tab_data.append(cred_detail) print(tabulate(tab_data, headers='keys', tablefmt='grid')) exit() else: log_utils.log_err('Missing parameter.') exit() if self.args['bucket']: tab_data = list() list_bucket = bucket.bucket_list() number = 1 for item in list_bucket: data_item = { 'No.': number, 'Name': item['Name'], 'Created': item['CreationDate'] } number = number + 1 tab_data.append(data_item) print(tabulate(tab_data, headers='keys', tablefmt='grid')) if self.args['object']: if self.args['--bucket']: tab_data = list() list_object = object.list_object(bucket=self.args['--bucket']) number = 1 for item in list_object: data_item = { 'No.': number, 'Name': item['Key'], 'LastModified': item['LastModified'], 'Size': '{} B'.format(item['Size']), 'OwnerName': item['Owner']['DisplayName'], } number = number + 1 tab_data.append(data_item) print(tabulate(tab_data, headers='keys', tablefmt='grid')) else: log_utils.log_err('Missing parameter.') exit()
def set_session(sess): try: with open('/tmp/obs-session.pkl', 'wb') as f: dill.dump(sess, f) except Exception as e: log.log_err("set session failed")