def list_elements(self, base_folder: str, pattern: [Optional[str]] = '*', return_absolute_paths: bool = True) \ -> List[str]: if return_absolute_paths: base_folder = os.path.abspath(base_folder) from com.obs.client.obs_client import ObsClient path_to_bucket_info_file = os.path.join(base_folder, 'bucket_info.json') file_names = [] with open(path_to_bucket_info_file, "r") as bucket_info_file: bucket_info = json.load(bucket_info_file) obs_client = ObsClient(access_key_id=self._access_key_id, secret_access_key=self._secret_access_key, server=_MUNDI_SERVER) prefixes = bucket_info['prefixes'] for prefix in prefixes: objects = obs_client.listObjects(bucketName=bucket_info['bucket'], prefix=prefix) if objects.status < 300: for content in objects.body.contents: remote_file_name = content.key.split('/')[-1] if fnmatch.fnmatch(remote_file_name, pattern): file_name = os.path.join(base_folder, remote_file_name) file_name = file_name.replace('\\', '/') file_names.append(file_name) else: logging.error(objects.errorCode) obs_client.close() return file_names
def _get_from_wrapped( self, data_set_meta_info: DataSetMetaInfo) -> Sequence[FileRef]: from com.obs.client.obs_client import ObsClient if data_set_meta_info.data_type not in _DATA_TYPE_PARAMETER_DICTS: logging.warning( f'Data Type {data_set_meta_info.data_type} not supported by MUNDI DIAS File System ' f'implementation.') return [] buckets = self._get_bucket_names(data_set_meta_info) prefix = self._get_prefix(data_set_meta_info) obs_client = ObsClient(access_key_id=self._access_key_id, secret_access_key=self._secret_access_key, server=_MUNDI_SERVER) keys = [] excludes = _DATA_TYPE_PARAMETER_DICTS[ data_set_meta_info.data_type]['excludes'] right_bucket = None for bucket in buckets: right_bucket = bucket objects = obs_client.listObjects(bucketName=bucket, prefix=prefix) if objects.status < 300: for content in objects.body.contents: if data_set_meta_info.identifier in content.key: move_on = False for exclude in excludes: if content.key.endswith(exclude): move_on = True if not move_on: keys.append(content.key) if len(keys) > 0: break if len(keys) == 0: return [] data_set_id = data_set_meta_info.identifier for key in keys: relative_path_to_file = key.split(data_set_meta_info.identifier)[1] target_file = f'{self._temp_dir}/{data_set_meta_info.identifier}{relative_path_to_file}' if len(keys) == 1: data_set_id = f'{data_set_meta_info.identifier}{relative_path_to_file}' resp = obs_client.getObject(right_bucket, key, downloadPath=target_file) if resp.status >= 300: return [] obs_client.close() file_ref = FileRef(f'{self._temp_dir}/{data_set_id}', data_set_meta_info.start_time, data_set_meta_info.end_time, get_mime_type(data_set_meta_info.identifier)) return [file_ref]
def download_data(objectURI): # Create an instance of ObsClient. obsClient = ObsClient(access_key_id='*** Provide your Access Key ***', secret_access_key='*** Provide your Secret Key ***', server='yourdomainname') # Use the instance to access OBS. resp = obsClient.getObject('bucketname', objectURI) if resp.status < 300: print('requestId:', resp.requestId) if resp.body and resp.body.response: while True: chunk = resp.body.response.read(65536) if not chunk: break print(chunk) resp.body.response.close() else: print('errorCode:', resp.errorCode) print('errorMessage:', resp.errorMessage) # Close obsClient. obsClient.close()
def assure_element_provided(self, name: str) -> bool: if os.path.exists(name): return True from com.obs.client.obs_client import ObsClient name = name.replace('\\', '/') base_folder = os.path.abspath(os.path.join(name, os.pardir)) path_to_bucket_info_file = f'{base_folder}/bucket_info.json' with open(path_to_bucket_info_file, "r") as bucket_info_file: bucket_info = json.load(bucket_info_file) prefixes = bucket_info['prefixes'] for prefix in prefixes: key = f"{prefix}{name.split('/')[-1]}" obs_client = ObsClient(access_key_id=self._access_key_id, secret_access_key=self._secret_access_key, server=_MUNDI_SERVER) resp = obs_client.getObject(bucketName=bucket_info['bucket'], objectKey=key, downloadPath=name) if resp.status < 300: logging.info(f"Downloaded from bucket {bucket_info['bucket']} and prefix {key} to {name}") obs_client.close() break return os.path.exists(name)
# 引入模块 from com.obs.client.obs_client import ObsClient # 创建ObsClient实例 obsClient = ObsClient(access_key_id='*** Provide your Access Key ***', secret_access_key='*** Provide your Secret Key ***', server='obs.myhwclouds.com') obsClient.abortMultipartUpload('bucketname', 'objectkey', 'upload id from initiateMultipartUpload') if resp.status < 300: print('requestId:', resp.requestId) else: print('status:', resp.status) # from com.obs.models.put_object_header import PutObjectHeader # headers = PutObjectHeader() # resp1 = obsClient.getObject('bucket005', 'test') # # headers.contentLength = dict(resp1.header)['content-length'] # # resp = obsClient.putContent('bucket008', 'test_python', resp1.body.response, headers=headers) # print('common msg:status:', resp.status, ',errorCode:', resp.errorCode, ',errorMessage:', resp.errorMessage) # # resp1.body.response.close() # 关闭obsClient obsClient.close()
f = open(file_name, 'wb') f.write(resp) f.close() return True #1. download web url to import obs #2. get obs read temp url def change_weburl_2_obs(url, client, bucket, prefix, temp_file): if not download_url_file(url, temp_file): return False, '' flag, key, url = upload_file_2_obs(client, bucket, prefix, temp_file) os.remove(temp_file) return flag, url if __name__ == "__main__": client = ObsClient(access_key_id='OBS AK', secret_access_key='OBS SK', server='obs.cn-north-1.myhwclouds.com') bucket = 'bucketXXX' print change_weburl_2_obs( 'http://c.hiphotos.baidu.com/image/pic/item/ae51f3deb48f8c540e2d1dd336292df5e1fe7f54.jpg', client, bucket, 'WEBURL', 'a.jpg') client.close() #print upload_file_2_obs(client, bucket, 'MANUAL', 'a.jpg') #print download_url_file('http://c.hiphotos.baidu.com/image/pic/item/ae51f3deb48f8c540e2d1dd336292df5e1fe7f54.jpg', 'a.jpg')
def run(): # Initialize Parameters (Proxy not Used in this script) ProxyHost = None ProxyPort = None ProxyUserName = None ProxyPassword = None Proxies = None VerifyCert = False LastMarker = None FinalMarker = None Prefix = None OBSEndpoint = "obs.eu-de.otc.t-systems.com" MaxKeys = 1000 # Read Parameters passed by Splunk Configuration # config = get_config() # Instance = config["name"] # IdpName = config["idpname"] # BucketFolder = config["bucketfolder"] # BucketName = config["bucketname"] # Prefix = config["logprefix"] # UserName = config["username"] # UserPass = config["userpass"] # CheckPoint_dir = config["checkpoint_dir"] Instance = "obs_ta_idp//VPCFLOW" IdpName = "IDP" BucketFolder = "LogTanks" BucketName = "obs-vpcflow" Prefix = "VPC" UserName = "******" UserPass = "******" CheckPoint_dir = "c:/temp" # # Setup Checkpoint file name based on Instance name. We ae parsing the name passed by Splunk slist = Instance.split("//") InstanceName = slist[1] CheckPoint = os.path.join(CheckPoint_dir, InstanceName + ".checkpoint") # Authenticate with IdP Initiated Federation and return Token (Powershell Script) TokenID = get_token(UserName, UserPass, IdpName) # Get Temporary AK/SK from IAM for Federated User AK, SK, TokenID = get_ak(TokenID, Proxies, VerifyCert) # Constructs a obs client instance with your account for accessing OBS # https://docs.otc.t-systems.com/en-us/sdk_python_api/obs/en-us_topic_0080493206.html obsClient = ObsClient(access_key_id=AK, secret_access_key=SK, security_token=TokenID, server=OBSEndpoint, proxy_host=ProxyHost, proxy_port=ProxyPort, proxy_username=ProxyUserName, proxy_password=ProxyPassword) # We check if the last run saved the checkpoint object so that we don't process already processed logs. if os.path.exists(CheckPoint): if os.path.getsize(CheckPoint): fo = open(CheckPoint, "r+") LastMarker = fo.readline() fo.close() while True: # Start Processing Logs using the listobjects function defined above. This may cycle multiple times of more than maxkey returned. FinalMarker, FinalMarkerTag = processlogs(obsClient, BucketName, BucketFolder, CheckPoint, prefix=Prefix, marker=LastMarker, max_keys=MaxKeys, source=InstanceName) if FinalMarkerTag is None: obsClient.close() break LastMarker = FinalMarker