def list_elements(self, base_folder: str, pattern: [Optional[str]] = '*', return_absolute_paths: bool = True) \
-> List[str]:
if return_absolute_paths:
base_folder = os.path.abspath(base_folder)
from com.obs.client.obs_client import ObsClient
path_to_bucket_info_file = os.path.join(base_folder, 'bucket_info.json')
file_names = []
with open(path_to_bucket_info_file, "r") as bucket_info_file:
bucket_info = json.load(bucket_info_file)
obs_client = ObsClient(access_key_id=self._access_key_id,
secret_access_key=self._secret_access_key,
server=_MUNDI_SERVER)
prefixes = bucket_info['prefixes']
for prefix in prefixes:
objects = obs_client.listObjects(bucketName=bucket_info['bucket'], prefix=prefix)
if objects.status < 300:
for content in objects.body.contents:
remote_file_name = content.key.split('/')[-1]
if fnmatch.fnmatch(remote_file_name, pattern):
file_name = os.path.join(base_folder, remote_file_name)
file_name = file_name.replace('\\', '/')
file_names.append(file_name)
else:
logging.error(objects.errorCode)
obs_client.close()
return file_names
def _get_from_wrapped(
self, data_set_meta_info: DataSetMetaInfo) -> Sequence[FileRef]:
from com.obs.client.obs_client import ObsClient
if data_set_meta_info.data_type not in _DATA_TYPE_PARAMETER_DICTS:
logging.warning(
f'Data Type {data_set_meta_info.data_type} not supported by MUNDI DIAS File System '
f'implementation.')
return []
buckets = self._get_bucket_names(data_set_meta_info)
prefix = self._get_prefix(data_set_meta_info)
obs_client = ObsClient(access_key_id=self._access_key_id,
secret_access_key=self._secret_access_key,
server=_MUNDI_SERVER)
keys = []
excludes = _DATA_TYPE_PARAMETER_DICTS[
data_set_meta_info.data_type]['excludes']
right_bucket = None
for bucket in buckets:
right_bucket = bucket
objects = obs_client.listObjects(bucketName=bucket, prefix=prefix)
if objects.status < 300:
for content in objects.body.contents:
if data_set_meta_info.identifier in content.key:
move_on = False
for exclude in excludes:
if content.key.endswith(exclude):
move_on = True
if not move_on:
keys.append(content.key)
if len(keys) > 0:
break
if len(keys) == 0:
return []
data_set_id = data_set_meta_info.identifier
for key in keys:
relative_path_to_file = key.split(data_set_meta_info.identifier)[1]
target_file = f'{self._temp_dir}/{data_set_meta_info.identifier}{relative_path_to_file}'
if len(keys) == 1:
data_set_id = f'{data_set_meta_info.identifier}{relative_path_to_file}'
resp = obs_client.getObject(right_bucket,
key,
downloadPath=target_file)
if resp.status >= 300:
return []
obs_client.close()
file_ref = FileRef(f'{self._temp_dir}/{data_set_id}',
data_set_meta_info.start_time,
data_set_meta_info.end_time,
get_mime_type(data_set_meta_info.identifier))
return [file_ref]
def download_data(objectURI):
# Create an instance of ObsClient.
obsClient = ObsClient(access_key_id='*** Provide your Access Key ***',
secret_access_key='*** Provide your Secret Key ***',
server='yourdomainname')
# Use the instance to access OBS.
resp = obsClient.getObject('bucketname', objectURI)
if resp.status < 300:
print('requestId:', resp.requestId)
if resp.body and resp.body.response:
while True:
chunk = resp.body.response.read(65536)
if not chunk:
break
print(chunk)
resp.body.response.close()
else:
print('errorCode:', resp.errorCode)
print('errorMessage:', resp.errorMessage)
# Close obsClient.
obsClient.close()
def assure_element_provided(self, name: str) -> bool:
if os.path.exists(name):
return True
from com.obs.client.obs_client import ObsClient
name = name.replace('\\', '/')
base_folder = os.path.abspath(os.path.join(name, os.pardir))
path_to_bucket_info_file = f'{base_folder}/bucket_info.json'
with open(path_to_bucket_info_file, "r") as bucket_info_file:
bucket_info = json.load(bucket_info_file)
prefixes = bucket_info['prefixes']
for prefix in prefixes:
key = f"{prefix}{name.split('/')[-1]}"
obs_client = ObsClient(access_key_id=self._access_key_id,
secret_access_key=self._secret_access_key,
server=_MUNDI_SERVER)
resp = obs_client.getObject(bucketName=bucket_info['bucket'], objectKey=key, downloadPath=name)
if resp.status < 300:
logging.info(f"Downloaded from bucket {bucket_info['bucket']} and prefix {key} to {name}")
obs_client.close()
break
return os.path.exists(name)
# 引入模块
from com.obs.client.obs_client import ObsClient
# 创建ObsClient实例
obsClient = ObsClient(access_key_id='*** Provide your Access Key ***',
secret_access_key='*** Provide your Secret Key ***',
server='obs.myhwclouds.com')
obsClient.abortMultipartUpload('bucketname', 'objectkey',
'upload id from initiateMultipartUpload')
if resp.status < 300:
print('requestId:', resp.requestId)
else:
print('status:', resp.status)
# from com.obs.models.put_object_header import PutObjectHeader
# headers = PutObjectHeader()
# resp1 = obsClient.getObject('bucket005', 'test')
#
# headers.contentLength = dict(resp1.header)['content-length']
#
# resp = obsClient.putContent('bucket008', 'test_python', resp1.body.response, headers=headers)
# print('common msg:status:', resp.status, ',errorCode:', resp.errorCode, ',errorMessage:', resp.errorMessage)
#
# resp1.body.response.close()
# 关闭obsClient
obsClient.close()
f = open(file_name, 'wb')
f.write(resp)
f.close()
return True
#1. download web url to import obs
#2. get obs read temp url
def change_weburl_2_obs(url, client, bucket, prefix, temp_file):
if not download_url_file(url, temp_file):
return False, ''
flag, key, url = upload_file_2_obs(client, bucket, prefix, temp_file)
os.remove(temp_file)
return flag, url
if __name__ == "__main__":
client = ObsClient(access_key_id='OBS AK',
secret_access_key='OBS SK',
server='obs.cn-north-1.myhwclouds.com')
bucket = 'bucketXXX'
print change_weburl_2_obs(
'http://c.hiphotos.baidu.com/image/pic/item/ae51f3deb48f8c540e2d1dd336292df5e1fe7f54.jpg',
client, bucket, 'WEBURL', 'a.jpg')
client.close()
#print upload_file_2_obs(client, bucket, 'MANUAL', 'a.jpg')
#print download_url_file('http://c.hiphotos.baidu.com/image/pic/item/ae51f3deb48f8c540e2d1dd336292df5e1fe7f54.jpg', 'a.jpg')
def run():
# Initialize Parameters (Proxy not Used in this script)
ProxyHost = None
ProxyPort = None
ProxyUserName = None
ProxyPassword = None
Proxies = None
VerifyCert = False
LastMarker = None
FinalMarker = None
Prefix = None
OBSEndpoint = "obs.eu-de.otc.t-systems.com"
MaxKeys = 1000
# Read Parameters passed by Splunk Configuration
# config = get_config()
# Instance = config["name"]
# IdpName = config["idpname"]
# BucketFolder = config["bucketfolder"]
# BucketName = config["bucketname"]
# Prefix = config["logprefix"]
# UserName = config["username"]
# UserPass = config["userpass"]
# CheckPoint_dir = config["checkpoint_dir"]
Instance = "obs_ta_idp//VPCFLOW"
IdpName = "IDP"
BucketFolder = "LogTanks"
BucketName = "obs-vpcflow"
Prefix = "VPC"
UserName = "******"
UserPass = "******"
CheckPoint_dir = "c:/temp"
# # Setup Checkpoint file name based on Instance name. We ae parsing the name passed by Splunk
slist = Instance.split("//")
InstanceName = slist[1]
CheckPoint = os.path.join(CheckPoint_dir, InstanceName + ".checkpoint")
# Authenticate with IdP Initiated Federation and return Token (Powershell Script)
TokenID = get_token(UserName, UserPass, IdpName)
# Get Temporary AK/SK from IAM for Federated User
AK, SK, TokenID = get_ak(TokenID, Proxies, VerifyCert)
# Constructs a obs client instance with your account for accessing OBS
# https://docs.otc.t-systems.com/en-us/sdk_python_api/obs/en-us_topic_0080493206.html
obsClient = ObsClient(access_key_id=AK,
secret_access_key=SK,
security_token=TokenID,
server=OBSEndpoint,
proxy_host=ProxyHost,
proxy_port=ProxyPort,
proxy_username=ProxyUserName,
proxy_password=ProxyPassword)
# We check if the last run saved the checkpoint object so that we don't process already processed logs.
if os.path.exists(CheckPoint):
if os.path.getsize(CheckPoint):
fo = open(CheckPoint, "r+")
LastMarker = fo.readline()
fo.close()
while True:
# Start Processing Logs using the listobjects function defined above. This may cycle multiple times of more than maxkey returned.
FinalMarker, FinalMarkerTag = processlogs(obsClient,
BucketName,
BucketFolder,
CheckPoint,
prefix=Prefix,
marker=LastMarker,
max_keys=MaxKeys,
source=InstanceName)
if FinalMarkerTag is None:
obsClient.close()
break
LastMarker = FinalMarker