def _check_standard(self, data: List[Any]) -> None:
"""
检查数据符合是否标准格式
:param data: 通过云 sdk 接口得到的数据
"""
format = safe_json_loads(self.action.format)
# 获取数据样本
assert data, '传入数据为空'
data_example = data[0]
# 将数据key变成下划线
new_data_example = {}
[new_data_example.update({camel_to_underline(k): v}) for k, v in data_example.items()]
# 获取数据有但是Model字段没有的数据
self.different = get_model_data_different(self.action_model, **new_data_example)
self.different = [underline_to_camel(item) for item in self.different]
for sub_data in data:
assert standard.keys() == sub_data.keys(), \
'传入数据格式不对, 缺少字段: {}, 多余字段: {}'.format(
[i for i in standard if i not in sub_data],
[i for i in sub_data if i not in standard],
)
def _clean_query_resp(self, resp: dict) -> dict:
"""
清洗查询类型动作的响应数据
:param resp: 响应数据
:return: 清洗后的数据
"""
# 提取数据列表，若结果为字符串则进行反序列化
data = self._extract_deep_value(
self._interface['output']['data'], resp, 0)
if isinstance(data, str):
data = safe_json_loads(data)
# 根据配置进行所以记录的清洗，k 和 v 分别表示最后得到的键和值
fields = self._interface['output']['fields']
new_data = []
num = 0
for d in data:
new_d = self._clean_single_query_data(d, num, fields)
new_data.append(new_d)
num += 1
# 获取现有长度和总长度
current = len(new_data)
total = resp.get('TotalCount', current)
# 返回标准结构
return {
'total': total,
'current': current,
'data': new_data,
}
def run(self, last_result: Any = None) -> Any:
"""
执行入库
:param last_result: 上个作业的结果
:return: None
"""
# 若备用文件存在，直接使用备用文件的数据进行存储
backup_file = self._get_backup_file()
if os.path.exists(backup_file):
with open(backup_file) as f:
data = safe_json_loads(f.read())
self._sync_to_storage(data)
os.remove(backup_file)
return
# 否则使用 SDK 请求得到的数据进行存储
client = CloudSDKClient()
req = CloudSDKRequest(self.idc, self.action, self.req_params)
data = client.execute(req)
try:
self._sync_to_storage(data)
except Exception as e:
with open(backup_file, 'w') as f:
f.write(safe_json_dumps(data))
raise e
def to_representation(self, value):
"""
数据是 json，取出前先对 json 解码
"""
if value is None:
return None
value = super().to_representation(value)
return safe_json_loads(value)
def request(self) -> dict:
"""
使用原生 sdk 发送请求，返回响应结果
:return: 包含响应结果的字典
"""
assert self._already, 'request info has not been set，should use self.set()'
url = self._url + self._build_url_params() + '&Signature=' + \
self._get_signature()
try:
resp = requests.get(url, timeout=(3, 30))
data = safe_json_loads(resp.content)
except (requests.exceptions.ReadTimeout,
requests.exceptions.ConnectTimeout):
resp = {'RetCode': '502', 'Message': 'request timeout'}
data = self._build_error_data(resp)
return data
def _do_query_monitor_data_start_hook(resp: dict) -> dict:
"""
开始钩子，转换其中的时间戳为日期，将值转为百分比
:param resp: 响应数据
:return: 清洗后数据
"""
data = safe_json_loads(resp['Datapoints'])
for d in data:
dt = get_datetime_with_tz(d['timestamp'] // 1000)
d.update({
'Day': dt.date(),
'Hour': dt.hour,
'Minute': dt.minute,
'Average': d['Average']
})
resp['Datapoints'] = data
return resp
def request(self) -> dict:
"""
使用原生 sdk 发送请求，返回响应结果
:return: 包含响应结果的字典
"""
assert self._already, 'request info has not been set，should use self.set()'
request = self._get_req()
client = self._get_client()
try:
response = client.do_action_with_exception(request)
except ServerException as e:
return self._build_error_data(e)
except ClientException as e:
raise CloudNativeSDKError(
f'client error: {e.error_code}, {e.message}')
return safe_json_loads(response)
def _request(self) -> dict:
"""
使用新版 sdk 发送请求，返回响应结果
:return: 包含响应结果的字典
"""
# 分别获取客户端对象、请求对象
client = self._get_client()
req = self._get_req()
# 进行请求，得到响应
try:
resp = getattr(client, self._interface['name'])(req)
except TencentCloudSDKException as e:
if not e.requestId:
raise CloudNativeSDKError(f'client error: {e.message}')
return self._build_error_data(e)
# 将结果反序列化为对象并输出
return safe_json_loads(resp.to_json_string())
def _old_request(self) -> dict:
"""
使用老版 sdk 发送请求，返回响应结果
:return: 包含响应结果的字典
"""
# 传入密钥，获取客户端
secret_params = ({
'secretId': self._ak_sk[0],
'secretKey': self._ak_sk[1]
})
client = QcloudApi(self._interface['module'], secret_params)
client.generateUrl(self._interface['name'], self._params)
# 进行请求，得到响应
try:
resp = client.call(self._interface['name'], self._params)
except TencentCloudSDKException as e:
if not e.requestId:
raise CloudNativeSDKError(f'client error: {e.message}')
return self._build_error_data(e)
# 将结果反序列化为对象并输出
return safe_json_loads(resp)
def chk_settings() -> Path:
p: Path = Path(SETTINGS_PATH)
if not p.exists():
print(strs.SET_NEXIST)
else:
SETTINGS = utils.safe_json_loads(p)
if SETTINGS != None:
try:
p = Path(SETTINGS[K_DIR])
except KeyError as err:
print(strs.E_KEY.format(err, p))
print(strs.E_SET_LOAD)
else:
return p
if not utils.yes_no(strs.SET_GEN):
if utils.safe_json_dumps(p, INIT_SETTINGS):
print(strs.E_SET_INIT)
return None
else:
print(strs.SET_EDIT)
return None
return Path(DEFAULT_PATH)
def to_representation(self, value):
"""
数据是 json，取出前先对 json 解码
"""
value = safe_json_loads(value)
return super().to_representation(value)