def _pseudo_send(self, method, project, body, resource, params, headers, respons_body_type='json'): if body: headers['Content-Length'] = str(len(body)) headers['Content-MD5'] = Util.cal_md5(str(body)) else: headers['Content-Length'] = '0' headers["x-log-bodyrawsize"] = '0' headers['x-log-apiversion'] = API_VERSION headers['x-log-signaturemethod'] = 'hmac-sha1' if self._isRowIp or not project: url = self.http_type + self._endpoint else: url = self.http_type + project + "." + self._endpoint if project: headers['Host'] = project + "." + self._logHost else: headers['Host'] = self._logHost headers['Date'] = self._getGMT() if self._securityToken: headers["x-acs-security-token"] = self._securityToken signature = Util.get_request_authorization(method, resource, self._accessKey, params, headers) headers['Authorization'] = "LOG " + self._accessKeyId + ':' + signature headers['x-log-date'] = headers['Date'] # bypass some proxy doesn't allow "Date" in header issue. url = url + resource return self._sendRequest(method, url, params, body, headers, respons_body_type)
def _sendRequest(self, method, url, params, body, headers, respons_body_type='json'):
(resp_status, resp_body, resp_header) = self._getHttpResponse(method, url, params, body, headers)
header = {}
for key, value in resp_header.items():
header[key] = value
requestId = Util.h_v_td(header, 'x-log-requestid', '')
if resp_status==200:
if respons_body_type=='json':
exJson = self._loadJson(resp_status, resp_header, resp_body, requestId)
exJson = Util.convert_unicode_to_str(exJson)
return exJson, header
else:
return resp_body, header
exJson = self._loadJson(resp_status, resp_header, resp_body, requestId)
exJson = Util.convert_unicode_to_str(exJson)
if 'errorCode' in exJson and 'errorMessage' in exJson:
raise LogException(exJson['errorCode'], exJson['errorMessage'], requestId,
resp_status, resp_header, resp_body)
else:
exJson = '. Return json is ' + str(exJson) if exJson else '.'
raise LogException('LogRequestError',
'Request is failed. Http code is ' + str(resp_status) + exJson, requestId,
resp_status, resp_header, resp_body)
def _sendRequest(self, method, url, params, body, headers, respons_body_type = 'json'):
(status, respText, respHeader) = self._getHttpResponse(method, url, params, body, headers)
header = {}
for key, value in respHeader.items():
header[key] = value
requestId = header['x-log-requestid'] if 'x-log-requestid' in header else ''
exJson = None
header = Util.convert_unicode_to_str(header)
if status == 200 :
if respons_body_type == 'json' :
exJson = self._loadJson(respText, requestId)
#exJson = Util.convert_unicode_to_str(exJson)
return (exJson, header)
else :
return (respText, header)
exJson = self._loadJson(respText.encode('utf-8'), requestId)
exJson = Util.convert_unicode_to_str(exJson)
if 'errorCode' in exJson and 'errorMessage' in exJson:
raise LogException(exJson['errorCode'], exJson['errorMessage'], requestId)
else:
exJson = '. Return json is '+str(exJson) if exJson else '.'
raise LogException('LogRequestError',
'Request is failed. Http code is '+str(status)+exJson, requestId)
def _send(self, method, project, body, resource, params, headers, respons_body_type ='json'):
if body:
headers['Content-Length'] = len(body)
headers['Content-MD5'] = Util.cal_md5(body)
else:
headers['Content-Length'] = 0
headers["x-log-bodyrawsize"] = 0
headers['x-log-apiversion'] = API_VERSION
headers['x-log-signaturemethod'] = 'hmac-sha1'
url = ''
if self._isRowIp:
url = "http://" + self._endpoint
else:
url = "http://" + project + "." + self._endpoint
headers['Host'] = project + "." + self._logHost
headers['Date'] = self._getGMT()
if self._securityToken is not None:
headers["x-acs-security-token"] = self._securityToken
signature = Util.get_request_authorization(method, resource,
self._accessKey, params, headers)
headers['Authorization'] = "LOG " + self._accessKeyId + ':' + signature
url = url + resource
return self._sendRequest(method, url, params, body, headers, respons_body_type)
def _send(self,
method,
project,
body,
resource,
params,
headers,
respons_body_type='json'):
if body:
headers['Content-Length'] = len(body)
headers['Content-MD5'] = Util.cal_md5(body)
else:
headers['Content-Length'] = 0
headers["x-log-bodyrawsize"] = 0
headers['x-log-apiversion'] = API_VERSION
headers['x-log-signaturemethod'] = 'hmac-sha1'
url = ''
if self._isRowIp:
url = "http://" + self._endpoint
else:
url = "http://" + project + "." + self._endpoint
headers['Host'] = project + "." + self._logHost
headers['Date'] = self._getGMT()
if self._securityToken != None and self._securityToken != "":
headers["x-acs-security-token"] = self._securityToken
signature = Util.get_request_authorization(method, resource,
self._accessKey, params,
headers)
headers['Authorization'] = "LOG " + self._accessKeyId + ':' + signature
url = url + resource
return self._sendRequest(method, url, params, body, headers,
respons_body_type)
def from_json(self, json_value):
self.token_list = json_value["token"]
self.case_sensitive = bool(json_value["caseSensitive"])
self.include_keys = Util.get_json_value(json_value, "include_keys",
None)
self.exclude_keys = Util.get_json_value(json_value, "exclude_keys",
None)
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.next_cursor = Util.convert_unicode_to_str(header["x-log-cursor"])
self.log_count = int(header["x-log-count"])
self.loggroup_list = LogGroupList()
self._parse_loggroup_list(resp)
self.loggroup_list_json = None
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.next_cursor = Util.convert_unicode_to_str(header["x-log-cursor"])
self.log_count = int(header["x-log-count"])
self.loggroup_list = LogGroupList()
self._parse_loggroup_list(resp)
self.loggroup_list_json = None
def from_json(self, json_value) :
self.group_name = Util.get_json_value(json_value, "groupName")
self.group_type = Util.get_json_value(json_value, "groupType", "")
self.group_attribute = Util.get_json_value(json_value, "groupAttribute", {})
self.machine_type = Util.get_json_value(json_value, "machineIdentifyType")
self.machine_list = Util.get_json_value(json_value, "machineList")
self.create_time = Util.get_json_value(json_value, "crateTime")
self.last_modify_time = Util.get_json_value(json_value, "lastModifyTime")
def put_logs(self, request):
""" Put logs to log service.
Unsuccessful opertaion will cause an LogException.
:type request: PutLogsRequest
:param request: the PutLogs request parameters class
:return: PutLogsResponse
:raise: LogException
"""
if len(request.get_log_items()) > 4096:
raise LogException(
'InvalidLogSize',
"logItems' length exceeds maximum limitation: 4096 lines.")
logGroup = LogGroup()
logGroup.Topic = request.get_topic()
if request.get_source():
logGroup.Source = request.get_source()
else:
if self._source == '127.0.0.1':
self._source = Util.get_host_ip(request.get_project() + '.' +
self._logHost)
logGroup.Source = self._source
for logItem in request.get_log_items():
log = logGroup.Logs.add()
log.Time = logItem.get_time()
contents = logItem.get_contents()
for key, value in contents:
content = log.Contents.add()
content.Key = self.get_unicode(key)
content.Value = self.get_unicode(value)
body = logGroup.SerializeToString()
if len(body) > 3 * 1024 * 1024: # 3 MB
raise LogException(
'InvalidLogSize',
"logItems' size exceeds maximum limitation: 3 MB.")
headers = {}
headers['x-log-bodyrawsize'] = len(body)
headers['Content-Type'] = 'application/x-protobuf'
params = {}
logstore = request.get_logstore()
project = request.get_project()
resource = '/logstores/' + logstore
if request.get_hash_key() is not None:
resource = '/logstores/' + logstore + "/shards/route"
params["key"] = request.get_hash_key()
else:
resource = '/logstores/' + logstore + "/shards/lb"
respHeaders = self._send('POST', project, body, resource, params,
headers)
return PutLogsResponse(respHeaders[1])
def _sendRequest(self,
method,
url,
params,
body,
headers,
respons_body_type='json'):
(status, respText,
respHeader) = self._getHttpResponse(method, url, params, body,
headers)
header = {}
for key, value in respHeader.items():
header[key] = value
requestId = header[
'x-log-requestid'] if 'x-log-requestid' in header else ''
exJson = None
header = Util.convert_unicode_to_str(header)
if status == 200:
if respons_body_type == 'json':
exJson = self._loadJson(respText, requestId)
#exJson = Util.convert_unicode_to_str(exJson)
return (exJson, header)
else:
return (respText, header)
exJson = self._loadJson(respText.encode('utf-8'), requestId)
exJson = Util.convert_unicode_to_str(exJson)
if 'errorCode' in exJson and 'errorMessage' in exJson:
raise LogException(exJson['errorCode'], exJson['errorMessage'],
requestId)
else:
exJson = '. Return json is ' + str(exJson) if exJson else '.'
raise LogException(
'LogRequestError',
'Request is failed. Http code is ' + str(status) + exJson,
requestId)
def from_json(self, json_value):
self.group_name = Util.get_json_value(json_value, "groupName")
self.group_type = Util.get_json_value(json_value, "groupType", "")
self.group_attribute = Util.get_json_value(json_value,
"groupAttribute", {})
self.machine_type = Util.get_json_value(json_value,
"machineIdentifyType")
self.machine_list = Util.get_json_value(json_value, "machineList")
self.create_time = Util.get_json_value(json_value, "crateTime")
self.last_modify_time = Util.get_json_value(json_value,
"lastModifyTime")
def _setendpoint(self, endpoint):
pos = endpoint.find('://')
if pos != -1:
endpoint = endpoint[pos + 3:] # strip http://
pos = endpoint.find('/')
if pos != -1:
endpoint = endpoint[:pos]
pos = endpoint.find(':')
if pos != -1:
self._port = int(endpoint[pos + 1:])
endpoint = endpoint[:pos]
self._isRowIp = Util.is_row_ip(endpoint)
self._logHost = endpoint
self._endpoint = endpoint + ':' + str(self._port)
def _setendpoint(self, endpoint):
pos = endpoint.find('://')
if pos != -1:
endpoint = endpoint[pos + 3:] # strip http://
pos = endpoint.find('/')
if pos != -1:
endpoint = endpoint[:pos]
pos = endpoint.find(':')
if pos != -1:
self._port = int(endpoint[pos + 1:])
endpoint = endpoint[:pos]
self._isRowIp = Util.is_row_ip(endpoint)
self._logHost = endpoint
self._endpoint = endpoint + ':' + str(self._port)
def __init__(self, endpoint, accessKeyId, accessKey,securityToken = None):
if isinstance(endpoint, unicode): # ensure is ascii str
endpoint = endpoint.encode('ascii')
if isinstance(accessKeyId, unicode):
accessKeyId = accessKeyId.encode('ascii')
if isinstance(accessKey, unicode):
accessKey = accessKey.encode('ascii')
self._isRowIp = True
self._port = 80
self._setendpoint(endpoint)
self._accessKeyId = accessKeyId
self._accessKey = accessKey
self._timeout = CONNECTION_TIME_OUT
self._source = Util.get_host_ip(self._logHost)
self._securityToken = securityToken;
def __init__(self, endpoint, accessKeyId, accessKey, securityToken=None):
if isinstance(endpoint, unicode): # ensure is ascii str
endpoint = endpoint.encode('ascii')
if isinstance(accessKeyId, unicode):
accessKeyId = accessKeyId.encode('ascii')
if isinstance(accessKey, unicode):
accessKey = accessKey.encode('ascii')
self._isRowIp = True
self._port = 80
self._setendpoint(endpoint)
self._accessKeyId = accessKeyId
self._accessKey = accessKey
self._timeout = CONNECTION_TIME_OUT
self._source = Util.get_host_ip(self._logHost)
self._securityToken = securityToken
def __init__(self, kafka_endpoint, elasticsearch_endpoint, accessKeyId, accessKey, securityToken=None, source=None):
self._isRowIp = Util.is_row_ip(kafka_endpoint)
self._setendpoint(kafka_endpoint, elasticsearch_endpoint)
self._accessKeyId = accessKeyId
self._accessKey = accessKey
self._timeout = CONNECTION_TIME_OUT
if source is None:
self._source = Util.get_host_ip(self._logHost)
else:
self._source = source
self._securityToken = securityToken
self._user_agent = USER_AGENT
self.kafka_endpoint = kafka_endpoint
self.elasticsearch_endpoint = elasticsearch_endpoint
try:
self.kafka_admin_client = KafkaAdminClient(bootstrap_servers=kafka_endpoint, client_id='1')
self.kafka_producer = KafkaProducer(bootstrap_servers=kafka_endpoint)
except Exception as e:
sys.stderr.write('[LOG_LITE_ERROR] init kafka error \n')
raise e
self.project_logstore = {}
def from_json(self, json_value) :
self.ttl = json_value["ttl"]
if "all_keys" in json_value :
self.all_keys_config = IndexKeyConfig()
self.all_keys_config.from_json(json_value["all_keys"])
if "line" in json_value :
self.line_config = IndexLineConfig()
self.line_config.from_json(json_value["line"])
if "keys" in json_value :
self.key_config_list = {}
key_configs = json_value["keys"]
for key, value in key_configs.items():
key_config = IndexKeyConfig()
key_config.from_json(value)
self.key_config_list[key] = key_config
self.modify_time = Util.get_json_value(json_value, "lastModifyTime", int(time.time()))
def put_logs(self, request):
""" Put logs to log service.
Unsuccessful opertaion will cause an LogException.
:type request: PutLogsRequest
:param request: the PutLogs request parameters class
:return: PutLogsResponse
:raise: LogException
"""
if len(request.get_log_items()) > 4096:
raise LogException('InvalidLogSize',
"logItems' length exceeds maximum limitation: 4096 lines.")
logGroup = LogGroup()
logGroup.Topic = request.get_topic()
if request.get_source():
logGroup.Source = request.get_source()
else:
if self._source=='127.0.0.1':
self._source = Util.get_host_ip(request.get_project() + '.' + self._logHost)
logGroup.Source = self._source
for logItem in request.get_log_items():
log = logGroup.Logs.add()
log.Time = logItem.get_time()
contents = logItem.get_contents()
for key, value in contents:
content = log.Contents.add()
content.Key = self.get_unicode(key)
content.Value = self.get_unicode(value)
body = logGroup.SerializeToString()
if len(body) > 3 * 1024 * 1024: # 3 MB
raise LogException('InvalidLogSize',
"logItems' size exceeds maximum limitation: 3 MB.")
headers = {}
headers['x-log-bodyrawsize'] = len(body)
headers['Content-Type'] = 'application/x-protobuf'
params = {}
logstore = request.get_logstore()
project = request.get_project()
resource = '/logstores/' + logstore
respHeaders = self._send('POST', project, body, resource, params, headers)
return PutLogsResponse(respHeaders[1])
def from_json(self, json_value):
self.ttl = json_value["ttl"]
if "all_keys" in json_value:
self.all_keys_config = IndexKeyConfig()
self.all_keys_config.from_json(json_value["all_keys"])
if "line" in json_value:
self.line_config = IndexLineConfig()
self.line_config.from_json(json_value["line"])
if "keys" in json_value:
self.key_config_list = {}
key_configs = json_value["keys"]
for key, value in key_configs.items():
key_config = IndexKeyConfig()
key_config.from_json(value)
self.key_config_list[key] = key_config
self.modify_time = Util.get_json_value(json_value, "lastModifyTime",
int(time.time()))
def generate_apsara_log_config(json_value) :
input_detail = json_value['inputDetail']
output_detail = json_value['outputDetail']
config_name = json_value['configName']
logstore_name = output_detail['logstoreName']
endpoint = Util.get_json_value(output_detail,'endpoint')
log_path = input_detail['logPath']
file_pattern = input_detail['filePattern']
time_format = input_detail['timeFormat']
log_begin_regex = input_detail['logBeginRegex']
topic_format = input_detail['topicFormat']
filter_keys = input_detail['filterKey']
filter_keys_reg = input_detail['filterRegex']
config = ApsaraLogConfigDetail(config_name, logstore_name, endpoint, log_path, file_pattern,
log_begin_regex, topic_format, filter_keys, filter_keys_reg)
return config
def from_dict(cls, dict_data):
schema_list = Util.convert_unicode_to_str(
json.loads(dict_data.get("schema"))).get("schema")
schema_instance_list = []
if schema_list:
for schema in schema_list:
schema_instance_list.append(
ResourceSchemaItem(column=schema.get("column"),
schema_type=schema.get("type"),
ext_info=schema.get("ext_info")))
resource = Resource()
resource.set_resource_name(dict_data.get("name"))
resource.set_description(dict_data.get('description'))
resource.set_resource_type(dict_data.get("type"))
resource.set_ext_info(dict_data.get("extInfo"))
resource.set_acl(json.loads(dict_data.get("acl")))
resource.set_schema_list(schema_instance_list)
resource.set_create_time(dict_data.get("create_time"))
resource.set_last_modify_time(dict_data.get("last_modify_time"))
return resource
def maker(json_str):
args_list, option_arg_pos = _parse_method(cls.__init__)
if hasattr(cls, 'from_json'):
# there's a from json method, try to use it
try:
j = json.loads(json_str)
extjson = Util.convert_unicode_to_str(j)
if option_arg_pos == 0:
obj = cls()
new_obj = obj.from_json(extjson)
if new_obj is None:
return obj # expecting it's updated
else:
return new_obj # expecting return a new obj
else:
# expect the from_json is static method.
return cls.from_json(extjson)
except Exception as ex:
logger.warn("fail to load input via method from_json, try to call constructor for cls: "
+ str(cls) + "\n\tex:" + str(ex))
method_type = _parse_method_params_from_doc(cls.__doc__)
j = json.loads(json_str)
# verify if all mandantory args exists
for i, arg in enumerate(args_list):
if i >= option_arg_pos:
break
if arg not in j:
raise ValueError("args:{0} is missed".format(arg))
# verify if all inputs exists in parameters
for k in j:
if k not in args_list:
raise ValueError("args:{0} is unexpected".format(k))
# convert all values to expected type
converted_args = _convert_args(j, method_type)
return cls(**converted_args)
def maker(json_str):
args_list, option_arg_pos = _parse_method(cls.__init__)
if json_str.startswith('file://'):
with open(json_str[7:], "r") as f:
json_str = f.read()
if option_arg_pos == 0 and hasattr(cls, 'from_json'):
# there's a from json method, try to use it
try:
j = json.loads(json_str)
extjson = Util.convert_unicode_to_str(j)
obj = cls()
obj.from_json(extjson)
return obj
except Exception as ex:
print("** fail to load input via method from_json, try to call constructor for cls: "
+ str(cls) + "\nex:" + str(ex))
method_type = _parse_method_params_from_doc(cls.__doc__)
j = json.loads(json_str)
# verify if all mandantory args exists
for i, arg in enumerate(args_list):
if i >= option_arg_pos:
break
if arg not in j:
raise ValueError("args:{0} is missed".format(arg))
# verify if all inputs exists in parameters
for k in j:
if k not in args_list:
raise ValueError("args:{0} is unexpected".format(k))
# convert all values to expected type
converted_args = _convert_args(j, method_type)
return cls(**converted_args)
def generate_common_reg_log_config(json_value) :
input_detail = json_value['inputDetail']
output_detail = json_value['outputDetail']
config_name = json_value['configName']
logstore_name = output_detail['logstoreName']
endpoint = Util.get_json_value(output_detail,'endpoint')
log_path = input_detail['logPath']
file_pattern = input_detail['filePattern']
time_format = input_detail['timeFormat']
log_begin_regex = input_detail['logBeginRegex']
log_parse_regex = input_detail['regex']
reg_keys = input_detail['key']
topic_format = input_detail['topicFormat']
filter_keys = input_detail['filterKey']
filter_keys_reg = input_detail['filterRegex']
config = CommonRegLogConfigDetail(config_name, logstore_name, endpoint, log_path , file_pattern, time_format, log_begin_regex, log_parse_regex, reg_keys,
topic_format, filter_keys, filter_keys_reg)
return config
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.logstore_name = Util.convert_unicode_to_str(resp["logstoreName"])
self.ttl = int(resp["ttl"])
self.shard_count = int(resp["shardCount"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.count = int(resp["count"])
self.logstores = Util.convert_unicode_to_str(resp["logstores"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.cursor = Util.convert_unicode_to_str(resp["cursor"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.count = int(resp["count"])
self.total = int(resp["total"])
self.machine_groups = Util.convert_unicode_to_str(resp["machinegroups"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.cursor = Util.convert_unicode_to_str(resp["cursor"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.count = int(resp["count"])
self.count = int(resp["total"])
self.logtail_configs = Util.convert_unicode_to_str(resp["configs"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.count = int(resp["count"])
self.count = int(resp["total"])
self.logtail_configs = Util.convert_unicode_to_str(resp["configs"])
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.count = int(resp["count"])
self.logstores = Util.convert_unicode_to_str(resp["logstores"])
def from_json(self, json_value):
self.token_list = json_value["token"]
self.case_sensitive = bool(json_value["caseSensitive"])
self.include_keys = Util.get_json_value(json_value, "include_keys", None)
self.exclude_keys = Util.get_json_value(json_value, "exclude_keys", None)
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.shards = Util.convert_unicode_to_str(resp)
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.shards = Util.convert_unicode_to_str(resp)
def __init__(self, resp, header):
LogResponse.__init__(self, header)
self.logstore_name = Util.convert_unicode_to_str(resp["logstoreName"])
self.ttl = int(resp["ttl"])
self.shard_count = int(resp["shardCount"])
