def collect_data(self):
"""Collects all data."""
if self.is_config_updated:
data = run_application()
data = run_application_elastic(index=0)
collectd.info("oozie workflow collection successful")
docs = [{
"wfId": 0,
"wfaId": 0,
"wfName": 0,
"wfaName": 0,
"time": int(math.floor(time.time())),
"jobId": 0,
'timePeriodStart': 0,
'timePeriodEnd': 0,
"mapTaskCount": 0,
"reduceTaskCount": 0,
'duration': 0,
"_plugin": plugin_name['oozie'],
"_documentType": "taskCounts",
"_tag_appName": tag_app_name['oozie']
}]
for doc in docs:
self.add_common_params(doc, doc['_documentType'])
write_json.write(doc)
def dispatch(data_dict):
"""Dispatches data to collectd."""
# add hostname to data_dict
data_dict[HOSTNAME] = gethostname()
internal_plugin_name = data_dict[ACTUALPLUGINTYPE]
# first dispatch to write json
# write json also cleans up internal plugin names from dictionary.
#
write_json.write(data_dict)
# dispatch to other write functions
metric = collectd.Values()
metric.plugin = internal_plugin_name
if PLUGIN_INS in data_dict:
metric.plugin_instance = data_dict[PLUGIN_INS]
if VAL_TYPE in data_dict:
metric.type = data_dict[VAL_TYPE]
else:
metric.type = DUMMY
if VAL_INS in data_dict:
metric.type_instance = data_dict[VAL_INS]
metric.meta = data_dict
metric.values = [DUMMY_VAL]
metric.dispatch()
def collect_data(self):
"""Collects all data."""
if self.is_config_updated:
data = run_application(index=0)
docs = [{
"nameNodeStatsRpcActivity": "_tag_hostname: manager-1",
"_tag_context": "rpc",
"RpcQueueTimeAvgTime": 0.066667,
"RpcProcessingTimeAvgTime": 0.244444,
"SentBytes": 0,
"RpcAuthenticationSuccesses": 0,
"_tag_numopenconnectionsperuser": 0,
"modelerType": "RpcActivityForPort8020",
"RpcClientBackoff": 0,
"RpcAuthenticationFailures": 0,
"_tag_port": 8020,
"_documentType": "nameNodeStatsRpcActivity",
"_plugin": "namenode",
"RpcSlowCalls": 0,
"ReceivedBytes": 0,
"RpcAuthorizationFailures": 0,
"NumOpenConnections": 0,
"RpcAuthorizationSuccesses": 0,
"name": "Hadoop:service=NameNode,name=RpcActivityForPort8020",
"RpcProcessingTimeNumOps": 0,
"RpcQueueTimeNumOps": 0,
"time": 0,
"CallQueueLength": 0
}]
for doc in docs:
self.add_common_params(doc, doc['_documentType'])
write_json.write(doc)
def collect_data(self):
"""Collects all data."""
if self.is_config_updated:
data = run_application(0)
docs = [{
"NumRebootedNMs": 0,
"_documentType": "yarnStatsClusterMetrics",
"NumDecommissionedNMs": 0,
"name": "Hadoop:service=ResourceManager,name=ClusterMetrics",
"AMLaunchDelayNumOps": 0,
"_tag_context": "yarn",
"AMRegisterDelayNumOps": 0,
"_tag_clustermetrics": "ResourceManager",
"modelerType": "ClusterMetrics",
"NumLostNMs": 0,
"time": 1543301379,
"_tag_appName": "hadoopapp1",
"NumUnhealthyNMs": 0,
"AMRegisterDelayAvgTime": 0,
"NumActiveNMs": 0,
"AMLaunchDelayAvgTime": 0
}]
for doc in docs:
self.add_common_params(doc, doc['_documentType'])
write_json.write(doc)
def collect_data(self):
"""Collects all data."""
if self.is_config_updated:
data = run_application(index=0)
docs = [{
'_documentType': "sparkTaskCounts",
'appName': 0,
'appId': 0,
'appAttemptId': 0,
'stageAttemptId': 0,
'stageId': 0,
'time': 0,
'timePeriodStart': 0,
'timePeriodEnd': 0,
'duration': 0,
'taskCount': 0
}]
for doc in docs:
self.add_common_params(doc, doc['_documentType'])
write_json.write(doc)
payload = search_delivery_ids.default_payload
payload['combine_categories'] = [combine_category]
delivery_ids_obj = search_delivery_ids.crawl(payload)
delivery_ids = delivery_ids_obj['reply']['delivery_ids']
append_unique(total_delivery_ids, delivery_ids)
promotion_ids_obj = get_promotion_ids.crawl()
promotion_ids = promotion_ids_obj['reply']['promotion_ids']
append_unique(total_delivery_ids, promotion_ids)
print('Total request ids:', len(total_delivery_ids))
data = []
count = 0
for request_id in total_delivery_ids:
detail = get_detail.crawl(request_id)
dishes = get_delivery_dishes.crawl(request_id)
request_data = {
'request_id': request_id,
'get_detail': detail,
'get_delivery_dishes': dishes
}
json_name = f'request_id-{request_id}'
write_json.write(request_data, json_name)
data.append(request_data)
count += 1
print('count:', count)