def serverlist(self):
# default tcp port
tcp_port = self.config.value("common.umcrunner.http.tcp-port", 1989)
# create dict object
server_list = {}
server_binding = self.config.value(
"common.umcrunner.http.server-binding")
if server_binding is not None:
for hostname in server_binding.split(","):
hostname = hostname.strip()
try:
df = Map(
hostname=hostname,
address=socket.gethostbyname(hostname),
tcp_port=tcp_port,
enabled=True,
me=(True if hostname.lower()
== socket.gethostname().lower() else False)
# TODO: check that address is on this host = better "me"
)
server_list[hostname] = df
except Exception as e:
if "Name or service not known" not in str(e):
Msg.err_msg(
"Error occured when obtaining configuration for server's hostname '%s': %s!"
% (hostname, e))
pass
# // server_binding
return server_list
def include_constructor(loader, node):
filename = os.path.join(env_constructor(loader, node))
if not (os.path.exists(filename)):
Msg.err_msg("The include file %s does not exist!" % filename)
return []
else:
with open(filename, 'r') as f:
return yaml.load(f)
def env_constructor(loader, node):
value = node.value
params = list(set(re.findall("(%s)" % ENVPARAM_PATTERN, value)))
if len(params) > 0:
for k in params:
env_value = os.environ.get(k[2:-1])
if env_value is None:
Msg.err_msg(
"The environment variable %s used in the configuration file does not exist!"
% (k))
else:
value = value.replace(k, env_value)
return value
def eval_filter(filter, timestamp, tags, fields):
try:
for k, v in tags.items():
if v is not None:
exec(k + "=\"" + str(v) + "\"")
for k, v in fields.items():
if v is not None:
exec(k + "=" + str(v))
return eval(filter)
except Exception as e:
Msg.err_msg("Error when evaluating the filter '%s': %s!" %
(filter, str(e)))
return False
def eval_transform(transform_exprs, timestamp, tags, fields):
try:
# declare variables and assign values to them
for k, v in tags.items():
if v is not None:
exec(k + "=\"" + v + "\"")
for k, v in fields.items():
if v is not None:
exec(k + "=" + str(v))
# transform
for expr in transform_exprs:
try:
exec(expr)
except Exception as ex:
pass
Msg.info2_msg("Error when evaluating transformation '%s': %s" %
(expr, str(ex)))
# get only variables that come from tags and fiedls, remove all local ones
# the list in the below expression must contain all local variables in this function prior to this call!
nf = {
k: v
for k, v in locals().items() if k not in [
"k", "v", "umc_id", "transform_exprs", "timestamp", "tags",
"fields", "expr", "ex"
]
}
__t2 = {}
__f2 = {}
for k, v in nf.items():
if k in tags.keys():
exec("__t2['%s']=%s" % (k, k))
elif k in fields.keys():
exec("__f2['%s']=%s" % (k, k))
else:
exec("value=%s" % (k))
if isinstance(value, int) or isinstance(value, float):
exec("__f2['%s']=%s" % (k, k))
else:
exec("__t2['%s']=%s" % (k, k))
# new tag or field that resulted from transformation
# // for
return __t2, __f2
except Exception as e:
Msg.err_msg("Error when evaluating transformations for %s: %s" %
(umc_id, str(e)))
return tags, fields
def append_instance(umcinstances, e):
exists = False
for u in umcinstances:
if e["umc-id"] == u["umc-id"]:
exists = True
break
# // if
# // for
if not (exists):
umcinstances.append(e)
else:
Msg.err_msg(
"The umc instance with the same id=%s already exists!" %
(e["umc-id"]))
def read_umcdefs(self, reader, writer):
allinstances = self.get_umc_instances()
umcdefs = {}
for instance in allinstances:
umc_id = instance["umc-id"]
umcdef = Map(umcid=umc_id,
enabled=False,
writer=None,
reader=None,
instance=instance)
umcdef.enabled = self.value_element(instance, "enabled", False)
umcdef.writer = writer.read_umcdef(umc_id, instance)
umcdef.reader = reader.read_umcdef(umc_id, instance)
Msg.info1_msg("Definition retrieved for umc %s" % (umc_id))
if not (umcdef.enabled):
Msg.info1_msg(
"umc id %s is disabled by configuration, no datapoints will be read."
% (umc_id))
elif umcdef.writer is None or umcdef.reader is None:
Msg.info2_msg(
"umc id %s does not have reader or writer definitions and it will be disabled."
% (umc_id))
umcdef.enabled = False
# disable if the writer is not enabled
if not (umcdef.writer.enabled):
Msg.info2_msg(
"umc id %s is disabled as its writer is disabled. No data will be read for this umc id."
% (umc_id))
umcdef.enabled = False
if umcdefs.get(umc_id) is not None:
Msg.err_msg(
"There is a duplicate umc instance with id '%s' in the configuration file!"
% (umc_id))
else:
umcdefs[umc_id] = umcdef
# // for
return umcdefs
def write(self,datapoints,exit_event=None):
Msg.info2_msg("Uploading %d records to OMC..."%len(datapoints))
Msg.info2_msg("The batch contains entity types %s"%self.get_all_entity_types(datapoints))
#if datapoints is not None:
#print "========= BATCH OUTPUT START"
#print json.dumps(datapoints)
#print "========= BATCH OUTPUT END"
response = self.run_request('POST',self.omc_params.data_url, datapoints, 'application/octet-stream')
if response.status_code<300:
resp=json.loads(response.text)
status_uri=resp["statusUri"]
Msg.info2_msg("Upload request sent, waiting for the result at %s up to %s seconds..."%(status_uri,self.omc_params.omc_inprogress_timeout))
start_t=time.time()
while resp["status"]=="IN_PROGRESS" and (exit_event is not None and not(exit_event.is_set())):
response=self.run_request('GET',status_uri)
if response.status_code>=300:
raise Exception("OMC status request failed with status code %d"%response.status_code)
resp=json.loads(response.text)
if resp["status"]=="IN_PROGRESS":
# wait only certain number of seconds
if time.time()-start_t>self.omc_params.omc_inprogress_timeout:
Msg.err_msg("Upload failed, the datapoints in the batch will be discarded, they contain the following entity types: %s"%self.get_all_entity_types(datapoints))
raise Exception("OMC upload failed due to a timeout of %d seconds while waiting for OMC to confirm the data was uploaded successfully! The status response payload is %s"%(self.omc_params.omc_inprogress_timeout,resp))
# wait
exit_event.wait(1) if exit_event is not None else sleep(1)
# // while
if resp["status"]=="FAILED":
raise Exception("OMC upload reuqest failed. %s. Response payload: %s"
%(resp["errorMessage"],resp))
elif exit_event is None or not(exit_event.is_set()):
Msg.info2_msg("OMC upload reuqest processed in %d seconds. %s: %s. Response payload: %s"
%(time.time()-start_t,resp["status"],resp["errorMessage"],resp))
else:
raise Exception("OMC data upload request failed with status code %d. Response payload: %s"%(response.status_code,response.text))
def read_datapoints(self, logfilename, umcdef, create_writeitem_func):
datapoints = []
notags = False
nofields = False
tzoffset = self.params.tzoffset
if umcdef.enabled:
# read datapoints
with open(logfilename, 'r') as csvfile:
reader = csv.DictReader(csvfile, delimiter=',')
for row in reader:
# remove None keys
row = {k: v for k, v in row.items() if k is not None}
# timestamp
try:
if not (umcdef.reader.timefield in row):
raise ValueError("Cannot find time field '" +
umcdef.reader.timefield +
"' in data row!")
if umcdef.reader.timeformat == "_unix_" or umcdef.reader.timeformat == "_time_s_":
timestamp = long(
row[umcdef.reader.timefield]) * 1000000000
elif umcdef.reader.timeformat == "_time_ms_":
timestamp = long(
row[umcdef.reader.timefield]) * 1000000
else:
if umcdef.reader.tzfield is not None and umcdef.reader.tzfield in row:
tzoffset = utils.float_ex(
row[umcdef.reader.tzfield],
self.params.tzoffset)
timestamp = (self.unix_time_millis(
datetime.datetime.strptime(
row[umcdef.reader.timefield],
umcdef.reader.timeformat)) - int(
tzoffset * 60 * 60 * 1000)) * 1000000
except Exception as e:
# output error and skip this row
Msg.err_msg(
"Cannot read or convert time to timestamp for %s: %s"
% (umcdef.umcid, str(e)))
continue
# create tags and fields
tags = {
k: str(v)
for k, v in row.items() if k in umcdef.reader.tcols
}
fields = {
k: utils.float_ex(v)
for k, v in row.items() if k in umcdef.reader.fcols
}
notags = (len(tags) == 0)
# only add this row if there is at least one field with some value
if len([v
for k, v in fields.items() if v is not None]) > 0:
# evaluate transformations
if umcdef.reader.transform is not None:
tags, fields = eval_transform(
umcdef.reader.transform, timestamp, tags,
fields)
# only add this row if filter holds on this row or there is no filter
if umcdef.reader.filter is None or eval_filter(
umcdef.reader.filter, timestamp, tags, fields):
try:
records = create_writeitem_func(
umcdef, timestamp, fields, tags)
if records is not None and isinstance(
records, list):
datapoints += records
except Exception as e:
Msg.err_msg(
"Error occured while creating data points item: %s"
% str(e))
# // if write data
# // end reading rows
# // end open file
# check for no tags
if notags and len(datapoints) > 0:
Msg.warn_msg(
"The definition of %s contains no tags presented in the log file %s!"
% (umcdef.umcid, os.path.basename(logfilename)))
return datapoints