def main():
config = configparser.ConfigParser()
config.read(CRED_FILE)
subscription_id = str(config['DEFAULT']['AZURE_SUBSCRIPTION_ID'])
appinsights_name = str(config['DEFAULT']['APPINSIGHTS_NAME'])
resource_group = config['DEFAULT']['RG_NAME']
credentials = ServicePrincipalCredentials(
client_id=config['DEFAULT']['azure_client_id'],
secret=config['DEFAULT']['azure_client_secret'],
tenant=config['DEFAULT']['azure_tenant_id'])
try:
resource_client = ResourceManagementClient(credentials,
subscription_id)
except Exception as e:
self.logger.error("Getting Azure Infra handlers failed %s" % str(e))
raise e
inst_key = get_appinsights_instr_key(resource_group, appinsights_name,
credentials, subscription_id)
tc = TelemetryClient(inst_key)
#logger1.info("[INFO]: Instrumentation key used {}".format(inst_key))
for metric in metric_list:
logger1.info("[INFO]: Publishing metrics {}".format(metric))
tc.track_metric(metric, 0)
tc.flush()
time.sleep(2)
def test_track_metric_works_as_expected(self):
def process(data, context):
data.properties["NEW_PROP"] = "MYPROP"
context.user.id = "BOTUSER"
context.session.id = "BOTSESSION"
return True
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient(
'99999999-9999-9999-9999-999999999999',
channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.track_metric('metric', 42,
channel.contracts.DataPointType.aggregation, 13, 1,
123, 111, {'foo': 'bar'})
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Metric", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "MetricData", "baseData": {"ver": 2, "metrics": [{"name": "metric", "kind": 1, "value": 42, "count": 13, "min": 1, "max": 123, "stdDev": 111}], "properties": {"NEW_PROP": "MYPROP", "foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags[
'ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def main():
inst_key = sys.argv[1].strip()
tc = TelemetryClient(inst_key)
logger1.info("[INFO]: Instrumentation key used {}".format(inst_key))
for metric in metric_list:
logger1.info("[INFO]: Publishing metrics {}".format(metric))
tc.track_metric(metric, 0)
tc.flush()
time.sleep(30)
def test_track_metric_works_as_expected(self):
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.context.device = None
client.track_metric('metric', 42, channel.contracts.DataPointType.aggregation, 13, 1, 123, 111, {'foo': 'bar'})
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Metric", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "MetricData", "baseData": {"ver": 2, "metrics": [{"name": "metric", "kind": 1, "value": 42, "count": 13, "min": 1, "max": 123, "stdDev": 111}], "properties": {"foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
def test_track_metric_works_as_expected(self):
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.context.device = None
client.track_metric('metric', 42, channel.contracts.DataPointType.aggregation, 13, 1, 123, 111, {'foo': 'bar'})
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Metric", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER"}, "data": {"baseType": "MetricData", "baseData": {"ver": 2, "metrics": [{"name": "metric", "kind": 1, "value": 42, "count": 13, "min": 1, "max": 123, "stdDev": 111}], "properties": {"foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
class AppInsightsClient:
def __init__(self):
self._client = None
def init_app(self, key, max_queue_length=0):
if not key or len(key) == 0:
return
self._client = TelemetryClient(key)
self._client.channel.queue.max_queue_length = max_queue_length
def track_metric(self, name, value, properties=None):
if not self._client:
return
self._client.track_metric(name, value, properties=properties)
class MetricsOutput(Metrics):
"""Send metrics data to app insights
"""
def __init__(self, ctx, config=None):
super(MetricsOutput, self).__init__(ctx, config)
self.namespace = self.ctx.policy.name
self.tc = None
def _initialize(self):
if self.tc is not None:
return
self.instrumentation_key = AppInsightsHelper.get_instrumentation_key(
self.config['url'])
self.tc = TelemetryClient(self.instrumentation_key)
self.subscription_id = local_session(
self.ctx.policy.session_factory).get_subscription_id()
def _format_metric(self, key, value, unit, dimensions):
self._initialize()
d = {
'Name': key,
'Value': value,
'Dimensions': {
'Policy': self.ctx.policy.name,
'ResType': self.ctx.policy.resource_type,
'SubscriptionId': self.subscription_id,
'ExecutionId': self.ctx.execution_id,
'ExecutionMode': self.ctx.policy.execution_mode,
'Unit': unit
}
}
for k, v in dimensions.items():
d['Dimensions'][k] = v
return d
def _put_metrics(self, ns, metrics):
self._initialize()
for m in metrics:
self.tc.track_metric(name=m['Name'],
value=m['Value'],
properties=m['Dimensions'])
self.tc.flush()
class AppInsightsReporter(object):
def __init__(self,
appinsightskey,
monitor_list,
upload_interval_seconds=60,
quiet=False):
self.monitors = monitor_list
self.telemetry = []
self.telemetry_client = TelemetryClient(appinsightskey)
self.upload_interval_seconds = upload_interval_seconds
self.quiet = quiet
def run(self):
while (True):
self.collect_metrics()
self.upload_metrics()
self.purge_metrics()
time.sleep(self.upload_interval_seconds)
def collect_metrics(self):
for monitor in self.monitors:
self.telemetry.extend(monitor.get_telemetry())
def upload_metrics(self):
# Can just upload our telemetry schema since it matches MS schema
if not self.quiet:
print("Uploading telemetry...")
for datapoint in self.telemetry:
self.telemetry_client.track_metric(datapoint.name, datapoint.value)
self.telemetry_client.flush()
if not self.quiet:
print("Upload complete")
def purge_metrics(self):
self.telemetry = []
def test_track_metric_works_as_expected(self):
def process(data, context):
data.properties["NEW_PROP"] = "MYPROP"
context.user.id = "BOTUSER"
context.session.id = "BOTSESSION"
return True
sender = MockTelemetrySender()
queue = channel.SynchronousQueue(sender)
client = TelemetryClient('99999999-9999-9999-9999-999999999999', channel.TelemetryChannel(context=None, queue=queue))
client.add_telemetry_processor(process)
client.context.device = None
client.track_metric('metric', 42, channel.contracts.DataPointType.aggregation, 13, 1, 123, 111, {'foo': 'bar'})
client.flush()
expected = '{"ver": 1, "name": "Microsoft.ApplicationInsights.Metric", "time": "TIME_PLACEHOLDER", "sampleRate": 100.0, "iKey": "99999999-9999-9999-9999-999999999999", "tags": {"ai.device.id": "DEVICE_ID_PLACEHOLDER", "ai.device.locale": "DEVICE_LOCALE_PLACEHOLDER", "ai.device.osVersion": "DEVICE_OS_VERSION_PLACEHOLDER", "ai.device.type": "DEVICE_TYPE_PLACEHOLDER", "ai.internal.sdkVersion": "SDK_VERSION_PLACEHOLDER", "ai.session.id": "BOTSESSION", "ai.user.id": "BOTUSER"}, "data": {"baseType": "MetricData", "baseData": {"ver": 2, "metrics": [{"name": "metric", "kind": 1, "value": 42, "count": 13, "min": 1, "max": 123, "stdDev": 111}], "properties": {"NEW_PROP": "MYPROP", "foo": "bar"}}}}'
sender.data.time = 'TIME_PLACEHOLDER'
sender.data.tags['ai.internal.sdkVersion'] = 'SDK_VERSION_PLACEHOLDER'
sender.data.tags['ai.device.id'] = "DEVICE_ID_PLACEHOLDER"
sender.data.tags['ai.device.locale'] = "DEVICE_LOCALE_PLACEHOLDER"
sender.data.tags['ai.device.osVersion'] = "DEVICE_OS_VERSION_PLACEHOLDER"
sender.data.tags['ai.device.type'] = "DEVICE_TYPE_PLACEHOLDER"
actual = json.dumps(sender.data.write())
self.maxDiff = None
self.assertEqual(expected, actual)
test_id = os.environ.get("TEST_ID", str(uuid.uuid4()))
instrumentation_key = os.environ.get("APPINSIGHTS_INSTRUMENTATIONKEY")
telemetry_client = None
if instrumentation_key:
telemetry_client = TelemetryClient(instrumentation_key)
print("Test run for '{}' started.".format(test_id))
queries_total = int(os.environ.get("QUERIES_TOTAL", -1))
queries_executed = 0
while queries_executed < queries_total or queries_total < 0:
raw_query = get_query()
print("\nTest '{}' executing #{}:\n{}\n".format(test_id, queries_executed,
raw_query))
t = timeit.Timer(functools.partial(cursor.execute, raw_query))
query_time = t.timeit(number=1)
print("Query took: {:.2f} seconds".format(query_time))
queries_executed += 1
if telemetry_client:
telemetry_client.track_metric("query_time",
query_time,
properties={"test_id": test_id})
telemetry_client.flush()
print("Test run for '{}' ended.".format(test_id))
class AI4EAppInsights(object):
def __init__(self):
self.grantee_key = None
raw_key = getenv(CONF_KEY_GRANTEE, None)
if (raw_key and len(raw_key.strip()) > 0):
self.grantee_key = raw_key.strip()
if (self.grantee_key):
self.sender = AsynchronousSender()
self.r_queue = AsynchronousQueue(self.sender)
self.r_context = AI4ETelemetryContext()
self.r_channel = TelemetryChannel(self.r_context, self.r_queue)
self.appinsights_grantee_client = TelemetryClient(
getenv(CONF_KEY_GRANTEE), self.r_channel)
self.appinsights_ai4e_client = None
if (getenv(CONF_KEY_AI4E)):
self.appinsights_ai4e_client = TelemetryClient(
getenv(CONF_KEY_AI4E), self.r_channel)
def _log(self, message, sev, taskId=None, additionalProperties=None):
if (self.grantee_key):
if (taskId):
if (additionalProperties is None):
additionalProperties = {'task_id': taskId}
else:
additionalProperties['task_id'] = taskId
self.appinsights_grantee_client.track_trace(
message, severity=sev, properties=additionalProperties)
self.appinsights_grantee_client.flush()
if (self.appinsights_ai4e_client):
self.appinsights_ai4e_client.track_trace(
message, severity=sev, properties=additionalProperties)
self.appinsights_ai4e_client.flush()
def log_debug(self, message, taskId=None, additionalProperties=None):
self._log(message, "DEBUG", taskId, additionalProperties)
def log_info(self, message, taskId=None, additionalProperties=None):
self._log(message, "INFO", taskId, additionalProperties)
def log_error(self, message, taskId=None, additionalProperties=None):
self._log(message, "ERROR", taskId, additionalProperties)
def log_warn(self, message, taskId=None, additionalProperties=None):
self._log(message, "WARNING", taskId, additionalProperties)
def log_exception(self, message, taskId=None, additionalProperties=None):
self._log(message, "CRITICAL", taskId, additionalProperties)
def track_metric(self, metric_name, metric_value):
if (self.grantee_key):
print("Tracking metric:" + metric_name + ", Value: " +
str(metric_value))
self.appinsights_grantee_client.track_metric(
metric_name, metric_value)
self.appinsights_grantee_client.flush()
if (self.appinsights_ai4e_client):
self.appinsights_ai4e_client.track_metric(
metric_name, metric_value)
self.appinsights_ai4e_client.flush()
def main():
command = 'az login --service-principal -u ' + sys.argv[
1] + ' -p ' + sys.argv[2] + ' --tenant ' + sys.argv[3]
logger1.info("[INFO]: Logging in {}".format(command))
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
proc_stdout = process.communicate()[0].strip()
#y = json.loads(proc_stdout)
logger1.info("[INFO]: output of az login {}".format(proc_stdout))
command = 'az resource show -g ' + sys.argv[
7] + ' --resource-type microsoft.insights/components -n ' + sys.argv[
6] + ' --query properties.InstrumentationKey -o tsv'
logger1.info("[INFO]: Show resources {}".format(command))
#inst_key = subprocess.check_output(shlex.split(command)).rstrip()
process = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
inst_key = process.communicate()[0].strip()
logger1.info("[INFO]: output of az resource show {}".format(inst_key))
logger1.info("[INFO]: publishing metrics {}".format(metric_list))
tc = TelemetryClient(inst_key.rstrip())
tc.track_metric('DataPlaneCPUUtilizationPct', 0)
tc.flush()
tc.track_metric('DataPlaneCPUUtilizationPct', 0)
tc.flush()
time.sleep(10)
tc.track_metric('panGPGatewayUtilizationPct', 0)
tc.flush()
tc.track_metric('panGPGatewayUtilizationPct', 0)
tc.flush()
time.sleep(10)
tc.track_metric('panGPGWUtilizationActiveTunnels', 0)
tc.flush()
tc.track_metric('panGPGWUtilizationActiveTunnels', 0)
tc.flush()
time.sleep(10)
tc.track_metric('DataPlanePacketBufferUtilization', 0)
tc.flush()
tc.track_metric('DataPlanePacketBufferUtilization', 0)
tc.flush()
time.sleep(10)
tc.track_metric('panSessionActive', 0)
tc.flush()
tc.track_metric('panSessionActive', 0)
tc.flush()
time.sleep(10)
tc.track_metric('panSessionSslProxyUtilization', 0)
tc.flush()
tc.track_metric('panSessionSslProxyUtilization', 0)
tc.flush()
time.sleep(10)
tc.track_metric('panSessionUtilization', 0)
tc.flush()
tc.track_metric('panSessionUtilization', 0)
tc.flush()
time.sleep(10)
class AppinsightsBotTelemetryClient(BotTelemetryClient):
def __init__(self, instrumentation_key: str):
self._instrumentation_key = instrumentation_key
self._context = TelemetryContext()
context.instrumentation_key = self._instrumentation_key
# context.user.id = 'BOTID' # telemetry_channel.context.session.
# context.session.id = 'BOTSESSION'
# set up channel with context
self._channel = TelemetryChannel(context)
# telemetry_channel.context.properties['my_property'] = 'my_value'
self._client = TelemetryClient(self._instrumentation_key,
self._channel)
def track_pageview(self,
name: str,
url: str,
duration: int = 0,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about the page viewed in the application (a web page for instance).
:param name: the name of the page that was viewed.
:param url: the URL of the page that was viewed.
:param duration: the duration of the page view in milliseconds. (defaults to: 0)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_pageview(name, url, duration, properties,
measurements)
def track_exception(self,
type_exception: type = None,
value: Exception = None,
tb: traceback = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about a single exception that occurred in the application.
:param type_exception: the type of the exception that was thrown.
:param value: the exception that the client wants to send.
:param tb: the traceback information as returned by :func:`sys.exc_info`.
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_exception(type_exception, value, tb, properties,
measurements)
def track_event(self,
name: str,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about a single event that has occurred in the context of the application.
:param name: the data to associate to this event.
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_event(name, properties, measurements)
def track_metric(self,
name: str,
value: float,
type: TelemetryDataPointType = None,
count: int = None,
min: float = None,
max: float = None,
std_dev: float = None,
properties: Dict[str, object] = None) -> NotImplemented:
"""
Send information about a single metric data point that was captured for the application.
:param name: The name of the metric that was captured.
:param value: The value of the metric that was captured.
:param type: The type of the metric. (defaults to: TelemetryDataPointType.aggregation`)
:param count: the number of metrics that were aggregated into this data point. (defaults to: None)
:param min: the minimum of all metrics collected that were aggregated into this data point. (defaults to: None)
:param max: the maximum of all metrics collected that were aggregated into this data point. (defaults to: None)
:param std_dev: the standard deviation of all metrics collected that were aggregated into this data point. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
"""
self._client.track_metric(name, value, type, count, min, max, std_dev,
properties)
def track_trace(self,
name: str,
properties: Dict[str, object] = None,
severity=None):
"""
Sends a single trace statement.
:param name: the trace statement.\n
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)\n
:param severity: the severity level of this trace, one of DEBUG, INFO, WARNING, ERROR, CRITICAL
"""
self._client.track_trace(name, properties, severity)
def track_request(self,
name: str,
url: str,
success: bool,
start_time: str = None,
duration: int = None,
response_code: str = None,
http_method: str = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None,
request_id: str = None):
"""
Sends a single request that was captured for the application.
:param name: The name for this request. All requests with the same name will be grouped together.
:param url: The actual URL for this request (to show in individual request instances).
:param success: True if the request ended in success, False otherwise.
:param start_time: the start time of the request. The value should look the same as the one returned by :func:`datetime.isoformat()` (defaults to: None)
:param duration: the number of milliseconds that this request lasted. (defaults to: None)
:param response_code: the response code that this request returned. (defaults to: None)
:param http_method: the HTTP method that triggered this request. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
:param request_id: the id for this request. If None, a new uuid will be generated. (defaults to: None)
"""
self._client.track_request(name, url, success, start_time, duration,
response_code, http_method, properties,
measurements, request_id)
def track_dependency(self,
name: str,
data: str,
type: str = None,
target: str = None,
duration: int = None,
success: bool = None,
result_code: str = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None,
dependency_id: str = None):
"""
Sends a single dependency telemetry that was captured for the application.
:param name: the name of the command initiated with this dependency call. Low cardinality value. Examples are stored procedure name and URL path template.
:param data: the command initiated by this dependency call. Examples are SQL statement and HTTP URL with all query parameters.
:param type: the dependency type name. Low cardinality value for logical grouping of dependencies and interpretation of other fields like commandName and resultCode. Examples are SQL, Azure table, and HTTP. (default to: None)
:param target: the target site of a dependency call. Examples are server name, host address. (default to: None)
:param duration: the number of milliseconds that this dependency call lasted. (defaults to: None)
:param success: true if the dependency call ended in success, false otherwise. (defaults to: None)
:param result_code: the result code of a dependency call. Examples are SQL error code and HTTP status code. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
:param id: the id for this dependency call. If None, a new uuid will be generated. (defaults to: None)
"""
self._client.track_dependency(name, data, type, target, duration,
success, result_code, properties,
measurements, dependency_id)
class Telemetry(metaclass=Singleton):
"""Singleton class that handles telemetry sending to AppInsights."""
def __init__(self, toggle):
"""Initialize Telemetry instance."""
self._toggle = toggle
if self._toggle:
self._telemetry_client = TelemetryClient(APP_INSIGHTS_KEY)
self._telemetry_channel = self._setup_telemetry_channel()
print("Telemetry enabled.")
else:
self._telemetry_client = None
self._telemetry_channel = None
print("Telemetry disabled.")
def track_event(self, name, properties=None, measurements=None):
"""Track a telemetry event."""
try:
self._telemetry_client.track_event(name, properties, measurements)
except AttributeError:
print(f"Telemetry Disabled: Event Name: {name}")
print(f"properties: {properties}")
print(f"measurements: {measurements}")
def track_metric(self,
name,
value,
type=None,
count=None,
min=None,
max=None,
std_dev=None,
properties=None):
"""Track a telemetry metric."""
try:
self._telemetry_client.track_metric(name, value, type, count, min,
max, std_dev, properties)
except AttributeError:
print(f"Telemetry Disabled: Metric Name: {name}")
print(f"value: {value}")
if type:
print(f"type: {type}")
if count:
print(f"count: {count}")
if min:
print(f"min: {min}")
if max:
print(f"max: {max}")
if std_dev:
print(f"std_dev: {std_dev}")
if properties:
print(f"properties: {properties}")
def flush(self):
"""Flush the telemetry client info to AppInsights."""
try:
self._telemetry_client.flush()
except AttributeError:
pass
def _setup_telemetry_channel(self):
"""Create telemetry_channel object.
Instantiates a telemetry channel that collects unhandled exceptions.
Return:
telemetry_channel
"""
from applicationinsights.exceptions import enable
from applicationinsights import channel
# set up channel with context
telemetry_channel = channel.TelemetryChannel()
telemetry_channel.context.application.ver = get_version()
# set up exception capture
telemetry_channel.context.properties['capture'] = 'exceptions'
enable(APP_INSIGHTS_KEY, telemetry_channel=telemetry_channel)
return telemetry_channel
def merge_rename_core_columns_CSV(vm_uuid, deploy_uuid, config_uuid,
schema_ver, inject_ver, container_name,
filesrootfolder, fileoutputfolder,
process_id):
#block_blob_service = BlockBlobService(account_name=SOURCE_CSV_BLOB_ACCOUNT, sas_token=SOURCE_CSV_BLOB_TOKEN)
block_blob_service = BlockBlobService(account_name=SOURCE_CSV_BLOB_ACCOUNT,
account_key=SOURCE_CSV_BLOB_KEY)
tc = TelemetryClient('')
print("Start merge CSV ", vm_uuid, ' ', deploy_uuid, ' ', config_uuid)
blobs = []
marker = None
while True:
batch = block_blob_service.list_blobs(container_name,
prefix=filesrootfolder)
blobs.extend(batch)
if not batch.next_marker:
break
marker = batch.next_marker
i = 0
blobpaths = []
for blob in blobs:
blobpaths.append(blob.name)
matchers = ['.csv']
matching = [s for s in blobpaths if any(xs in s for xs in matchers)]
mergelog = {}
mergelog["vm_uuid"] = vm_uuid
mergelog["process_type"] = "MERGE_METRIC_CSV"
mergelog["DOC_TYPE"] = "MERGE_METRIC_FILES_LOG"
mergelog["file_folder"] = filesrootfolder
mergelog["process_time"] = time.time()
mergelog["files"] = []
mergelog["defect_files"] = []
a_mergelog = copy.deepcopy(mergelog)
dfagg = pd.DataFrame(columns=[])
mixagg = AGGREGATION_FILES_NUM
aggcount = 0
aggcount_total = 0
aggoutcount = 0
aggsize = 0
error_files = []
merged_files = []
totoal_rows = 0
alldfs = []
outfilenamebase = fileoutputfolder + filesrootfolder + "_aggr_"
t1 = time.time()
#print (outfilenamebase)
source_col = ['']
target_col = ['']
tc.track_trace('Prepare to process ' + str(len(matching)) +
' Metric CSV files ')
tc.flush()
for fname in matching:
#print(aggcount)
head, tail = os.path.split(fname)
aggcount += 1
aggcount_total += 1
blobstring = block_blob_service.get_blob_to_text(
container_name, fname).content
aggsize += len(blobstring)
#print('Prepare to merge '+str(aggcount_total)+' / '+str(len(matching)) +' Memeory '+str(aggsize)+' File Name: '+tail)
#tc.track_trace('Prepare to merge '+tail)
#tc.flush()
try: # Rread CSV And Try Processing
dfone = pd.read_csv(StringIO(blobstring))
dfAll_cols = dfone.columns
#colname0=dfAll_cols
dfAll_newcols = []
pc_name = re.search(r'(\\{2}.*\\)(.*\\)', dfAll_cols[1]).group(1)
for col in dfAll_cols:
dfAll_newcols.append(
col.replace(pc_name, '').replace('`', '').replace(
'\\', '').replace(' ', '').replace('/', '').replace(
'.', '').replace('-', '').replace('%', '').replace(
'(', '').replace(')', ''))
dfAll_newcols[0] = "Universal_datetime"
# Rename all columns
dfone.columns = dfAll_newcols
alldfs.append(dfone)
a_mergelog['files'].append(tail)
#if (aggcount>=mixagg) or (aggcount_total==len(matching)):
if (aggsize > MAX_FILESIZE) or (aggcount_total == len(matching)):
if (aggcount_total == len(matching)):
print("Processing Final File")
tc.track_trace('Processing Final File')
tc.flush()
alldfs.append(pd.DataFrame(columns=source_col))
dfagg = pd.concat(alldfs, ignore_index=True)
dfagg_out = dfagg[source_col]
dfagg_out.columns = target_col
dfagg_out['schema_ver'] = schema_ver
dfagg_out['inject_ver'] = inject_ver
output = dfagg_out.to_csv(index=False, encoding="utf-8")
outfile = outfilenamebase + str(aggoutcount) + ".csv"
block_blob_service.create_blob_from_text(
container_name, outfile, output)
print(
"Output aggregated file to " + container_name,
outfile + " Data Shape " + str(dfagg.shape) + ' uuid: ' +
str(vm_uuid) + str(deploy_uuid) + str(config_uuid))
totoal_rows += dfagg_out.shape[0]
merged_files.append(outfile)
a_mergelog['output_file'] = outfile
a_mergelog['merged_files_num'] = len(a_mergelog['files'])
a_mergelog['defect_files_num'] = len(
a_mergelog['defect_files'])
# Insert Process Log to COSMOS DB
insert_json_cosmos(a_mergelog)
a_mergelog = copy.deepcopy(mergelog)
t2 = time.time()
print(("It takes %s seconds to merge " + str(aggcount) +
" CSV Metrics") % (t2 - t1))
aggoutcount += 1
aggcount = 0
aggsize = 0
alldfs = []
t1 = time.time()
file_size = BlockBlobService.get_blob_properties(
block_blob_service, container_name,
outfile).properties.content_length
print(outfile + " File Size " + str(file_size))
# Ingest to AXX
ingest_to_ADX(outfile, file_size)
except Exception as e:
print('Error While process ' + fname)
error_class = e.__class__.__name__
detail = e.args[0]
cl, exc, tb = sys.exc_info()
lastCallStack = traceback.extract_tb(tb)[-1]
fileName = lastCallStack[0]
lineNum = lastCallStack[1]
funcName = lastCallStack[2]
errMsg = "File \"{}\", line {}, in {}: [{}] {}".format(
fileName, lineNum, funcName, error_class, detail)
print("Unexpected error:", sys.exc_info()[0])
traceback.print_exc()
msg = errMsg + traceback.format_exc()
tc = TelemetryClient('')
tc.context.application.ver = '1.0'
tc.context.properties["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a"
tc.context.properties["DATA_FOLDER"] = metricspath
tc.track_trace(msg)
tc.flush()
# print("Unexpected error:", sys.exc_info()[0])
a_mergelog["defect_files"].append(tail)
error_files.append(fname) # Add No-Well Formed JSON to error file
print('Total Rows ' + str(totoal_rows))
tc.track_trace('Proccessed Rows: ' + str(totoal_rows))
tc.track_metric('BATHCH_INGEST_METRIC_CSV_TOTAL_ROWS', str(totoal_rows))
tc.flush()
return error_files, merged_files, matching
def process(filesrootfolder, forceinsert):
# Create process id as identify of this process
process_id = time.time()
tc = TelemetryClient('')
tc.context.application.ver = '1.0'
tc.context.properties["PROCESS_PROGRAM"] = "BATCH_CSV_V001a"
tc.context.properties["PROCESS_START"] = time.time()
tc.context.properties["DATA_FOLDER"] = filesrootfolder
tc.context.properties["PROCESS_ID"] = process_id
tc.track_trace('STRAT RUN BATHCH INGEST CSV DATA from folder ' +
filesrootfolder)
tc.track_event('BATHCH_INGEST_CSV_START', {
'PROCESS_ID': process_id,
'DATA_FOLDER': filesrootfolder
}, {})
tc.flush()
tc.flush()
#print (vm_uuid,deploy_uuid,config_uuid)
# Prepare COSMOS Link
url = COSMOS_URL
#key = os.environ['ACCOUNT_KEY']
key = COSMOS_KEY
client = cosmos_client.CosmosClient(url, {'masterKey': key})
database_id = COSMOS_DATABASE
container_id = COSMOS_CONTAINER
database_link = 'dbs/' + database_id
collection_link = database_link + '/colls/' + container_id
doc_id = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_Metric'
doc_link = collection_link + '/docs/' + doc_id
options = {}
options['enableCrossPartitionQuery'] = True
options['maxItemCount'] = 5
options['partitionKey'] = vm_uuid
proc_log_doc = None
try:
proc_log_doc = client.ReadItem(doc_link, options)
except:
print("New Process Metric Doc")
if (proc_log_doc is not None):
print("Find Existing Metric Doc ")
if str(forceinsert).lower(
) != 'true': # Stop Proccess if data is already been proccessed
return 400, doc_id + " is already been processed"
else: # New process log
proc_log_doc = {}
proc_log_doc["PROCESSES"] = []
proc_log_doc["DOC_TYPE"] = "PROCESS_METRIC"
proc_log_doc["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a"
proc_log_doc['id'] = doc_id
tc.track_event('BATHCH_INGEST_METRIC_CSV', {'PROCESS_ID': process_id},
{'DATA_FOLDER': filesrootfolder})
#+'_'+config_uuid+'_'+deploy_uuid , { 'DATA_FOLDER': telemetriespath }
tc.flush()
proc_log_this = {}
proc_log_this["PROCESS_PROGRAM"] = "BATCH_METRIC_CSV_V001a"
proc_log_this["PROCESS_START"] = time.time()
proc_log_this["DATA_FOLDER"] = filesrootfolder
proc_log_this[
'id'] = vm_uuid + '_' + config_uuid + '_' + deploy_uuid + '_' + str(
process_id)
error_files, merged_files, source_files = merge_rename_core_columns_CSV(
vm_uuid, deploy_uuid, config_uuid, 'defualt_metrics_csv_001A', 0,
SOURCE_CSV_CONTAINER, filesrootfolder, FILE_OUTPUT_FOLDER, process_id)
# ToDo ...
proc_log_this["PROCESS_ID"] = process_id
proc_log_this["ERROR_SOURCE_FILES_COUNT"] = len(error_files)
proc_log_this["SOURCE_FILES_COUNT"] = len(source_files)
tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_FILES_COUNT',
len(error_files))
tc.track_metric('BATHCH_INGEST_CSV_ERROR_SOURCE_SOURCE_FILES_COUNT',
len(source_files))
tc.flush()
# print(str(len(error_files)),' ',str(len(merged_files)))
proc_log_this["PROCESS_END"] = time.time()
proc_log_this["STATUS"] = "OK"
proc_log_this["STATUS_MESSAGE"] = (
"It takes %s seconds to ingest CSV file from Blob Storage") % (
proc_log_this["PROCESS_END"] - proc_log_this["PROCESS_START"])
proc_log_doc["PROCESSES"].append(proc_log_this)
proc_log_doc['LATEST_UPDATE_TIMESTAMP'] = time.time()
# Update Process Log
client.UpsertItem(collection_link, proc_log_doc, options)
tc.track_trace('END RUN BATHCH INGEST METRIC CSV DATA from folder ' +
filesrootfolder)
tc.track_event('BATHCH_INGEST_METRIC_CSV_END', {
'PROCESS_ID': process_id,
'DATA_FOLDER': filesrootfolder
}, {
'DEFECT_FILES_COUNT': len(error_files),
'MERGED_FILES_COUNT': len(merged_files),
'SOURCE_FILES_COUNT': len(source_files)
})
tc.flush()
#Sending an event telemetry item with custom properties and measurements
from applicationinsights import TelemetryClient
tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>')
tc.track_event('Test event', { 'foo': 'bar' }, { 'baz': 42 })
tc.flush()
#Sending a trace telemetry item with custom properties
from applicationinsights import TelemetryClient
tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>')
tc.track_trace('Test trace', { 'foo': 'bar' })
tc.flush()
#Sending a metric telemetry item
from applicationinsights import TelemetryClient
tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>')
tc.track_metric('My Metric', 42)
tc.flush()
#Sending an exception telemetry item with custom properties and measurements
import sys
from applicationinsights import TelemetryClient
tc = TelemetryClient('<YOUR INSTRUMENTATION KEY GOES HERE>')
try:
raise Exception('blah')
except:
tc.track_exception()
try:
raise Exception("blah")
except:
tc.track_exception(*sys.exc_info(), properties={ 'foo': 'bar' }, measurements={ 'x': 42 })
class ApplicationInsightsTelemetryClient(BotTelemetryClient):
def __init__(self, instrumentation_key: str):
self._instrumentation_key = instrumentation_key
self._client = TelemetryClient(self._instrumentation_key)
# Telemetry Processor
def telemetry_processor(data, context):
post_data = IntegrationPostData().activity_json
# Override session and user id
from_prop = post_data['from'] if 'from' in post_data else None
user_id = from_prop['id'] if from_prop != None else None
channel_id = post_data[
'channelId'] if 'channelId' in post_data else None
conversation = post_data[
'conversation'] if 'conversation' in post_data else None
conversation_id = conversation[
'id'] if 'id' in conversation else None
context.user.id = channel_id + user_id
context.session.id = conversation_id
# Additional bot-specific properties
if 'activityId' in post_data:
data.properties["activityId"] = post_data['activityId']
if 'channelId' in post_data:
data.properties["channelId"] = post_data['channelId']
if 'activityType' in post_data:
data.properties["activityType"] = post_data['activityType']
self._client.add_telemetry_processor(telemetry_processor)
def track_pageview(self,
name: str,
url: str,
duration: int = 0,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about the page viewed in the application (a web page for instance).
:param name: the name of the page that was viewed.
:param url: the URL of the page that was viewed.
:param duration: the duration of the page view in milliseconds. (defaults to: 0)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_pageview(name, url, duration, properties,
measurements)
def track_exception(self,
type_exception: type = None,
value: Exception = None,
tb: traceback = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about a single exception that occurred in the application.
:param type_exception: the type of the exception that was thrown.
:param value: the exception that the client wants to send.
:param tb: the traceback information as returned by :func:`sys.exc_info`.
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_exception(type_exception, value, tb, properties,
measurements)
def track_event(self,
name: str,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None) -> None:
"""
Send information about a single event that has occurred in the context of the application.
:param name: the data to associate to this event.
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
"""
self._client.track_event(name, properties, measurements)
def track_metric(self,
name: str,
value: float,
type: TelemetryDataPointType = None,
count: int = None,
min: float = None,
max: float = None,
std_dev: float = None,
properties: Dict[str, object] = None) -> NotImplemented:
"""
Send information about a single metric data point that was captured for the application.
:param name: The name of the metric that was captured.
:param value: The value of the metric that was captured.
:param type: The type of the metric. (defaults to: TelemetryDataPointType.aggregation`)
:param count: the number of metrics that were aggregated into this data point. (defaults to: None)
:param min: the minimum of all metrics collected that were aggregated into this data point. (defaults to: None)
:param max: the maximum of all metrics collected that were aggregated into this data point. (defaults to: None)
:param std_dev: the standard deviation of all metrics collected that were aggregated into this data point. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
"""
self._client.track_metric(name, value, type, count, min, max, std_dev,
properties)
def track_trace(self,
name: str,
properties: Dict[str, object] = None,
severity=None):
"""
Sends a single trace statement.
:param name: the trace statement.\n
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)\n
:param severity: the severity level of this trace, one of DEBUG, INFO, WARNING, ERROR, CRITICAL
"""
self._client.track_trace(name, properties, severity)
def track_request(self,
name: str,
url: str,
success: bool,
start_time: str = None,
duration: int = None,
response_code: str = None,
http_method: str = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None,
request_id: str = None):
"""
Sends a single request that was captured for the application.
:param name: The name for this request. All requests with the same name will be grouped together.
:param url: The actual URL for this request (to show in individual request instances).
:param success: True if the request ended in success, False otherwise.
:param start_time: the start time of the request. The value should look the same as the one returned by :func:`datetime.isoformat()` (defaults to: None)
:param duration: the number of milliseconds that this request lasted. (defaults to: None)
:param response_code: the response code that this request returned. (defaults to: None)
:param http_method: the HTTP method that triggered this request. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
:param request_id: the id for this request. If None, a new uuid will be generated. (defaults to: None)
"""
self._client.track_request(name, url, success, start_time, duration,
response_code, http_method, properties,
measurements, request_id)
def track_dependency(self,
name: str,
data: str,
type: str = None,
target: str = None,
duration: int = None,
success: bool = None,
result_code: str = None,
properties: Dict[str, object] = None,
measurements: Dict[str, object] = None,
dependency_id: str = None):
"""
Sends a single dependency telemetry that was captured for the application.
:param name: the name of the command initiated with this dependency call. Low cardinality value. Examples are stored procedure name and URL path template.
:param data: the command initiated by this dependency call. Examples are SQL statement and HTTP URL with all query parameters.
:param type: the dependency type name. Low cardinality value for logical grouping of dependencies and interpretation of other fields like commandName and resultCode. Examples are SQL, Azure table, and HTTP. (default to: None)
:param target: the target site of a dependency call. Examples are server name, host address. (default to: None)
:param duration: the number of milliseconds that this dependency call lasted. (defaults to: None)
:param success: true if the dependency call ended in success, false otherwise. (defaults to: None)
:param result_code: the result code of a dependency call. Examples are SQL error code and HTTP status code. (defaults to: None)
:param properties: the set of custom properties the client wants attached to this data item. (defaults to: None)
:param measurements: the set of custom measurements the client wants to attach to this data item. (defaults to: None)
:param id: the id for this dependency call. If None, a new uuid will be generated. (defaults to: None)
"""
self._client.track_dependency(name, data, type, target, duration,
success, result_code, properties,
measurements, dependency_id)
def flush(self):
"""Flushes data in the queue. Data in the queue will be sent either immediately irrespective of what sender is
being used.
"""
self._client.flush()
