def joinMasterEntities(df, spark):
fileStatus = S3FilesDsl.readFile("C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190909_TICKET_STATUS.txt",
spark)
fileSubstatus = S3FilesDsl.readFile(
"C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190923_TICKET_SUBSTATUS.txt", spark)
fileUrgency = S3FilesDsl.readFile("C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190909_TICKET_URGENCY.txt",
spark)
filePriority = S3FilesDsl.readFile(
"C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190909_TICKET_PRIORITY.txt", spark)
fileImpact = S3FilesDsl.readFile("C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190909_TICKET_IMPACT.txt",
spark)
rodTicketStatus = TicketStatus.statusColumns(fileStatus)
rodTicketSubstatus = TicketSubstatus.substatusColumns(fileSubstatus)
rodTicketUrgency = TicketUrgency.urgencyColumns(fileUrgency)
rodTicketPriority = TicketPriority.priorityColumns(filePriority)
rodTicketImpact = TicketImpact.impactColumns(fileImpact)
df2 = df.join(rodTicketStatus, ["status_id"], "left"). \
join(rodTicketSubstatus, ["substatus_id", "status_id"], "left"). \
drop("status_id"). \
drop("substatus_id"). \
join(rodTicketUrgency, ["urgency_id"], "left"). \
drop("urgency_id"). \
join(rodTicketPriority, ["priority_id"], "left"). \
drop("priority_id"). \
join(rodTicketImpact, ["impact_id"], "left"). \
drop("impact_id")
return df2
def persistRelations(esIndexRel, s3confPath, spark):
sqlContext = SQLContext(spark)
relationsDF = esIndexRel.select("ticket_id", "related_ticket_id")
total = None
try:
total = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_relations_parquet_path)
except Exception as e:
message = str(e)
if message.find("Path does not exist"):
raise e
else:
logging.info("catched Path does not exist (first job execution): " + str(e))
relationsDF \
.repartition(1) \
.write \
.mode("overwite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_relations_parquet_path)
total = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_relations_parquet_path)
total.cache()
logging.info("total.count---------------------------------------------" + str(total.count()))
result = total \
.unionByName(relationsDF) \
.distinct() \
.repartition(1) \
.write \
.mode("overwrite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_relations_parquet_path)
return result
def removeClosedAgentSmc(esIndex, s3confPath, spark):
sqlContext = SQLContext(spark)
closed = esIndex.select("ticket_id", "assigned_agent", "smc_cluster", "reported_source_id").distinct()
auxOuterJoin = closed.select("ticket_id")
total = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
total.cache()
logging.info("totalWithoutClosed.count---------------------------------------------" + str(total.count()))
totalWithoutClosed = total \
.join(auxOuterJoin, ["ticket_id"], "left") \
.where(auxOuterJoin["ticket_id"].isNull())
result = totalWithoutClosed \
.repartition(1) \
.write \
.mode("overwrite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
logging.info(
"totalWithoutClosed.count---------------------------------------------" + str(totalWithoutClosed.count()))
return result
def adminNumberTags(tagsAdminPath):
rodTicketANTags = AdminNumberTags.antagsColumns(S3FilesDsl.readFile())
ticketANTags = rodTicketANTags \
.groupBy("admin_number") \
.agg(F.concat_ws(",", F.collect_set("tags").alias("tags"))) \
.withColumn("tags", Utils.stringToArray("tags"))
return ticketANTags
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info(
"Start batch Coptero ROD for s3confPath:" + s3confPath + "--------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(s3filePath,
S3FilesDsl.readFileSchema(s3filePath,
getClosedSchema(s3filePath),
spark),
spark, s3confPath)
logging.info("validatedRecords.count().." + str(validatedRecords.count()))
ticketToCloseDS = detailClosedColumns(validatedRecords, spark)
logging.info("ticketToCloseDS.count().." + str(ticketToCloseDS.count()))
esIndex = ticketToCloseDS\
.withColumn("open", F.lit(Constants.OPEN_NO))\
.withColumn("file", F.lit(s3filePath))\
.withColumn("ticket_max_value_partition", Utils.getIndexPartition("ticket_id"))
logging.info("Persisting ES index..")
dfCount = esIndex.count()
logging.info("indexDataFrame.count.." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(esIndex, "copt-rod-closed-{ticket_max_value_partition}", "ticket_id", s3confPath)
except Exception as e:
message = str(e)
if message.find("index_closed_exception"):
raise e
else:
# TODO saveToEs {partitioned} works fine but ends with exception ?¿
logging.info("catched index_closed_exception: " + str(e))
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info("End batch Coptero ROD ----------------------------------------------------")
except Exception as e:
logStatus = copy.deepcopy(logStatus)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = str(e)
logStatus.end_date = ""
logging.error("catched: " + str(e))
raise e
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count, logStatus.success, logStatus.exception,
logStatus.start_date, logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(copy.deepcopy(logStatus_data))
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", s3confPath)
def writeESCorruptRecordsIndex(index, name, conf):
prefix = S3FilesDsl.readConfigJson(conf).elastic_env_index_prefix
config = {
"elastic_nodes": "127.0.0.1",
"elastic_port": "9200",
"elastic_user": "******",
"elastic_pass": "******"
}
toElastic(config, index, addId,
prefix + name + datetime.now().strftime("%Y"))
'''index.write.format(
def fullPersistAgentSmc(esIndex, s3confPath):
preload = esIndex.select("ticket_id", "assigned_agent", "smc_cluster", "reported_source_id")
result = preload \
.repartition(1) \
.write \
.mode("overwrite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
logging.info("total.count---------------------------------------------" + str(preload.count()))
return result
def persistAgentSmc(esIndex, s3confPath, spark):
sqlContext = SQLContext(spark)
newOrUpdated = esIndex.select("ticket_id", "assigned_agent", "smc_cluster", "reported_source_id").distinct()
auxOuterJoin = newOrUpdated.select("ticket_id")
total = None
try:
total = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
except Exception as e:
message = str(e)
if message.find("Path does not exist"):
raise e
else:
logging.info("catched Path does not exist (first job execution): " + str(e))
newOrUpdated \
.repartition(1) \
.write \
.mode("overwite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
total = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
total.cache()
logging.info("total.count---------------------------------------------" + str(total.count()))
totalWithoutNewOrUpdated = total \
.join(auxOuterJoin, ["ticket_id"], "left") \
.where(auxOuterJoin["ticket_id"].isNull())
updatedToParquet = totalWithoutNewOrUpdated.unionByName(newOrUpdated)
result = updatedToParquet \
.repartition(1) \
.write \
.mode("overwrite") \
.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
return result
def getRelations(esIndex, s3confPath, spark):
sqlContext = SQLContext(spark)
parquet = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
agents = parquet \
.filter(parquet.reported_source_id != "Vendor") \
.withColumnRenamed("ticket_id", "agent_ticket_id")
relations = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_relations_parquet_path)
relatedAgents = relations \
.join(agents, relations.related_ticket_id == agents.agent_ticket_id, "inner") \
.groupBy("ticket_id") \
.agg(F.collect_set("assigned_agent").alias("assignee"),
F.collect_set("smc_cluster").alias("smc"))
result = esIndex \
.join(relatedAgents, ["ticket_id"], "left") \
.withColumn("assignee", Utils.addToArray("assigned_agent", "assignee")) \
.withColumn("smc", Utils.addToArray("smc_cluster", "smc")) \
.drop("assigned_agent", "smc_cluster")
return result
def checkCount(indexName, fileName, dfCount, spark, conf):
prefix = S3FilesDsl.readConfigJson(conf).elastic_env_index_prefix
sqlContext = SQLContext(spark)
logging.info('dfCount.. ' + str(dfCount))
path = fileName.replace(':', '\\:').replace("/", "\\/")
qResultDF1 = sqlContext.read \
.option("es.resource",prefix + indexName) \
.option("es.query", "?q=file:\''" + path + " '\'") \
.format("org.elasticsearch.spark.sql") \
.load()
# ¿Equivalente de qResultDF = spark.esDF("${indexName}", "?q=file:\"" + path + "\"").select("ticket_id") ?
qResultDF = qResultDF1.select("ticket_id")
qResultDF.cache()
queryCount = qResultDF.count()
qResultDF.unpersist()
logging.info("queryCount.. " + str(queryCount))
if dfCount != queryCount:
alertDataFrame = sqlContext.createDataFrame(
[(fileName, dfCount, queryCount, datetime.now().strftime("%Y%m%d%H%M%S"))],
["file", "expected_count", "result_count", "date"])
ElasticDsl.writeESAlertsIndex(alertDataFrame, conf)
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
conf = s3confPath
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info("Start batch Coptero ROD for s3confPath:" +
s3confPath + "--------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath,
getIncidSchema(s3filePath), spark),
spark, conf)
logging.info("fileDetailHelpdesk.count().." +
str(validatedRecords.count()))
rodTicketDetailHelpdesk = TicketDetailHelpdesk.detailHPDColumns(
validatedRecords)
logging.info("rodTicketDetailHelpdesk.count.." +
str(rodTicketDetailHelpdesk.count()))
calendar = datetime.now().strftime("%Y%m%d%H%M%S")
filtered = rodTicketDetailHelpdesk. \
filter((rodTicketDetailHelpdesk.status_id == "5") | (rodTicketDetailHelpdesk.status_id == "6")). \
filter(rodTicketDetailHelpdesk.last_modification_date < calendar)
#TODO
#valfileStatus: DataFrame = readFile(getAuxTablePath("TICKET_STATUS"))
#val rodTicketStatus: Dataset[TicketStatus] = statusColumns(fileStatus)
# rodTicketDetailHelpdesk
#.join(rodTicketStatus, Seq("status_id"), "left")
#.filter($"status_desc" == = "Closed" | | $"status_desc" == = "Cancelled")
#.drop("status_desc")
#filter($"last_modification_date" < new impleDateFormat("yyyyMMddHHmmss").format(calendar.getTime))
# val cisClosedDates = getCIsLastClosedDates(rodTicketDetailHelpdesk)
esIndex = RemedyDsl.buildESIndex("helpdesk", filtered, s3confPath,
s3filePath, spark)
# TODO ? esIndex.as[IncidESIndex]with Option[String] = None
logging.info("Persisting ES index..")
#dfCount = esIndex.count()
#logging.info("indexDataFrame.count.." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(
esIndex, "copt-rod-closed-{ticket_max_value_partition}",
"ticket_id", conf)
except Exception as e:
message = str(e)
if message.find("index_closed_exception"):
raise e
else:
# TODO saveToEs {partitioned} works fine but ends with exception ?¿
logging.info("catched index_closed_exception: " + str(e))
removeClosedAgentSmc(esIndex, s3confPath, spark)
AlertDsl.checkCount("copt-rod-closed-*", s3filePath, dfCount,
spark, s3confPath)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as e:
logStatus = copy.deepcopy(logStatus)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = str(e)
logStatus.end_date = ""
logging.error("catched: " + str(e))
raise e
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
logDataFrame.show(5)
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", conf)
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
conf = s3confPath
logStatus = startLogStatus(s3filePath)
dfCountRelation = 0
dfCountIncid = 0
try:
logging.info("Start batch Coptero ROD for s3confPath: " +
s3confPath + " -------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath,
getRelationSchema(s3filePath),
spark), spark, conf)
rodTicketRelation1 = relationColumns(validatedRecords, spark)
rodTicketRelation = rodTicketRelation1.withColumn(
"relation_id",
F.concat(rodTicketRelation1["ticket_id"], F.lit('-'),
rodTicketRelation1["related_ticket_id"]))
esIndexRel = rodTicketRelation.select(
'relation_id', 'ticket_id', 'ticket_type', 'related_ticket_id',
'related_ticket_type', 'association_type', 'submit_date',
'relation_summary', 'status', 'submitter', 'instanceid')
partitioned = esIndexRel \
.withColumn("ticket_max_value_partition", Utils.getIndexPartition("ticket_id")) \
.withColumn("file", F.lit(s3filePath))
dfCountRelation = partitioned.count()
logging.info("Persisting ES index..")
logging.info("indexRelationDataFrame.count().." +
str(dfCountRelation))
try:
ElasticDsl.writeMappedESIndex(
partitioned, "copt-rod-rel-{ticket_max_value_partition}",
"relation_id", conf)
except Exception as ex:
e = str(ex)
if e.find("index_closed_exception"):
logging.info("catched index_closed_exception: " + e)
else:
raise ex
AlertDsl.checkCount("copt-rod-rel-*", s3filePath, dfCountRelation,
spark, conf)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCountRelation
logStatus.exception = ""
logStatus.end_date = ""
persistRelations(esIndexRel, conf, spark)
'''relationsDF = esIndexRel \
.filter(esIndexRel.ticket_type == "Incident") \
.groupBy("ticket_id") \
.agg(F.collect_list("related_ticket_id").alias("relations")) \
.withColumn("ticket_max_value_partition", Utils.getIndexPartition("ticket_id")) \
.withColumn("file", F.lit(s3filePath))
# TODO writeMappedESIndex CRQ Y PBI
dfCountIncid = relationsDF.count()
logging.info("Persisting ES index..")
logging.info("relationsDF.count().."+ str(dfCountIncid))
try:
ElasticDsl.writeMappedESIndex(
relationsDF, "copt-rod-closed-{ticket_max_value_partition}", "ticket_id", conf)
except Exception as ex:
e = str(ex)
if e.find("index_closed_exception"):
logging.info("catched index_closed_exception: " + e)
else:
raise ex
AlertDsl.checkCount("copt-rod-closed-*", s3filePath, dfCountIncid, spark, conf)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCountIncid
logStatus.exception = ""
logStatus.end_date = ""'''
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as ex:
e = str(ex)
logStatus.success = False
logStatus.count = 0
logStatus.exception = str(e)
logStatus.end_date = ""
logging.info("catched: " + e)
raise ex
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", conf)
def writeESAlertsIndex(index, config):
prefix = S3FilesDsl.readConfigJson(config).elastic_env_index_prefix
index.write.format('org.elasticsearch.spark.sql').mode(
'append').option('es.write.operation', 'index').option(
'es.resource', prefix + 'copt-rod-alerts').save()
def buildESIndex(detailType, detail, s3confPath, s3filePath, spark):
sqlContext = SQLContext(spark)
# TODO.confinsteadof.json val
confJson = S3FilesDsl.readConfigJson(s3confPath)
rodTicketANTags = AdminNumberTags.antagsColumns(
S3FilesDsl.readFile(confJson.tags_admin_path, spark), spark)
parquetPath = confJson.fast_parquet_path
rodPostgreAdminNumber = sqlContext.read.parquet(parquetPath)
logging.info("FAST joins..")
networkFast = sqlContext.read.parquet(confJson.fast_network_parquet_path)
logging.info("common joins..")
# TODO: añadir import de utils.constantes
# TODO: comprobar parametros que se pasan a los metodos de Utils
common3 = joinMasterEntities(detail, spark)
common2 = common3.join(rodPostgreAdminNumber, ["admin_number"], "left")
common1 = Utils.fillEmptyFastColumns(common2)
common = common1.join(networkFast, ["admin_number"], "left"). \
withColumn("networkinfo", Utils.networkNestedObject("fast_customer", "fast_end_customer",
"router_interface_vendor_type_set")). \
drop("router_interface_vendor_type_set"). \
join(rodTicketANTags, ["admin_number"], "left"). \
withColumn("open", F.when(common1.status_desc.isin(Constants.openStatus), Constants.OPEN_YES).
otherwise(F.when(common1.status_desc.isin(Constants.notOpenStatus), Constants.OPEN_NO).
otherwise(Constants.EMPTY_STRING))). \
withColumn("ticket_max_value_partition", Utils.getIndexPartition("ticket_id")). \
withColumn("admin_number_escaped", Utils.urlWhitespaces("admin_number")). \
withColumn("fast_max_resolution_time", Utils.validateNumeric("fast_max_resolution_time")). \
withColumn("file", F.lit(s3filePath)). \
fillna(Constants.EMPTY_STRING, ["assigned_agent"])
if detailType == "helpdesk":
rodTicketReportedSource = getReportedSource(spark)
operationalManager = getOperationalManager(confJson.operational_path, spark)
opTags = OperatingTags.operatingTagsColumns(S3FilesDsl.readFile(confJson.tags_operating_path, spark))
customer = Customer.customerColumns(S3FilesDsl.readFile(confJson.customer_path, spark), spark)
endCustomer = EndCustomer.endCustomerColumns(S3FilesDsl.readFile(confJson.end_customer_path, spark), spark)
index1 = common \
.join(rodTicketReportedSource, ["reported_source_id"], "left") \
.drop("reported_source_id") \
.join(operationalManager, ["operating_company_name", "operating_le"], "left") \
.na.fill(Constants.EMPTY_STRING, ["operational_manager"]) \
.join(opTags, ["operating_company_name", "operating_le"], "left") \
.withColumn("tags", Utils.mergeArrays("tags", "operating_tags")) \
.drop("operating_tags") \
.join(customer, ["operating_company_name"], "left") \
.fillna(Constants.EMPTY_STRING, ["customer_correct"]) \
.join(endCustomer, ["operating_le"], "left") \
.fillna(Constants.EMPTY_STRING, ["end_customer_correct"]) \
.withColumn("end_customer_correct",
Utils.emptyEndCustomerCorrect("customer_correct", "end_customer_correct")) \
.withColumn("ci_country", Utils.kibanaCountry("ci_country")) \
.withColumn("end_user_country", Utils.kibanaCountry("end_user_country")) \
.withColumn("smc_cluster", Utils.smcClusterFromGroup("assigned_support_group")) \
.withColumn("ci_name_escaped", Utils.urlWhitespaces("ci_name")) \
.withColumn("product_categorization_all_tiers",
Utils.concat3Columns("product_categorization_tier_1", "product_categorization_tier_2",
"product_categorization_tier_3")) \
.withColumn("closure_categorization_all_tiers",
Utils.concat3Columns("closure_categorization_tier_1", "closure_categorization_tier_2",
"closure_categorization_tier_3")) \
.withColumn("operational_categorization_all_tiers",
Utils.concat3Columns("operational_categorization_tier_1",
"operational_categorization_tier_2",
"operational_categorization_tier_3")) \
.withColumnRenamed("reported_source_desc", "reported_source_id")
index = FastDsl.fastCircuitFields(index1, confJson, spark)
elif detailType == "problems":
index1 = common \
.withColumn("ci_country", Utils.kibanaCountry("ci_country")) \
.withColumn("ci_name_escaped", Utils.urlWhitespaces("ci_name"))
index = FastDsl.fastCircuitFields(index1, confJson, spark)
elif detailType == "changes":
rodTicketReportedSource = getReportedSource(spark)
index = common \
.join(rodTicketReportedSource, ["reported_source_id"], "left") \
.drop("reported_source_id") \
.withColumn("ci_country", Utils.kibanaCountry("ci_country")) \
.withColumn("company_country", Utils.kibanaCountry("company_country")) \
.withColumnRenamed("reported_source_desc", "reported_source_id")
# EL USUARIO SOLICITA QUE LAS DESCRIPCIONES DE LOS MAESTROS SE RENOMBREN COMO _id
indexRenamed = index \
.withColumnRenamed("status_desc", "status_id") \
.withColumnRenamed("substatus_desc", "substatus_id") \
.withColumnRenamed("urgency_desc", "urgency_id") \
.withColumnRenamed("priority_desc", "priority_id") \
.withColumnRenamed("impact_desc", "impact_id")
return indexRenamed
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
conf = s3confPath
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info("Start batch Coptero ROD for s3confPath: " +
s3confPath + "-------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath, getPBISchema(s3filePath),
spark), spark, conf)
rodTicketDetailProblems = detailPBMColumns(validatedRecords, spark)
esIndexPBM = RemedyDsl.buildESIndex("problems",
rodTicketDetailProblems,
s3confPath, s3filePath, spark)
dfCount = esIndexPBM.count()
logging.info("Persisting ES indexes..")
logging.info("indexProblemDataFrame.count().." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(
esIndexPBM, "copt-rod-pbi-{ticket_max_value_partition}",
"ticket_id", conf)
except Exception as ex:
e = str(ex)
if e.find("index_closed_exception"):
logging.info("catched index_closed_exception: " + e)
else:
raise ex
AlertDsl.checkCount("copt-rod-pbi-*", s3filePath, dfCount, spark,
conf)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as ex:
e = str(ex)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = e
logStatus.end_date = ""
logging.info("catched: " + e)
raise ex
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", conf)
def writeMappedESIndex(index, name, mapId, config):
prefix = S3FilesDsl.readConfigJson(config).elastic_env_index_prefix
index.write.format('org.elasticsearch.spark.sql').option(
'es.mapping.id', mapId).option('es.resource',
prefix + name).save()
def getOperationalManager(s3path, spark):
fileOperationalManager = S3FilesDsl.readFile(s3path, spark)
return OperationalManager.operationalManagerColumns(fileOperationalManager)
def getReportedSource(spark):
fileReportedSource = S3FilesDsl.readFile(
"C:/Users/gonza/Downloads/PruebasLAB/resources/RD_TR_20190909_TICKET_REPORTED_SOURCE.txt", spark)
return TicketReportedSource.reportedSourceColumns(fileReportedSource)
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
conf = s3confPath
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info("Start batch Coptero ROD for s3confPath:" +
s3confPath + "--------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath, getCRQSchema(s3filePath),
spark), spark, conf)
rodTicketDetailChanges = detailCHGColumns(validatedRecords)
esIndexCHG = RemedyDsl.buildESIndex("changes",
rodTicketDetailChanges,
s3confPath, s3filePath, spark)
logging.info("Persisting ES index..")
dfCount = esIndexCHG.count()
logging.info("indexDataFrame.count.." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(
esIndexCHG, "copt-rod-crq-{ticket_max_value_partition}",
"ticket_id", conf)
except Exception as e:
ex = str(e)
if ex.find("index_closed_exception"):
raise e
else:
# TODO saveToEs {partitioned} works fine but ends with exception ?¿
logging.info("catched index_closed_exception: " + ex)
AlertDsl.checkCount("copt-rod-crq-*", s3filePath, dfCount, spark,
s3confPath)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as e:
ex = str(e)
logStatus = copy.deepcopy(logStatus)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = ex
logStatus.end_date = ""
logging.error("catched: " + ex)
raise e
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", conf)
def getAgentSmcCluster(esIndex, s3confPath, spark):
sqlContext = SQLContext(spark)
parquet = sqlContext.read.parquet(S3FilesDsl.readConfigJson(s3confPath).rod_agent_smc_parquet_path)
return esIndex.join(parquet.select("ticket_id", "smc_cluster", "assigned_agent"), ["ticket_id"], "left")
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
conf = s3confPath
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info("Start batch Coptero ROD for s3confPath:" +
s3confPath + "--------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath,
getIncidSchema(s3filePath), spark),
spark, conf)
logging.info("fileDetailHelpdesk.count().." +
str(validatedRecords.count()))
rodTicketDetailHelpdesk = TicketDetailHelpdesk.detailHPDColumns(
validatedRecords)
logging.info("rodTicketDetailHelpdesk.count.." +
str(rodTicketDetailHelpdesk.count()))
# val cisClosedDates = getCIsLastClosedDates(rodTicketDetailHelpdesk)
esIndex = RemedyDsl.buildESIndex("helpdesk",
rodTicketDetailHelpdesk,
s3confPath, s3filePath, spark)
print("NUESTRO DATAFRAME")
esIndex.show()
print("VUESTRO DATAFRAME")
# TODO ? esIndex.as[IncidESIndex]with Option[String] = None
logging.info("Persisting ES index..")
dfCount = esIndex.count()
logging.info("indexDataFrame.count.." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(
esIndex, "copt-rod-closed-{ticket_max_value_partition}",
"ticket_id", conf)
except Exception as e:
message = str(e)
if message.find("index_closed_exception"):
raise e
else:
# TODO saveToEs {partitioned} works fine but ends with exception ?¿
logging.info("catched index_closed_exception: " + str(e))
persistAgentSmc(esIndex, s3confPath, spark)
AlertDsl.checkCount("copt-rod-closed-*", s3filePath, dfCount,
spark, conf)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as e:
logStatus = copy.deepcopy(logStatus)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = str(e)
logStatus.end_date = ""
logging.error("catched: " + str(e))
raise e
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
ElasticDsl.writeESLogIndex(logDataFrame, "copt-rod-log-", conf)
def runJob(sparkSession, s3confPath, s3filePath):
spark = sparkSession
logStatus = startLogStatus(s3filePath)
dfCount = 0
try:
logging.info("Start batch Coptero ROD for s3confPath: " +
s3confPath + "--------------------------------------")
validatedRecords = ValidationsDsl.validateTickets(
s3filePath,
S3FilesDsl.readFileSchema(s3filePath, getWISchema(s3filePath),
spark), spark, s3confPath)
indexAgentSmcCluster = getAgentSmcCluster(validatedRecords,
s3confPath, spark)
indexWithRelations = getRelations(indexAgentSmcCluster, s3confPath,
spark)
partitioned = indexWithRelations \
.withColumn("ticket_max_value_partition", Utils.getIndexPartition("ticket_id")) \
.withColumn("file", F.lit(s3filePath)) \
.withColumn("work_info_category", Utils.getWorkInfoCategory("work_info_notes"))
dfCount = partitioned.count()
logging.info("Persisting ES index..")
logging.info("indexWorkInfoDataFrame.count().." + str(dfCount))
try:
ElasticDsl.writeMappedESIndex(
partitioned, "copt-rod-wif-{ticket_max_value_partition}",
"instanceid", s3confPath)
except Exception as e:
message = str(e)
if message.find("index_closed_exception"):
raise e
else:
# TODO saveToEs {partitioned} works fine but ends with exception ?¿
logging.info("catched index_closed_exception: " + str(e))
AlertDsl.checkCount("copt-rod-wif*", s3filePath, dfCount, spark,
s3confPath)
logStatus = copy.deepcopy(logStatus)
logStatus.success = True
logStatus.count = dfCount
logStatus.exception = ""
logStatus.end_date = ""
logging.info(
"End batch Coptero ROD ----------------------------------------------------"
)
except Exception as e:
logStatus = copy.deepcopy(logStatus)
logStatus.success = False
logStatus.count = dfCount
logStatus.exception = str(e)
logStatus.end_date = ""
logging.error("catched: " + str(e))
raise e
finally:
sqlContext = SQLContext(spark)
logStatus.end_date = datetime.now().strftime("%Y%m%d%H%M%S")
logStatus_data = logESindexSchema(logStatus.file, logStatus.count,
logStatus.success,
logStatus.exception,
logStatus.start_date,
logStatus.end_date)
logDataFrame = sqlContext.createDataFrame(
copy.deepcopy(logStatus_data))
def writeESLogIndex(index, name, config):
prefix = S3FilesDsl.readConfigJson(config).elastic_env_index_prefix
index.write.format('org.elasticsearch.spark.sql').mode(
'append').option('es.write.operation', 'index').option(
'es.resource',
prefix + name + datetime.now().strftime("%Y")).save()