def create_security_group(self):
ec2_client = boto3.client('ec2')
response_describe_vpcs = ec2_client.describe_vpcs()
vpc_id = response_describe_vpcs.get('Vpcs', [{}])[0].get('VpcId', '')
try:
response_create_security_group = ec2_client.create_security_group(
GroupName=self.GroupName,
Description=self.Description,
VpcId=vpc_id)
security_group_id = response_create_security_group[self.GroupId]
print('Security Group Created %s in vpc %s.' %
(security_group_id, vpc_id))
data = ec2_client.authorize_security_group_ingress(
GroupId=security_group_id,
IpPermissions=[
{
'IpProtocol': self.IpProtocol,
'FromPort': self.FromPort,
'ToPort': self.ToPort,
'IpRanges': self.IpRanges
},
])
print('Ingress Successfully Set %s' % data)
except ClientError as error:
logger.exception(error)
def create_IAM_client(self):
iam_client = boto3.client('iam')
# Create user
try:
response_create_user = iam_client.create_user(
UserName=self.UserName)
s3_session = boto3.Session(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
aws_session_token=self.aws_session_token,
)
s3_session = boto3.Session(profile_name=self.Profile_User_name)
# Any clients created from this session will use credentials
# from the [Profile_User_name] section of ~/.aws/credentials.
user1_s3_client = s3_session.client('s3')
print(user1_s3_client)
except ClientError as error:
logger.exception(error)
def describe_security_groups(self):
try:
response_describe_security_groups = ec2.describe_security_groups(
GroupIds=self.GroupIds)
print(response_describe_security_groups)
except ClientError as error:
logger.exception(error)
def fetchRecordsFromESJson(self, jsonObj):
recordList = []
try:
tempJsonObj = jsonObj["hits"]["hits"]
for record in tempJsonObj:
recordList.append(record["_source"])
except:
logger.exception("Exception in fetchRecordsFromESJson")
return recordList
def connectServer(self):
self.mqttc = mqtt.Client("python_pub")
try:
#self.mqttc.username_pw_set()
self.connectionFailed = self.mqttc.connect(self.uploadIp, int(self.uploadPort))
return True
except Exception as e:
print "Unable to connect to the server trying to reconnect. "
logger.exception(e)
traceback.print_exc()
self.connectionFailed = 1
return False
def queryElasticSearchById(self, indexName, indexType, indexId):
try:
ES_Response = self.__ES_OBJ__.get(index=indexName,
doc_type=indexType,
id=indexId)['_source']
except:
logger.exception("Index not found in Elastic Search. " +
"Index -->" + indexName + " indexType-->" +
indexType + " indexId-->" + indexId)
ES_Response = {}
json_ES_ResponseObject = json.loads(json.dumps(ES_Response),
object_hook=_decode_dict)
return json_ES_ResponseObject
def queryElasticSearch(self, indexName, indexType, size=100000000):
try:
ES_Response = self.__ES_OBJ__.search(index=indexName,
doc_type=indexType,
size=size,
timeout=60)
except:
logger.exception("Index not found in Elastic Search. " +
"Index -->" + indexName + " indexType-->" +
indexType)
ES_Response = {}
json_ES_ResponseObject = json.loads(json.dumps(ES_Response),
object_hook=_decode_dict)
return json_ES_ResponseObject
def updateDB(self):
finalJson = {}
finalJson["stationId"] = self.stationId
finalJson["subStationId"] = self.subStationId
finalJson["timeStamp"] = int(time.mktime(datetime.now().timetuple()))
finalJson["data"] = self.finalDataJson
try:
print self.dbserver
response = requests.post(self.dbserver+"updateDB", json=finalJson, timeout=5)
print response
logger.debug(response.text)
except Exception as e:
traceback.print_exc()
print "Failed to upload"
logger.exception(e)
def security_group_authorizetion_egress(self):
ec2_resource = boto3.resource('ec2')
security_group = ec2_resource.SecurityGroup(self.SecurityGroupId)
try:
response_authorize_egress = security_group.authorize_egress(
DryRun=True | False,
IpPermissions=[
{
'FromPort':
int(self.IpPermissions_FromPort),
'IpProtocol':
self.IpPermissions_IpProtocol,
'IpRanges': [
{
'CidrIp': self.IpRanges_CidrIp,
'Description': self.IpRanges_Description
},
],
'Ipv6Ranges': [
{
'CidrIpv6': self.Ipv6Ranges_CidrIpv6,
'Description': self.Ipv6Ranges_Description
},
],
'PrefixListIds': [
{
'Description': self.PrefixListIds_Description,
'PrefixListId': self.PrefixListIds_PrefixListId
},
],
'ToPort':
self.ToPort,
'UserIdGroupPairs': [
{
'Description':
self.UserIdGroupPairs_Description,
'GroupId':
self.UserIdGroupPairs_GroupId,
'GroupName':
self.UserIdGroupPairs_GroupName,
'PeeringStatus':
self.UserIdGroupPairs_PeeringStatus,
'UserId':
self.UserIdGroupPairs_UserId,
'VpcId':
self.UserIdGroupPairs_VpcId,
'VpcPeeringConnectionId':
self.UserIdGroupPairs_VpcPeeringConnectionId
},
]
},
],
CidrIp=self.CidrIp,
FromPort=self.FromPort,
IpProtocol=self.IpProtocol,
ToPort=self.ToPort,
SourceSecurityGroupName=self.SourceSecurityGroupName,
SourceSecurityGroupOwnerId=self.SourceSecurityGroupOwnerId)
except ClientError as error:
logger.exception(error)
def readConfig(self):
parser = SafeConfigParser()
CONFIGURATION_FILE = "setting.conf"
parser.read(CONFIGURATION_FILE)
try:
self.GroupIds = parser.get("Settings", "GroupIds")
self.GroupName = parser.get("Settings", "GroupName")
self.Description = parser.get("Settings", "Description")
self.GroupId = parser.get("Settings", "GroupId")
self.IpProtocol = parser.get("Settings", "IpProtocol")
self.FromPort = int(parser.get("Settings", "FromPort"))
self.ToPort = parser.get("Settings", "ToPort")
self.IpRanges = parser.get("Setting", "IpRanges")
self.UserName = int(parser.get("Settings", "UserName"))
self.aws_access_key_id = parser.get("Settings",
"aws_access_key_id")
self.aws_secret_access_key = parser.get("Settings",
"aws_secret_access_key")
self.aws_session_token = parser.get("Settings",
"aws_session_token")
self.Profile_User_name = parser.get("Settings",
"Profile_User_name)")
self.SecurityGroupId = parser.get("ingress", "SecurityGroupId")
self.CidrIp = parser.get("ingress", "CidrIp")
self.FromPort = parser.get("ingress", "FromPort")
self.GroupName = parser.get("ingress", "GroupName")
self.FromPort = parser.get("ingress", "FromPort")
self.IpProtocol = parser.get("ingress", "IpProtocol")
self.IpRanges_CidrIp = parser.get("ingress", "CidrIp")
self.IpRanges_Description = parser.get("ingress", "Description")
self.Ipv6Ranges_CidrIpv6 = parser.get("ingress", "CidrIp")
self.Ipv6Ranges_Description = parser.get("ingress", "Description")
self.PrefixListIds_PrefixListId = parser.get("ingress", "CidrIp")
self.PrefixListIds_Description = parser.get(
"ingress", "Description")
self.ToPort = parser.get("ingress", "ToPort"),
self.UserIdGroupPairs_Description = parser.get(
"ingress", "Description")
self.UserIdGroupPairs_GroupName = parser.get(
"ingress", "GroupName")
self.UserIdGroupPairs_GroupId = parser.get("ingress", "GroupId")
self.UserIdGroupPairs_PeeringStatus = parser.get(
"ingress", "PeeringStatus")
self.UserIdGroupPairs_UserId = parser.get("ingress", "UserId")
self.UserIdGroupPairs_VpcId = parser.get("ingress", "VpcId")
self.UserIdGroupPairs_VpcPeeringConnectionId = parser.get(
"ingress", "VpcPeeringConnectionId")
self.IpProtocol = parser.get("ingress", "IpProtocol")
self.SourceSecurityGroupName = parser.get(
"ingress", "SourceSecurityGroupName")
self.SourceSecurityGroupOwnerId = parser.get(
"ingress", "SourceSecurityGroupOwnerId")
self.ToPort = parser.get("ingress", "ToPort")
self.DryRun = parser.get("ingress", "DryRun")
self.DryRun = parser.get("egress", "DryRun")
self.IpPermissions_FromPort = parser.get("egress",
"IpPermissions_FromPort")
self.IpPermissions_IpProtocol = parser.get(
"egress", "IpPermissions_IpProtocol")
self.CidrIp = parser.get("egress", "CidrIp")
self.FromPort = parser.get("egress", "FromPort")
self.GroupName = parser.get("egress", "GroupName")
self.FromPort = parser.get("egress", "FromPort")
self.IpProtocol = parser.get("egress", "IpProtocol")
self.IpRanges_CidrIp = parser.get("egress", "CidrIp")
self.IpRanges_Description = parser.get("egress", "Description")
self.Ipv6Ranges_CidrIpv6 = parser.get("egress", "CidrIp")
self.Ipv6Ranges_Description = parser.get("egress", "Description")
self.PrefixListIds_PrefixListId = parser.get("egress", "CidrIp")
self.PrefixListIds_Description = parser.get(
"egress", "Description")
self.ToPort = parser.get("egress", "ToPort"),
self.UserIdGroupPairs_Description = parser.get(
"egress", "Description")
self.UserIdGroupPairs_GroupName = parser.get("egress", "GroupName")
self.UserIdGroupPairs_GroupId = parser.get("egress", "GroupId")
self.UserIdGroupPairs_PeeringStatus = parser.get(
"egress", "PeeringStatus")
self.UserIdGroupPairs_UserId = parser.get("egress", "UserId")
self.UserIdGroupPairs_VpcId = parser.get("egress", "VpcId")
self.UserIdGroupPairs_VpcPeeringConnectionId = parser.get(
"egress", "VpcPeeringConnectionId")
self.CidrIp = parser.get("egress", "CidrIp")
self.FromPort = parser.get("egress", "FromPort")
self.ToPort = parser.get("egress", "ToPort")
self.SourceSecurityGroupName = parser.get(
"egress", "SourceSecurityGroupName")
self.SourceSecurityGroupOwnerId = parser.get(
"egress", "SourceSecurityGroupOwnerId")
except Exception as error:
logger.exception(error)
def readConfig(self):
parser = SafeConfigParser()
global sleepTime, uploadData, triggerSMS
parser.read("setup.conf")
try:
self.uploadIp = parser.get("siteSettings", "uploadIp")
self.dataDumpJsonFilename = parser.get("siteSettings", "jsonFilename") # file to upload temporary data during offline state
self.uploadPort = parser.get("siteSettings", "uploadPort")
self.serviceUrl = parser.get("siteSettings", "serviceUrl")
self.totalCollector = int(parser.get("siteSettings", "totalCollectors"))
self.totalParamteters = int(parser.get("siteSettings", "totalParameters"))
self.topic = parser.get("siteSettings", "topic")
self.compare = int(parser.get("siteSettings", "compare"))
self.dbserver = parser.get("siteSettings", "dbserver")
self.dataFormat = parser.get("siteSettings", "dataFormat")
sleepTime = int(parser.get("siteSettings", "sleepTime"))
uploadData = int(parser.get("siteSettings", "uploadData"))
try:
self.numbers = parser.get("siteSettings", "numbers")
triggerSMS = int(parser.get("siteSettings", "triggerSMS"))
except:
self.numbers = ""
triggerSMS = 0
except Exception as e:
traceback.print_exc()
logger.debug("Exception in reading siteSettings")
logger.exception(e)
try:
self.stationId = parser.get("siteDetails", "stationId")
self.subStationId = parser.get("siteDetails", "subStationId")
self.siteLocation = parser.get("siteDetails", "siteLocation")
self.dataJson["stationId"] = self.stationId
self.dataJson["siteLocation"] = self.siteLocation
except Exception as e:
traceback.print_exc()
logger.debug("Exception in reading siteDetails")
logger.exception(e)
for i in range(1, self.totalCollector + 1):
try:
tempCollectorId = parser.get("Collector"+str(i), "CollectorId")
tempCollectorType = parser.get("Collector"+str(i), "CollectorType")
deviceIp = parser.get("Collector"+str(i), "deviceIp")
start = parser.get("Collector"+str(i), "start")
proxy = parser.get("Collector"+str(i), "proxy")
comPort = parser.get("Collector"+str(i), "comPort")
deviceId = parser.get("Collector"+str(i), "deviceId")
totalRegister = parser.get("Collector"+str(i), "totalRegister")
baudRate = parser.get("Collector"+str(i), "baudRate")
self.dataJson["collectors"][tempCollectorId] = {}
self.dataJson["collectors"][tempCollectorId]["CollectorId"] = tempCollectorId
self.dataJson["collectors"][tempCollectorId]["CollectorType"] = tempCollectorType
self.dataJson["collectors"][tempCollectorId]["deviceIp"] = deviceIp
self.dataJson["collectors"][tempCollectorId]["start"] = start
self.dataJson["collectors"][tempCollectorId]["comPort"] = comPort
self.dataJson["collectors"][tempCollectorId]["deviceId"] = deviceId
self.dataJson["collectors"][tempCollectorId]["proxy"] = proxy
self.dataJson["collectors"][tempCollectorId]["totalRegister"] = totalRegister
self.dataJson["collectors"][tempCollectorId]["baudRate"] = baudRate
self.dataJson["collectors"][tempCollectorId]["parameters"] = {}
except Exception as e:
traceback.print_exc()
logger.exception(e)
for i in range(1, self.totalParamteters + 1):
try:
tempParameterId = parser.get("Parameter"+str(i), "parameterId")
tempParameterName = parser.get("Parameter"+str(i), "parameterName")
dataType = parser.get("Parameter"+str(i), "dataType")
channelNo = parser.get("Parameter"+str(i), "channelNo")
collector = parser.get("Parameter"+str(i), "collector")
coeffA = parser.get("Parameter"+str(i), "coeffA")
coeffB = parser.get("Parameter"+str(i), "coeffB")
feeder = str(parser.get("Parameter"+str(i), "feeder")).strip()
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)] = {}
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["parameterId"] = tempParameterId
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["parameterName"] = tempParameterName
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["dataType"] = dataType
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["feeder"] = feeder
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["channelNo"] = channelNo
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["coeffA"] = coeffA
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["coeffB"] = coeffB
self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["value"] = ""
#self.dataJson["collectors"][collector]["parameters"][tempParameterId]["value"] = ""
except Exception as e:
traceback.print_exc()
logger.exception(e)
#print self.dataJson
logger.debug("DataJson: ")
logger.debug(self.dataJson)
def disconnectServer(self):
try:
self.mqttc.disconnect()
except Exception as e:
logger.exception(e)
print "Failed to disconnect from server"
def uploadData(self):
try:
finalData = {}
finalData["stationId"] = self.stationId
finalData["subStationId"] = self.subStationId
finalData["data"] = self.finalDataJson
finalData["timestamp"] = str(datetime.now()).split(".")[0]
jsonFileObj = open(self.dataDumpJsonFilename, "r+")
try:
jsonDumpData = json.load(jsonFileObj)
except:
jsonDumpData = []
#print "FinalData: ", finalData
#print "finaldatajson: ", str(finalData["data"])
#print "previousData : ", str(self.previousDataJson)
if self.compare == 1:
if not self.compareJson(finalData["data"],self.previousDataJson):
try:
tempDataJson = copy.deepcopy(finalData["data"])
for everyItem in tempDataJson:
if self.previousDataJson!= '' and finalData["data"][everyItem] == self.previousDataJson[everyItem]:
finalData["data"].pop(everyItem)
except Exception as e:
traceback.print_exc()
print e
pass
print "Uploading: ", str(finalData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F")
try:
resp = requests.post(self.serviceUrl + "updateCircuit",
data=str(finalData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F"))
logger.debug(resp)
except:
# store the finalDataJson until connectivity returns and gets pushed back to the server
pass
if self.connectServer():
MQTTresponse = self.mqttc.publish(self.topic, str(self.finalDataJson).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F"))
if not MQTTresponse.is_published():
self.mqttc.reconnect()
MQTTresponse = self.mqttc.publish(self.topic,
str(self.finalDataJson).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F"))
logger.debug(MQTTresponse.is_published())
self.mqttc.loop(2) # timeout = 2s
self.previousDataJson = copy.deepcopy(finalData["data"])
print "Uploaded: ", MQTTresponse.is_published()
self.connectivity = True # use this flag to check for internet connectivity
else:
print "Failed to connect to mqtt"
logger.debug("Error while uploading to the mqtt server")
logger.debug("Failed to connect to the mqtt server")
self.connectivity = False # use this flag to check for internet connectivity
# in that case, store the finalDataJson until connectivity returns and gets pushed back to the server
# write JSON into the file
jsonFileObj.seek(0)
jsonDumpData.append(finalData)
json.dump(jsonDumpData, jsonFileObj)
fileUploaderObj.writeintoJSON(finalData, self.stationId)
self.uploadHistorical = True
else:
tempData = {}
tempData["values"] = self.finalDataJson
tempData["stationId"] = self.stationId
tempData["subStationId"] = self.subStationId
tempData["timeStamp"] = int(time.mktime(datetime.now().timetuple()))
data = json.dumps(tempData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F")
print "data:", data
if self.connectServer():
self.mqttc.publish(self.topic, data)
self.mqttc.loop(2) # timeout = 2s
self.disconnectServer()
self.previousDataJson = copy.deepcopy(finalData["data"])
print ("Uploaded")
self.connectivity = True # use this flag to check for internet connectivity
else:
print "Failed to connect to mqtt"
logger.debug("Error while uploading to the mqtt server")
logger.debug("Failed to connect to the mqtt server")
self.connectivity = False # use this flag to check for internet connectivity
# if device fails to transmit the data, save it into a JSON for future publication
# write JSON into the file
logger.info("Writing data into json")
jsonFileObj.seek(0)
jsonDumpData.append(tempData)
json.dump(jsonDumpData, jsonFileObj)
fileUploaderObj.writeintoJSON(finalData, self.stationId)
self.uploadHistorical = True # True if data dump from json file is required
except Exception as e:
traceback.print_exc()
print "Exception"
logger.exception(e)
try:
self.mqttc.disconnect()
self.connectionFailed = 1
except Exception as e:
traceback.print_exc()
logger.exception(e)
self.connectionFailed = 1
finally:
try:
jsonFileObj.close()
except:
pass
def readData(self):
self.dataJson["timeStamp"] = str(datetime.now()).split(".")[0]
for everyCollector in self.dataJson["collectors"]:
#print self.dataJson["collectors"][everyCollector]
collectorType = self.dataJson["collectors"][everyCollector]["CollectorType"]
deviceIp = self.dataJson["collectors"][everyCollector]["deviceIp"]
# totalRegister = self.dataJson["collectors"][everyCollector]["totalRegister"]
# start = self.dataJson["collectors"][everyCollector]["start"]
if "dl8-event" in collectorType.lower():
data = self.readEventJson(deviceIp, collectorType)
elif "dl8-datajson" in collectorType.lower():
data = self.readDataJson(deviceIp, collectorType)
client = None
if "modbus" in collectorType.lower():
try:
collector = self.dataJson["collectors"][everyCollector]
if "-rtu" in collectorType.lower():
client = ModbusSERIALClient(method="rtu", port=collector["comPort"], stopbits=1, bytesize=8,
parity='N', baudrate=int(collector["baudRate"]))
elif "-tcp" in collectorType.lower():
client = ModbusTcpClient(host=deviceIp, port=502)
if "-hr" in collectorType.lower():
data = self.readModbus_HR(collector, client)
elif "-ir" in collectorType.lower():
data = self.readModbus_IR(collector, client)
elif "-cs" in collectorType.lower():
data = self.readModbus_CS(collector, client)
elif "-is" in collectorType.lower():
data = self.readModbus_IS(collector, client)
try:
client.close()
except Exception as e:
print "Exception while closing the connection"
logger.exception(e)
except Exception as e:
logger.exception(e)
print "Unable to connect to Modbus Server"
try:
for everyParameter in self.dataJson["collectors"][everyCollector]["parameters"]:
try:
dataType = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["dataType"]
channelData = data[dataType]
except Exception as e:
#logger.exception(e)
channelData = data
coeffA = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["coeffA"]
coeffB = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["coeffB"]
channelno = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["channelNo"]
feeder = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["feeder"]
parameterId = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["parameterId"]
parameterId = feeder + "&"+ parameterId
try:
if feeder == "":
print "Feeder not Defined"
raise Exception
if coeffA == "" and coeffB == "":
if "-bit" in collectorType.lower():
bitdata = str(bin(channelData[int(str(channelno).split("-")[0])])[2:]).zfill(16)[::-1]
print bitdata
self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = bitdata[int(str(channelno).split("-")[1])]
else:
self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = str(
channelData[int(channelno)])
else:
tempValue = float(channelData[int(channelno)])
tempValue = tempValue * float(coeffA)
tempValue = round(tempValue + float(coeffB), 2)
self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = str(tempValue)
self.finalDataJson[parameterId] = \
self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"]
except Exception as e:
traceback.print_exc()
logger.debug("invalid channel number for parameter: ")
except Exception as e:
print e.message
traceback.print_exc()
logger.exception(e)
logger.debug("FinalData: ")
logger.debug(self.dataJson)
return data
Object = Monitor()
fileUploaderObj = fileUploader.uploadData()
while True:
try:
global sleepTime, uploadData, triggerSMS
Object.readData()
if uploadData == 1:
Object.uploadData()
if len(Object.dbserver) > 5:
Object.updateDB()
if triggerSMS == 1:
Object.sendSMS()
if Object.connectivity and Object.uploadHistorical:
print "Uploading historical data ..."
if fileUploaderObj.uploadMqtt():
Object.uploadHistorical = False
print "Historical data uploaded"
else:
print "Failed to upload historical data on MQTT"
except Exception as e:
traceback.print_exc()
print e.message
logger.exception(e)
time.sleep(int(sleepTime))
