def create_security_group(self): ec2_client = boto3.client('ec2') response_describe_vpcs = ec2_client.describe_vpcs() vpc_id = response_describe_vpcs.get('Vpcs', [{}])[0].get('VpcId', '') try: response_create_security_group = ec2_client.create_security_group( GroupName=self.GroupName, Description=self.Description, VpcId=vpc_id) security_group_id = response_create_security_group[self.GroupId] print('Security Group Created %s in vpc %s.' % (security_group_id, vpc_id)) data = ec2_client.authorize_security_group_ingress( GroupId=security_group_id, IpPermissions=[ { 'IpProtocol': self.IpProtocol, 'FromPort': self.FromPort, 'ToPort': self.ToPort, 'IpRanges': self.IpRanges }, ]) print('Ingress Successfully Set %s' % data) except ClientError as error: logger.exception(error)
def create_IAM_client(self): iam_client = boto3.client('iam') # Create user try: response_create_user = iam_client.create_user( UserName=self.UserName) s3_session = boto3.Session( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, aws_session_token=self.aws_session_token, ) s3_session = boto3.Session(profile_name=self.Profile_User_name) # Any clients created from this session will use credentials # from the [Profile_User_name] section of ~/.aws/credentials. user1_s3_client = s3_session.client('s3') print(user1_s3_client) except ClientError as error: logger.exception(error)
def describe_security_groups(self): try: response_describe_security_groups = ec2.describe_security_groups( GroupIds=self.GroupIds) print(response_describe_security_groups) except ClientError as error: logger.exception(error)
def fetchRecordsFromESJson(self, jsonObj): recordList = [] try: tempJsonObj = jsonObj["hits"]["hits"] for record in tempJsonObj: recordList.append(record["_source"]) except: logger.exception("Exception in fetchRecordsFromESJson") return recordList
def connectServer(self): self.mqttc = mqtt.Client("python_pub") try: #self.mqttc.username_pw_set() self.connectionFailed = self.mqttc.connect(self.uploadIp, int(self.uploadPort)) return True except Exception as e: print "Unable to connect to the server trying to reconnect. " logger.exception(e) traceback.print_exc() self.connectionFailed = 1 return False
def queryElasticSearchById(self, indexName, indexType, indexId): try: ES_Response = self.__ES_OBJ__.get(index=indexName, doc_type=indexType, id=indexId)['_source'] except: logger.exception("Index not found in Elastic Search. " + "Index -->" + indexName + " indexType-->" + indexType + " indexId-->" + indexId) ES_Response = {} json_ES_ResponseObject = json.loads(json.dumps(ES_Response), object_hook=_decode_dict) return json_ES_ResponseObject
def queryElasticSearch(self, indexName, indexType, size=100000000): try: ES_Response = self.__ES_OBJ__.search(index=indexName, doc_type=indexType, size=size, timeout=60) except: logger.exception("Index not found in Elastic Search. " + "Index -->" + indexName + " indexType-->" + indexType) ES_Response = {} json_ES_ResponseObject = json.loads(json.dumps(ES_Response), object_hook=_decode_dict) return json_ES_ResponseObject
def updateDB(self): finalJson = {} finalJson["stationId"] = self.stationId finalJson["subStationId"] = self.subStationId finalJson["timeStamp"] = int(time.mktime(datetime.now().timetuple())) finalJson["data"] = self.finalDataJson try: print self.dbserver response = requests.post(self.dbserver+"updateDB", json=finalJson, timeout=5) print response logger.debug(response.text) except Exception as e: traceback.print_exc() print "Failed to upload" logger.exception(e)
def security_group_authorizetion_egress(self): ec2_resource = boto3.resource('ec2') security_group = ec2_resource.SecurityGroup(self.SecurityGroupId) try: response_authorize_egress = security_group.authorize_egress( DryRun=True | False, IpPermissions=[ { 'FromPort': int(self.IpPermissions_FromPort), 'IpProtocol': self.IpPermissions_IpProtocol, 'IpRanges': [ { 'CidrIp': self.IpRanges_CidrIp, 'Description': self.IpRanges_Description }, ], 'Ipv6Ranges': [ { 'CidrIpv6': self.Ipv6Ranges_CidrIpv6, 'Description': self.Ipv6Ranges_Description }, ], 'PrefixListIds': [ { 'Description': self.PrefixListIds_Description, 'PrefixListId': self.PrefixListIds_PrefixListId }, ], 'ToPort': self.ToPort, 'UserIdGroupPairs': [ { 'Description': self.UserIdGroupPairs_Description, 'GroupId': self.UserIdGroupPairs_GroupId, 'GroupName': self.UserIdGroupPairs_GroupName, 'PeeringStatus': self.UserIdGroupPairs_PeeringStatus, 'UserId': self.UserIdGroupPairs_UserId, 'VpcId': self.UserIdGroupPairs_VpcId, 'VpcPeeringConnectionId': self.UserIdGroupPairs_VpcPeeringConnectionId }, ] }, ], CidrIp=self.CidrIp, FromPort=self.FromPort, IpProtocol=self.IpProtocol, ToPort=self.ToPort, SourceSecurityGroupName=self.SourceSecurityGroupName, SourceSecurityGroupOwnerId=self.SourceSecurityGroupOwnerId) except ClientError as error: logger.exception(error)
def readConfig(self): parser = SafeConfigParser() CONFIGURATION_FILE = "setting.conf" parser.read(CONFIGURATION_FILE) try: self.GroupIds = parser.get("Settings", "GroupIds") self.GroupName = parser.get("Settings", "GroupName") self.Description = parser.get("Settings", "Description") self.GroupId = parser.get("Settings", "GroupId") self.IpProtocol = parser.get("Settings", "IpProtocol") self.FromPort = int(parser.get("Settings", "FromPort")) self.ToPort = parser.get("Settings", "ToPort") self.IpRanges = parser.get("Setting", "IpRanges") self.UserName = int(parser.get("Settings", "UserName")) self.aws_access_key_id = parser.get("Settings", "aws_access_key_id") self.aws_secret_access_key = parser.get("Settings", "aws_secret_access_key") self.aws_session_token = parser.get("Settings", "aws_session_token") self.Profile_User_name = parser.get("Settings", "Profile_User_name)") self.SecurityGroupId = parser.get("ingress", "SecurityGroupId") self.CidrIp = parser.get("ingress", "CidrIp") self.FromPort = parser.get("ingress", "FromPort") self.GroupName = parser.get("ingress", "GroupName") self.FromPort = parser.get("ingress", "FromPort") self.IpProtocol = parser.get("ingress", "IpProtocol") self.IpRanges_CidrIp = parser.get("ingress", "CidrIp") self.IpRanges_Description = parser.get("ingress", "Description") self.Ipv6Ranges_CidrIpv6 = parser.get("ingress", "CidrIp") self.Ipv6Ranges_Description = parser.get("ingress", "Description") self.PrefixListIds_PrefixListId = parser.get("ingress", "CidrIp") self.PrefixListIds_Description = parser.get( "ingress", "Description") self.ToPort = parser.get("ingress", "ToPort"), self.UserIdGroupPairs_Description = parser.get( "ingress", "Description") self.UserIdGroupPairs_GroupName = parser.get( "ingress", "GroupName") self.UserIdGroupPairs_GroupId = parser.get("ingress", "GroupId") self.UserIdGroupPairs_PeeringStatus = parser.get( "ingress", "PeeringStatus") self.UserIdGroupPairs_UserId = parser.get("ingress", "UserId") self.UserIdGroupPairs_VpcId = parser.get("ingress", "VpcId") self.UserIdGroupPairs_VpcPeeringConnectionId = parser.get( "ingress", "VpcPeeringConnectionId") self.IpProtocol = parser.get("ingress", "IpProtocol") self.SourceSecurityGroupName = parser.get( "ingress", "SourceSecurityGroupName") self.SourceSecurityGroupOwnerId = parser.get( "ingress", "SourceSecurityGroupOwnerId") self.ToPort = parser.get("ingress", "ToPort") self.DryRun = parser.get("ingress", "DryRun") self.DryRun = parser.get("egress", "DryRun") self.IpPermissions_FromPort = parser.get("egress", "IpPermissions_FromPort") self.IpPermissions_IpProtocol = parser.get( "egress", "IpPermissions_IpProtocol") self.CidrIp = parser.get("egress", "CidrIp") self.FromPort = parser.get("egress", "FromPort") self.GroupName = parser.get("egress", "GroupName") self.FromPort = parser.get("egress", "FromPort") self.IpProtocol = parser.get("egress", "IpProtocol") self.IpRanges_CidrIp = parser.get("egress", "CidrIp") self.IpRanges_Description = parser.get("egress", "Description") self.Ipv6Ranges_CidrIpv6 = parser.get("egress", "CidrIp") self.Ipv6Ranges_Description = parser.get("egress", "Description") self.PrefixListIds_PrefixListId = parser.get("egress", "CidrIp") self.PrefixListIds_Description = parser.get( "egress", "Description") self.ToPort = parser.get("egress", "ToPort"), self.UserIdGroupPairs_Description = parser.get( "egress", "Description") self.UserIdGroupPairs_GroupName = parser.get("egress", "GroupName") self.UserIdGroupPairs_GroupId = parser.get("egress", "GroupId") self.UserIdGroupPairs_PeeringStatus = parser.get( "egress", "PeeringStatus") self.UserIdGroupPairs_UserId = parser.get("egress", "UserId") self.UserIdGroupPairs_VpcId = parser.get("egress", "VpcId") self.UserIdGroupPairs_VpcPeeringConnectionId = parser.get( "egress", "VpcPeeringConnectionId") self.CidrIp = parser.get("egress", "CidrIp") self.FromPort = parser.get("egress", "FromPort") self.ToPort = parser.get("egress", "ToPort") self.SourceSecurityGroupName = parser.get( "egress", "SourceSecurityGroupName") self.SourceSecurityGroupOwnerId = parser.get( "egress", "SourceSecurityGroupOwnerId") except Exception as error: logger.exception(error)
def readConfig(self): parser = SafeConfigParser() global sleepTime, uploadData, triggerSMS parser.read("setup.conf") try: self.uploadIp = parser.get("siteSettings", "uploadIp") self.dataDumpJsonFilename = parser.get("siteSettings", "jsonFilename") # file to upload temporary data during offline state self.uploadPort = parser.get("siteSettings", "uploadPort") self.serviceUrl = parser.get("siteSettings", "serviceUrl") self.totalCollector = int(parser.get("siteSettings", "totalCollectors")) self.totalParamteters = int(parser.get("siteSettings", "totalParameters")) self.topic = parser.get("siteSettings", "topic") self.compare = int(parser.get("siteSettings", "compare")) self.dbserver = parser.get("siteSettings", "dbserver") self.dataFormat = parser.get("siteSettings", "dataFormat") sleepTime = int(parser.get("siteSettings", "sleepTime")) uploadData = int(parser.get("siteSettings", "uploadData")) try: self.numbers = parser.get("siteSettings", "numbers") triggerSMS = int(parser.get("siteSettings", "triggerSMS")) except: self.numbers = "" triggerSMS = 0 except Exception as e: traceback.print_exc() logger.debug("Exception in reading siteSettings") logger.exception(e) try: self.stationId = parser.get("siteDetails", "stationId") self.subStationId = parser.get("siteDetails", "subStationId") self.siteLocation = parser.get("siteDetails", "siteLocation") self.dataJson["stationId"] = self.stationId self.dataJson["siteLocation"] = self.siteLocation except Exception as e: traceback.print_exc() logger.debug("Exception in reading siteDetails") logger.exception(e) for i in range(1, self.totalCollector + 1): try: tempCollectorId = parser.get("Collector"+str(i), "CollectorId") tempCollectorType = parser.get("Collector"+str(i), "CollectorType") deviceIp = parser.get("Collector"+str(i), "deviceIp") start = parser.get("Collector"+str(i), "start") proxy = parser.get("Collector"+str(i), "proxy") comPort = parser.get("Collector"+str(i), "comPort") deviceId = parser.get("Collector"+str(i), "deviceId") totalRegister = parser.get("Collector"+str(i), "totalRegister") baudRate = parser.get("Collector"+str(i), "baudRate") self.dataJson["collectors"][tempCollectorId] = {} self.dataJson["collectors"][tempCollectorId]["CollectorId"] = tempCollectorId self.dataJson["collectors"][tempCollectorId]["CollectorType"] = tempCollectorType self.dataJson["collectors"][tempCollectorId]["deviceIp"] = deviceIp self.dataJson["collectors"][tempCollectorId]["start"] = start self.dataJson["collectors"][tempCollectorId]["comPort"] = comPort self.dataJson["collectors"][tempCollectorId]["deviceId"] = deviceId self.dataJson["collectors"][tempCollectorId]["proxy"] = proxy self.dataJson["collectors"][tempCollectorId]["totalRegister"] = totalRegister self.dataJson["collectors"][tempCollectorId]["baudRate"] = baudRate self.dataJson["collectors"][tempCollectorId]["parameters"] = {} except Exception as e: traceback.print_exc() logger.exception(e) for i in range(1, self.totalParamteters + 1): try: tempParameterId = parser.get("Parameter"+str(i), "parameterId") tempParameterName = parser.get("Parameter"+str(i), "parameterName") dataType = parser.get("Parameter"+str(i), "dataType") channelNo = parser.get("Parameter"+str(i), "channelNo") collector = parser.get("Parameter"+str(i), "collector") coeffA = parser.get("Parameter"+str(i), "coeffA") coeffB = parser.get("Parameter"+str(i), "coeffB") feeder = str(parser.get("Parameter"+str(i), "feeder")).strip() self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)] = {} self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["parameterId"] = tempParameterId self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["parameterName"] = tempParameterName self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["dataType"] = dataType self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["feeder"] = feeder self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["channelNo"] = channelNo self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["coeffA"] = coeffA self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["coeffB"] = coeffB self.dataJson["collectors"][collector]["parameters"]["parameter"+str(i)]["value"] = "" #self.dataJson["collectors"][collector]["parameters"][tempParameterId]["value"] = "" except Exception as e: traceback.print_exc() logger.exception(e) #print self.dataJson logger.debug("DataJson: ") logger.debug(self.dataJson)
def disconnectServer(self): try: self.mqttc.disconnect() except Exception as e: logger.exception(e) print "Failed to disconnect from server"
def uploadData(self): try: finalData = {} finalData["stationId"] = self.stationId finalData["subStationId"] = self.subStationId finalData["data"] = self.finalDataJson finalData["timestamp"] = str(datetime.now()).split(".")[0] jsonFileObj = open(self.dataDumpJsonFilename, "r+") try: jsonDumpData = json.load(jsonFileObj) except: jsonDumpData = [] #print "FinalData: ", finalData #print "finaldatajson: ", str(finalData["data"]) #print "previousData : ", str(self.previousDataJson) if self.compare == 1: if not self.compareJson(finalData["data"],self.previousDataJson): try: tempDataJson = copy.deepcopy(finalData["data"]) for everyItem in tempDataJson: if self.previousDataJson!= '' and finalData["data"][everyItem] == self.previousDataJson[everyItem]: finalData["data"].pop(everyItem) except Exception as e: traceback.print_exc() print e pass print "Uploading: ", str(finalData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F") try: resp = requests.post(self.serviceUrl + "updateCircuit", data=str(finalData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F")) logger.debug(resp) except: # store the finalDataJson until connectivity returns and gets pushed back to the server pass if self.connectServer(): MQTTresponse = self.mqttc.publish(self.topic, str(self.finalDataJson).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F")) if not MQTTresponse.is_published(): self.mqttc.reconnect() MQTTresponse = self.mqttc.publish(self.topic, str(self.finalDataJson).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F")) logger.debug(MQTTresponse.is_published()) self.mqttc.loop(2) # timeout = 2s self.previousDataJson = copy.deepcopy(finalData["data"]) print "Uploaded: ", MQTTresponse.is_published() self.connectivity = True # use this flag to check for internet connectivity else: print "Failed to connect to mqtt" logger.debug("Error while uploading to the mqtt server") logger.debug("Failed to connect to the mqtt server") self.connectivity = False # use this flag to check for internet connectivity # in that case, store the finalDataJson until connectivity returns and gets pushed back to the server # write JSON into the file jsonFileObj.seek(0) jsonDumpData.append(finalData) json.dump(jsonDumpData, jsonFileObj) fileUploaderObj.writeintoJSON(finalData, self.stationId) self.uploadHistorical = True else: tempData = {} tempData["values"] = self.finalDataJson tempData["stationId"] = self.stationId tempData["subStationId"] = self.subStationId tempData["timeStamp"] = int(time.mktime(datetime.now().timetuple())) data = json.dumps(tempData).replace("ON", "T").replace("OFF", "F").replace("True", "T").replace("False", "F") print "data:", data if self.connectServer(): self.mqttc.publish(self.topic, data) self.mqttc.loop(2) # timeout = 2s self.disconnectServer() self.previousDataJson = copy.deepcopy(finalData["data"]) print ("Uploaded") self.connectivity = True # use this flag to check for internet connectivity else: print "Failed to connect to mqtt" logger.debug("Error while uploading to the mqtt server") logger.debug("Failed to connect to the mqtt server") self.connectivity = False # use this flag to check for internet connectivity # if device fails to transmit the data, save it into a JSON for future publication # write JSON into the file logger.info("Writing data into json") jsonFileObj.seek(0) jsonDumpData.append(tempData) json.dump(jsonDumpData, jsonFileObj) fileUploaderObj.writeintoJSON(finalData, self.stationId) self.uploadHistorical = True # True if data dump from json file is required except Exception as e: traceback.print_exc() print "Exception" logger.exception(e) try: self.mqttc.disconnect() self.connectionFailed = 1 except Exception as e: traceback.print_exc() logger.exception(e) self.connectionFailed = 1 finally: try: jsonFileObj.close() except: pass
def readData(self): self.dataJson["timeStamp"] = str(datetime.now()).split(".")[0] for everyCollector in self.dataJson["collectors"]: #print self.dataJson["collectors"][everyCollector] collectorType = self.dataJson["collectors"][everyCollector]["CollectorType"] deviceIp = self.dataJson["collectors"][everyCollector]["deviceIp"] # totalRegister = self.dataJson["collectors"][everyCollector]["totalRegister"] # start = self.dataJson["collectors"][everyCollector]["start"] if "dl8-event" in collectorType.lower(): data = self.readEventJson(deviceIp, collectorType) elif "dl8-datajson" in collectorType.lower(): data = self.readDataJson(deviceIp, collectorType) client = None if "modbus" in collectorType.lower(): try: collector = self.dataJson["collectors"][everyCollector] if "-rtu" in collectorType.lower(): client = ModbusSERIALClient(method="rtu", port=collector["comPort"], stopbits=1, bytesize=8, parity='N', baudrate=int(collector["baudRate"])) elif "-tcp" in collectorType.lower(): client = ModbusTcpClient(host=deviceIp, port=502) if "-hr" in collectorType.lower(): data = self.readModbus_HR(collector, client) elif "-ir" in collectorType.lower(): data = self.readModbus_IR(collector, client) elif "-cs" in collectorType.lower(): data = self.readModbus_CS(collector, client) elif "-is" in collectorType.lower(): data = self.readModbus_IS(collector, client) try: client.close() except Exception as e: print "Exception while closing the connection" logger.exception(e) except Exception as e: logger.exception(e) print "Unable to connect to Modbus Server" try: for everyParameter in self.dataJson["collectors"][everyCollector]["parameters"]: try: dataType = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["dataType"] channelData = data[dataType] except Exception as e: #logger.exception(e) channelData = data coeffA = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["coeffA"] coeffB = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["coeffB"] channelno = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["channelNo"] feeder = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["feeder"] parameterId = self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["parameterId"] parameterId = feeder + "&"+ parameterId try: if feeder == "": print "Feeder not Defined" raise Exception if coeffA == "" and coeffB == "": if "-bit" in collectorType.lower(): bitdata = str(bin(channelData[int(str(channelno).split("-")[0])])[2:]).zfill(16)[::-1] print bitdata self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = bitdata[int(str(channelno).split("-")[1])] else: self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = str( channelData[int(channelno)]) else: tempValue = float(channelData[int(channelno)]) tempValue = tempValue * float(coeffA) tempValue = round(tempValue + float(coeffB), 2) self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] = str(tempValue) self.finalDataJson[parameterId] = \ self.dataJson["collectors"][everyCollector]["parameters"][everyParameter]["value"] except Exception as e: traceback.print_exc() logger.debug("invalid channel number for parameter: ") except Exception as e: print e.message traceback.print_exc() logger.exception(e) logger.debug("FinalData: ") logger.debug(self.dataJson)
return data Object = Monitor() fileUploaderObj = fileUploader.uploadData() while True: try: global sleepTime, uploadData, triggerSMS Object.readData() if uploadData == 1: Object.uploadData() if len(Object.dbserver) > 5: Object.updateDB() if triggerSMS == 1: Object.sendSMS() if Object.connectivity and Object.uploadHistorical: print "Uploading historical data ..." if fileUploaderObj.uploadMqtt(): Object.uploadHistorical = False print "Historical data uploaded" else: print "Failed to upload historical data on MQTT" except Exception as e: traceback.print_exc() print e.message logger.exception(e) time.sleep(int(sleepTime))