def buildClassifier(self, instances: Instances): self.getCapabilities().testWithFail(instances) sumOfWeights = 0 self.m_Class = instances.classAttribute() self.m_ClassValue = 0 attrType = instances.classAttribute().type() if attrType == Attribute.NUMERIC: self.m_Counts = None elif attrType == Attribute.NOMINAL: self.m_Counts = [] for i in range(instances.numClasses()): self.m_Counts.append(1) sumOfWeights = instances.numClasses() for instance in instances: classValue = instance.classValue() if not Utils.isMissingValue(classValue): if instances.classAttribute().isNominal(): self.m_Counts[classValue] += instance.weight() else: self.m_ClassValue += instance.weight() * classValue sumOfWeights += instance.weight() if instances.classAttribute().isNumeric(): if Utils.gr(sumOfWeights, 0): self.m_ClassValue /= sumOfWeights else: self.m_ClassValue = Utils.maxIndex(self.m_Counts) Utils.normalize(self.m_Counts, sumOfWeights)
def makeDistribution(self,neighbours:Instances,distances:List)->List[float]: distribution=[0]*self.m_NumClasses total=0 if self.m_ClassType == Attribute.NOMINAL: for i in range(self.m_NumClasses): distribution[i]=1/max(1,self.m_Train.numInstances()) total=self.m_NumClasses/max(1,self.m_Train.numInstances()) for i in range(neighbours.numInstances()): current=neighbours.instance(i) distances[i]=distances[i]*distances[i] distances[i]=math.sqrt(distances[i]/self.m_NumAttributesUsed) if self.DistanceWeighting == self.WEIGHT_INVERSE: weight=1/distances[i] elif self.DistanceWeighting == self.WEIGHT_SIMILARITY: weight=1-distances[i] else: weight=1 weight*=current.weight() if self.m_ClassType == Attribute.NOMINAL: distribution[int(current.classValue())]+=weight elif self.m_ClassType == Attribute.NUMERIC: distribution[0]+=current.classValue()*weight total+=weight if total > 0: Utils.normalize(distribution, total) return distribution
def splitCritValue(self, bags: Distribution, totalNoInst: float = None, numerator: float = None): if totalNoInst is None and numerator is None: numerator = self.oldEnt(bags) - self.newEnt(bags) if Utils.equal(numerator, 0): return float('inf') denumerator = self.splitEnt(bags) if Utils.equal(denumerator, 0): return float('inf') return denumerator / numerator elif numerator is None: res = 0 noUnkown = totalNoInst - bags.total() if Utils.gr(bags.total(), 0): for i in range(bags.numBags()): res = res - self.lnFunc(bags.perBag(i)) res = res - self.lnFunc(noUnkown) res = res + self.lnFunc(totalNoInst) return res / math.log(2) else: denumerator = self.splitEnt(bags, totalNoInst) if Utils.equal(denumerator, 0): return 0 denumerator /= totalNoInst return numerator / denumerator
def split(self,data:Instances)->List[Instances]: subsetSize=[0]*self.m_numSubsets for inst in data: subset=self.whichSubset(inst) if subset > -1: subsetSize[subset]+=1 else: weights=self.weights(inst) for j in range(self.m_numSubsets): if Utils.gr(weights[j], 0): subsetSize[j]+=1 instances=[] #type:List[Instances] for j in range(self.m_numSubsets): instances.append(Instances(data,subsetSize[j])) for inst in data: subset=self.whichSubset(inst) if subset > -1: instances[subset].add(inst) else: weights=self.weights(inst) for j in range(self.m_numSubsets): if Utils.gr(weights[j], 0): instances[j].add(inst) instances[j].lastInstance().setWeight(float(weights[j]*inst.weight())) return instances
def createMenu(self): menu = QMenu() showClassifierErrors = menu.addAction( u"Visualize classifier errors") #type:QAction showVisualizeTree = menu.addAction(u"Visualize tree") #type:QAction selectedNames = [i.text() for i in self.m_History.selectedItems()] o = None # type:List if selectedNames is not None and len(selectedNames) == 1: Utils.debugOut("history_name: ", selectedNames) o = self.m_History.getNamedObject(selectedNames[0]) temp_vp = None # type:VisualizePanel temp_grph = None # type:str if o is not None: for i in range(len(o)): temp = o[i] if isinstance(temp, VisualizePanel): temp_vp = temp elif isinstance(temp, str): temp_grph = temp if temp_vp is not None: showClassifierErrors.setEnabled(True) showClassifierErrors.triggered.connect( self.classifierErrorTrigger(temp_vp)) else: showClassifierErrors.setEnabled(False) if temp_grph is not None: showVisualizeTree.setEnabled(True) showVisualizeTree.triggered.connect( self.visualizeTreeTrigger(temp_grph)) else: showVisualizeTree.setEnabled(False) menu.exec_(QCursor.pos())
def computeAverageClassValues(self): avgClassValues = [[] for i in range(self.getInputFormat().numAttributes()) ] self.m_Indices = [[] for i in range(self.getInputFormat().numAttributes()) ] for j in range(self.getInputFormat().numAttributes()): att = self.getInputFormat().attribute(j) if att.isNominal(): avgClassValues[j] = [0] * att.numValues() counts = [0] * att.numValues() for i in range(self.getInputFormat().numInstances()): instance = self.getInputFormat().instance(i) if not instance.classIsMissing( ) and not instance.isMissing(j): counts[int(instance.value(j))] += instance.weight() avgClassValues[j][int(instance.value( j))] += instance.weight() * instance.weight() sums = sum(avgClassValues[j]) totalCounts = sum(counts) if Utils.gr(totalCounts, 0): for k in range(att.numValues()): if Utils.gr(counts[k], 0): avgClassValues[j][k] /= counts[k] else: avgClassValues[j][k] = sums / totalCounts self.m_Indices[j] = Utils.sortDouble(avgClassValues[j])
def addUndoPoint(self): if self.m_Data is not None: temp = tempfile.TemporaryFile() pickle.dump(self.m_Data, temp) temp.seek(0) self.m_UndoList.append(temp) self.state_changed_signal.emit() Utils.debugOut("Now undo list len:", len(self.m_UndoList))
def dumpLabel(self,index:int,data:Instances): text="" text+=data.classAttribute().value(self.m_distribution.maxClass(index)) text+=" ("+str(Utils.roundDouble(self.m_distribution.perBag(index), 2)) if Utils.gr(self.m_distribution.numIncorrect(index), 0): text+="/"+str(Utils.roundDouble(self.m_distribution.numIncorrect(index), 2)) text+=")" return text
def signal_handler(signal, frame): """ Used to catch a keyboard signal like Ctrl+C in order to kill the kalliope program :param signal: signal handler :param frame: execution frame """ print "\n" Utils.print_info("Ctrl+C pressed. Killing Kalliope") sys.exit(0)
def rightSide(self, index: int, data: Instances): text = "" if data.attribute(self.m_attIndex).isNominal(): text += " = " + data.attribute(self.m_attIndex).value(index) elif index == 0: text += " <= " + Utils.doubleToString(self.m_splitPoint, 6) else: text += " > " + Utils.doubleToString(self.m_splitPoint, 6) return text
def insertInstanceEvent(self, inst: Instances, row: int): self.m_Pass = True self.m_Table.insertRow(row) self.m_Table.setItem(row, 0, QTableWidgetItem(str(row + 1))) self.m_Table.setRawItem(inst, row) self.adjustRowNo(row) Utils.debugOut("insert after current instance count:", self.model.getInstance().numInstances()) self.m_Pass = False
def maxBag(self): max = 0 maxIndex = -1 for i in range(len(self.m_perBag)): if Utils.gr(self.m_perBag[i], max) or Utils.equal( self.m_perBag[i], max): max = self.m_perBag[i] maxIndex = i return maxIndex
def check(self, minNoObj: float): counter = 0 for i in range(len(self.m_perBag)): if Utils.gr(self.m_perBag[i], minNoObj) or Utils.equal( self.m_perBag[i], minNoObj): counter += 1 if counter > 1: return True return False
def prob(self, classIndex: int, intIndex: int = None): if intIndex is None: if not Utils.equal(self.totaL, 0): return self.m_perClass[classIndex] / self.totaL return 0 else: if Utils.gr(self.m_perBag[intIndex], 0): return self.m_perClassPerBag[intIndex][ classIndex] / self.m_perBag[intIndex] return self.prob(classIndex)
def prune(self): if not self.m_isLeaf: for i in range(len(self.m_sons)): self.son(i).prune() if Utils.gr(self.errorsForTree(), self.errorsForLeaf()) or Utils.equal( self.errorsForTree(), self.errorsForLeaf()): self.m_sons = None self.m_isLeaf = None self.m_localModel = NoSplit(self.localModel().distribution())
def load_stt_plugin(self): if self.stt is None: self.stt_module_name = self.settings.default_stt_name for stt_object in self.settings.stts: if stt_object.name == self.stt_module_name: stt_object.parameters["callback"] = self.callback Utils.get_dynamic_class_instantiation('stt', stt_object.name.capitalize(), parameters=stt_object.parameters)
def evaluateClusterer(self, test: Instances, outputModel: bool): i = loglk = unclusteredInstances = 0 cc = self.m_Clusterer.numberOfClusters() self.m_numClusters = cc instanceStats = [0] * cc hasClass = test.classIndex() >= 0 clusterAssignments = [] filter = None #type:Filter testRaw = copy.deepcopy(test) testRaw.setClassIndex(test.classIndex()) if hasClass: if testRaw.classAttribute().isNumeric(): raise Exception(unclusteredInstances) filter = Remove() filter.setAttributeIndices(str(testRaw.classIndex() + 1)) filter.setInvertSelection(False) filter.setInputFormat(testRaw) for inst in testRaw: if filter is not None: filter.input(inst) filter.batchFinished() inst = filter.output() cnum = self.m_Clusterer.clusterInstance(inst) clusterAssignments.append(cnum) if cnum != -1: instanceStats[cnum] += 1 sumNum = sum(instanceStats) loglk /= sumNum self.m_logL = loglk self.m_clusterAssignments = [] # for i in clusterAssignments: # print(",",i,end="") # print() for i in range(len(clusterAssignments)): self.m_clusterAssignments.append(clusterAssignments[i]) numInstFieldWidth = int( math.log(len(clusterAssignments)) / math.log(10) + 1) if outputModel: self.m_clusteringResult += str(self.m_Clusterer) self.m_clusteringResult += "Clustered Instances\n\n" clustFieldWidth = int((math.log(cc) / math.log(10)) + 1) for i in range(cc): if instanceStats[i] > 0: self.m_clusteringResult+= Utils.doubleToString(i, clustFieldWidth, 0) \ +" " \ + Utils.doubleToString(instanceStats[i], numInstFieldWidth, 0) \ +"(" + Utils.doubleToString((instanceStats[i] / sumNum * 100), 3, 0) \ +"%)\n" if unclusteredInstances > 0: self.m_clusteringResult += "\nUnclustered instances : " + str( unclusteredInstances) if hasClass: self.evaluateClustersWithRespectToClass(test)
def useFilter(cls, data: Instances, filter: 'Filter'): for i in range(data.numInstances()): filter.input(data.instance(i)) filter.batchFinished() newData = filter.getOutputFormat() Utils.debugOut("Queue size:", filter.m_OutputQueue.qsize()) processed = filter.output() while processed is not None: newData.add(processed) processed = filter.output() return newData
def parsePost(self, post): # View pretty json - http://jsonformatter.curiousconcept.com/ print(post) pid = post["id"] user = post["user"]["username"] name = post["user"]["full_name"] tags = post["tags"] caption = post["caption"] location = post["location"] created_time = datetime.fromtimestamp(int(post["created_time"])).strftime('%Y-%m-%d %H:%M:%S') images = post["images"] photo = images["standard_resolution"]["url"] profile_picture = post["user"]["profile_picture"] likes = post["likes"] comments = post["comments"] text = "" if caption: text = caption["text"] if location: latitude = location["latitude"] longitude = location["longitude"] print("[*] Post ID: %s" % pid) print('[*] User: %s' % user) print('[*] Full name: %s' % name) print('[*] Date of creation: %s' % created_time) print('[*] Tags: %s' % tags) print('[*] Latitude: %s - Longitude: %s' % (latitude, longitude)) print('[*] Photo: %s' % (photo)) print('[*] Text: %s' % text) print('') utils = Utils() person = utils.get_user(user, name, profile_picture) p = utils.save_post(pid, person, created_time, photo, text, latitude, longitude, "Instagram") utils.save_tags(tags, p) if len(likes["data"]) > 0: c = utils.save_connection(p, 0) for like in likes["data"]: person_who_likes = utils.get_user(like["username"], like["full_name"], like["profile_picture"]) c.user.add(person_who_likes) print('[+] Like from: %s' % (person_who_likes.login)) if comments["count"] > 0: for comment in comments["data"]: c = utils.save_connection(p, 1, comment["text"]) person_who_comments = utils.get_user(comment["from"]["username"], comment["from"]["full_name"], comment["from"]["profile_picture"]) c.user.add(person_who_comments) print('[+] Comment from: %s' % person_who_comments.login) print('') return created_time
def __init__(self, target, logger, noprint=False, nolog=False, verbose=False): self.URL = 'http://ip-api.com' self.RequestURL = self.URL + '/json/{}' self.BOLD = '\033[1m' self.Proxy = None self.UserAgentFile = None self.UserAgents = None self.Proxies = None self.TargetsFile = None self.ProxiesFile = None self.Targets = None self.NoPrint = noprint self.Target = target self.Logger = logger self.Utils = Utils(nolog, verbose)
def getCurve(self, predictions: List[Prediction], classIndex: int) -> Instances: if len(predictions) == 0 or len( predictions[0].distribution()) <= classIndex: return None totPos = totNeg = 0 probs = self.getProbabilities(predictions, classIndex) for i in range(len(probs)): pred = predictions[i] if pred.actual() == Utils.missingValue(): continue if pred.weight() < 0: continue if pred.actual() == classIndex: totPos += pred.weight() else: totNeg += pred.weight() insts = self.makeHeader() sorted = Utils.sortDouble(probs) tc = TwoClassStats(totPos, totNeg, 0, 0) threshold = cumulativePos = cumulativeNeg = 0 for i in range(len(sorted)): if i == 0 or probs[sorted[i]] > threshold: tc.setTruePositive(tc.getTruePositive() - cumulativePos) tc.setFalseNegative(tc.getFalseNegative() + cumulativePos) tc.setFalsePositive(tc.getFalsePositive() - cumulativeNeg) tc.setTrueNegative(tc.getTrueNegative() + cumulativeNeg) threshold = probs[sorted[i]] insts.add(self.makeInstance(tc, threshold)) cumulativePos = 0 cumulativeNeg = 0 if i == len(sorted) - 1: break pred = predictions[sorted[i]] if pred.actual() == Utils.missingValue(): continue if pred.weight() < 0: continue if pred.actual() == classIndex: cumulativePos += pred.weight() else: cumulativeNeg += pred.weight() if tc.getFalseNegative() != totPos or tc.getTrueNegative() != totNeg: tc = TwoClassStats(0, 0, totNeg, totPos) threshold = probs[sorted[-1]] + 10e-6 insts.add(self.makeInstance(tc, threshold)) return insts
def clusterInstance(self, instance: Instance): dist = self.distributionForInstance(instance) if dict is None: raise Exception("Null distribution predicted") if sum(dist) <= 0: raise Exception("Unable to cluster instance") return Utils.maxIndex(dist)
def truePositiveRate(self,classIndex:int): correct=total=0 for j in range(self.m_NumClasses): if j == classIndex: correct+=self.m_ConfusionMatrix[classIndex][j] total+=self.m_ConfusionMatrix[classIndex][j] return Utils.division(correct, total)
def precision(self,classIndex:int): correct=total=0 for i in range(self.m_NumClasses): if i == classIndex: correct+=self.m_ConfusionMatrix[i][classIndex] total+=self.m_ConfusionMatrix[i][classIndex] return Utils.division(correct, total)
def areaUnderROC(self,classIndex:int): if self.m_Predictions is None: return Utils.missingValue() else: tc=ThresholdCurve() result=tc.getCurve(self.m_Predictions,classIndex) return ThresholdCurve.getROCArea(result)
def classifyInstance(self,instance:Instance): dist=self.distributionForInstance(instance) if dist is None: raise Exception("Null distribution predicted") if instance.classAttribute().type() == Attribute.NOMINAL: max=maxIndex=0 for i in range(len(dist)): if dist[i]>max: maxIndex=i max=dist[i] if max > 0: return maxIndex return Utils.missingValue() elif instance.classAttribute().type() == Attribute.NUMERIC or instance.classAttribute().type() == Attribute.DATE: return dist[0] return Utils.missingValue()
def exceptions(e): """ logger = Logger() logger.pid = Session.getPid() logger.user = Session.getUserDisplay() # ---------------------------------------------------- # Trace dans l'activité d'une erreur dans activity.log logger.error('{APP_NAME} à rencontré une erreur'.format(APP_NAME=cfg._APP_NAME)) # ---------------------------------------------------- # Trace de l'exception dans un fichier à part # import traceback # traceback.print_exc() logger.critical( nom=exc_type.__name__, message=str(exc_value), trace=exc_tb ) """ # Recuperation ERREUR et trace dans Activité exc_type, exc_value, exc_tb = sys.exc_info() # print(exc_value) # loggerAct.exception(e) # Renvoi Erreur if Utils.isAjaxRequest(request) is True: return Render.jsonTemplate( operation='OOPS !! Une erreur est arrivé', message='{MESSAGE}'.format(MESSAGE=exc_value), categorie="ERROR", code=500) return Render.htmlTemplate("errors/{}.html".format(str(500)), data=None, code=500)
def setValueAt(self, value, rowIndexList: List, columnIndex: int): self.addUndoPoint() for rowIndex in rowIndexList: type = self.getType(columnIndex) index = columnIndex - 1 inst = self.m_Data.instance(rowIndex) att = inst.attribute(index) if value is None: inst.setValue(index, Utils.missingValue()) else: if type == Attribute.NUMERIC: inst.setValue(index, float(value)) elif type == Attribute.NOMINAL: if att.indexOfValue(value) > -1: inst.setValue(index, att.indexOfValue(value)) # print(att.indexOfValue(value)) else: rowIndexList = [] break elif type == Attribute.STRING: inst.setValue(index, value) elif type == Attribute.DATE: #TODO pass self.update_instance_value_signal.emit(rowIndexList, columnIndex, value)
def after_request(response): """Tracage des requetes URL.""" # Cela évite la duplication du registre dans le journal, # puisque ce 500 est déjà connecté via @app.errorhandler. if (response.status_code != 500 and not request.full_path.startswith('/static') # and cfg._ENV[cfg._ENVIRONNEMENT]["LOG_WATCHER"] is True ): # Construction du Message text = "{} :: {} :: {} :: {} :: {} :: {} :: {} :: {}".format( request.remote_addr, Session.getUserDisplay() if Session.getUserDisplay() is not None else "", Session.getUserId() if Session.getUserId() is not None else "", request.method, request.scheme, request.full_path, response.status, '1' if Utils.isAjaxRequest(request) else '0') # Enregistrement # loggerWatch.info(text) """ file_object = open(cfg._ENV[cfg._ENVIRONNEMENT]["LOG_WATCHER_FILE"], "a+") file_object.write("{} :: {} :: {} :: {} :: {} :: {} :: {} :: {}\n".format( datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'), request.remote_addr, Session.getUserDisplay() if Session.getUserDisplay() is not None else "", Session.getUserId() if Session.getUserId() is not None else "", request.method, request.scheme, request.full_path, response.status )) file_object.close() """ return response
def addUser(): if request.method == 'POST': # Recuperation + traitement des données du formulaire data = Utils.parseForm(dict(request.form)) # Cryptage des mots des passe + Decryptage des id data["password"] = Crypt.encode(cfg._APP_SECRET_KEY, data["password"]) data["id"] = Crypt.decode(cfg._APP_SECRET_KEY, data["id"]) mdl = AuthUsersModel(cfg._BDD_PATH) # Verification if mdl.is_unique_username(data["username"]) is False: return Render.jsonTemplate( _OPERATION, "Un utilisateur avec cet identifiant existe déjà", categorie="WARNING") # Traitement en BDD try: mdl.insertUser(data) except SqliteAdapterException as errorSQL: return Render.jsonTemplate( _OPERATION, 'Ajout Utilisateur Impossible : {}'.format(str(errorSQL)), categorie="ERROR") else: return Render.jsonTemplate(_OPERATION, 'Ajout Utilisateur', categorie="SUCCESS") else: abort(400)
def updateParam(): if request.method == 'POST': # Recuperation + traitement des données du formulaire data = Utils.parseForm(dict(request.form)) # Initialisation de la Config config = Configuration.from_filepath() # Modification de la config if 'time_zone' in data: config.set_time_zone(data['time_zone']) if 'hour_start' in data: config.set_hour_start(data['hour_start']) if 'hour_end' in data: config.set_hour_end(data['hour_end']) if 'filepathCredentials' in data: config.set_filepathCredentials(data['filepathCredentials']) if 'use_demo_account' in data: config.set_use_demo_account(True if data['use_demo_account'] == 'true' else False) if 'spin_interval' in data: config.set_spin_interval(data['spin_interval']) # Envoi du message return Render.jsonTemplate(_OPERATION, 'Enregistrement de la Configuration', categorie="SUCCESS") else: abort(400)
class IpGeoLocationLib: """Retrieve IP Geolocation information from http://ip-api.com""" def __init__(self, target, logger, noprint=False, nolog=False, verbose=False): self.URL = 'http://ip-api.com' self.RequestURL = self.URL + '/json/{}' self.BOLD = '\033[1m' self.Proxy = None self.UserAgentFile = None self.UserAgents = None self.Proxies = None self.TargetsFile = None self.ProxiesFile = None self.Targets = None self.NoPrint = noprint self.Target = target self.Logger = logger self.Utils = Utils(nolog, verbose) def GetInfo(self, userAgent, targetsFile=None, userAgentFile=None, proxy=False, proxiesFile=None, exportToCSVFile=None, exportToXMLFile=None, exportToTXTFile=None, googleMaps=False): """Retrieve information""" self.UserAgent = userAgent try: #check proxies file and load it if proxiesFile and os.path.isfile(proxiesFile) and os.access(proxiesFile, os.R_OK): self.ProxiesFile = proxiesFile self.Logger.Print('Loading Proxies from file {}..'.format(self.ProxiesFile)) self.__loadProxies() #check user-agent strings file and load it if userAgentFile and os.path.isfile(userAgentFile) and os.access(userAgentFile, os.R_OK): self.UserAgentFile = userAgentFile self.Logger.Print('Loading User-Agent strings from file {}..'.format(self.UserAgentFile)) self.__loadUserAgents() #check targets file and load it if targetsFile and os.path.isfile(targetsFile) and os.access(targetsFile, os.R_OK): self.TargetsFile = targetsFile self.Logger.Print('Loading targets from file {}..'.format(self.TargetsFile)) self.__loadTargets() #check if proxy valid and configure connection if proxy: self.__configureProxy(proxy) #retrieve information results = None if self.TargetsFile: results = self.__retrieveGeolocations() else: results = self.__retrieveGeolocation(self.Target) #export information if exportToCSVFile and not os.path.exists(exportToCSVFile) and os.access(os.path.dirname(exportToCSVFile), os.W_OK): self.__exportResultsToCSV(results, exportToCSVFile) if exportToXMLFile and not os.path.exists(exportToXMLFile) and os.access(os.path.dirname(exportToXMLFile), os.W_OK): self.__exportResultsToXML(results, exportToXMLFile) if exportToTXTFile and not os.path.exists(exportToTXTFile) and os.access(os.path.dirname(exportToTXTFile), os.W_OK): self.__exportResultsToTXT(results, exportToTXTFile) #open location in Google Maps with default browser if googleMaps and type(results) is IpGeoLocation: self.Utils.openLocationInGoogleMaps(results) return True except UserAgentFileEmptyError: self.Logger.PrintError("User-Agent strings file is empty!") except InvalidTargetError: self.Logger.PrintError('Please provide a valid Domain or IP address!') except TargetsFileEmptyError: self.Logger.PrintError('Targets file is empty!') except UserAgentFileNotSpecifiedError: self.Logger.PrintError('User-Agent strings file has not been provided!') except TargetsFileNotSpecifiedError: self.Logger.PrintError('Targets file has not been provided!') except ProxyServerNotReachableError: self.Logger.PrintError('Proxy server not reachable!') except ProxiesFileNotSpecifiedError: self.Logger.PrintError('Proxies file has not been provided!') except ProxiesFileEmptyError: self.Logger.PrintError('Proxies file is empty!') except InvalidProxyUrlError: self.Logger.PrintError('Proxy URL is not valid!') except Exception as error: self.Logger.PrintError('An unexpected error occurred {}!'.format(error)) return False def __checkProxyUrl(self, url): """Check if proxy url is valid""" url_checked = urlparse(url) if (url_checked.scheme not in ('http', 'https')) | (url_checked.netloc == ''): return False return url_checked def __configureProxy(self, proxy): #proxy = self.__checkProxyUrl(proxy) #if not proxy: # raise MyExceptions.InvalidProxyUrlError() self.Utils.checkProxyConn(self.URL, proxy.netloc) self.Proxy = proxy proxyHandler = request.ProxyHandler({'http':proxy.scheme + '://' + proxy.netloc}) opener = request.build_opener(proxyHandler) request.install_opener(opener) self.Logger.Print('Proxy ({}) has been configured.'.format(proxy.scheme + '://' + proxy.netloc)) def __exportResultsToCSV(self, objToExport, csvFile): """Export results to csv file""" fileExporter = FileExporter() self.Logger.Print('Saving results to {} CSV file.'.format(csvFile)) success = False if type(objToExport) is IpGeoLocation: success = fileExporter.ExportToCSV(objToExport, csvFile) elif type(objToExport) is list: success = fileExporter.ExportListToCSV(objToExport, csvFile) if not success: self.Logger.PrintError('Saving results to {} CSV file failed.'.format(csvFile)) def __exportResultsToXML(self, objToExport, xmlFile): """Export results to xml file""" fileExporter = FileExporter() self.Logger.Print('Saving results to {} XML file.'.format(xmlFile)) success = False if type(objToExport) is IpGeoLocation: success = fileExporter.ExportToXML(objToExport, xmlFile) elif type(objToExport) is list: success = fileExporter.ExportListToXML(objToExport, xmlFile) if not success: self.Logger.PrintError('Saving results to {} XML file failed.'.format(xmlFile)) def __exportResultsToTXT(self, objToExport, txtFile): """Export results to text file""" fileExporter = FileExporter() self.Logger.Print('Saving results to {} text file.'.format(txtFile)) success = False if type(objToExport) is IpGeoLocation: success = fileExporter.ExportToTXT(objToExport, txtFile) elif type(objToExport) is list: success = fileExporter.ExportListToTXT(objToExport, txtFile) if not success: self.Logger.PrintError('Saving results to {} text file failed.'.format(txtFile)) def __retrieveGeolocations (self): """Retrieve IP Geolocation for each target in the list""" IpGeoLocObjs = [] for target in self.Targets: IpGeoLocObjs.append(self.__retrieveGeolocation(target)) if len(self.Targets)>=150: sleep(.500) #1/2 sec - ip-api will automatically ban any IP address doing over 150 requests per minute return IpGeoLocObjs def __retrieveGeolocation(self, target): """Retrieve IP Geolocation for single target""" if not target: query = 'My IP' target='' elif self.Utils.isValidIPAddress(target): query = target else: ip = self.Utils.hostnameToIP(target)#domain? if not ip: raise InvalidTargetError() query = target target = ip #pick random user-agent string if self.UserAgentFile: self.__pickRandomUserAgent() #pick random proxy connection if self.ProxiesFile: self.__pickRandomProxy() self.Logger.Print('Retrieving {} Geolocation..'.format(query)) req = request.Request(self.RequestURL.format(target), data=None, headers={ 'User-Agent':self.UserAgent }) response = request.urlopen(req) if response.code == 200: self.Logger.Print('User-Agent used: {}'.format(self.UserAgent)) encoding = response.headers.get_content_charset() ipGeoLocObj = IpGeoLocation(query, json.loads(response.read().decode(encoding))) self.Logger.Print('Geolocation information has been retrieved for {}({}).'.format(query, ipGeoLocObj.IP)) if not self.NoPrint: self.Logger.PrintIPGeoLocation(ipGeoLocObj) return ipGeoLocObj return False def __loadProxies(self): """Load proxies from file""" if not self.ProxiesFile: raise ProxiesFileNotSpecifiedError() self.Proxies = [line.strip() for line in open(self.ProxiesFile, 'r') if line.strip()] self.Logger.Print('{} Proxies loaded.'.format(len(self.Proxies))) if len(self.Proxies) == 0: raise ProxiesFileEmptyError() def __loadUserAgents(self): """Load user-agent strings from file""" if not self.UserAgentFile: raise UserAgentFileNotSpecifiedError() self.UserAgents = [line.strip() for line in open(self.UserAgentFile, 'r') if line.strip()] self.Logger.Print('{} User-Agent strings loaded.'.format(len(self.UserAgents))) if len(self.UserAgents) == 0: raise UserAgentFileEmptyError() def __loadTargets(self): """Load targets from file""" if not self.TargetsFile: raise TargetsFileNotSpecifiedError() self.Targets = [line.strip() for line in open(self.TargetsFile, 'r') if line.strip()] self.Logger.Print('{} Targets loaded.'.format(len(self.Targets))) if len(self.Targets) == 0: raise TargetsFileEmptyError() def __pickRandomProxy(self): """Pick randomly a proxy from the list""" if not self.Proxies or len(self.Proxies) == 0: raise ProxiesFileEmptyError() self.__configureProxy(random.choice(self.Proxies)) def __pickRandomUserAgent(self): """Pick randomly a user-agent string from the list""" if not self.UserAgents or len(self.UserAgents) == 0: raise UserAgentFileEmptyError() self.UserAgent = random.choice(self.UserAgents)