def buildClassifier(self, instances: Instances):
self.getCapabilities().testWithFail(instances)
sumOfWeights = 0
self.m_Class = instances.classAttribute()
self.m_ClassValue = 0
attrType = instances.classAttribute().type()
if attrType == Attribute.NUMERIC:
self.m_Counts = None
elif attrType == Attribute.NOMINAL:
self.m_Counts = []
for i in range(instances.numClasses()):
self.m_Counts.append(1)
sumOfWeights = instances.numClasses()
for instance in instances:
classValue = instance.classValue()
if not Utils.isMissingValue(classValue):
if instances.classAttribute().isNominal():
self.m_Counts[classValue] += instance.weight()
else:
self.m_ClassValue += instance.weight() * classValue
sumOfWeights += instance.weight()
if instances.classAttribute().isNumeric():
if Utils.gr(sumOfWeights, 0):
self.m_ClassValue /= sumOfWeights
else:
self.m_ClassValue = Utils.maxIndex(self.m_Counts)
Utils.normalize(self.m_Counts, sumOfWeights)
def makeDistribution(self,neighbours:Instances,distances:List)->List[float]:
distribution=[0]*self.m_NumClasses
total=0
if self.m_ClassType == Attribute.NOMINAL:
for i in range(self.m_NumClasses):
distribution[i]=1/max(1,self.m_Train.numInstances())
total=self.m_NumClasses/max(1,self.m_Train.numInstances())
for i in range(neighbours.numInstances()):
current=neighbours.instance(i)
distances[i]=distances[i]*distances[i]
distances[i]=math.sqrt(distances[i]/self.m_NumAttributesUsed)
if self.DistanceWeighting == self.WEIGHT_INVERSE:
weight=1/distances[i]
elif self.DistanceWeighting == self.WEIGHT_SIMILARITY:
weight=1-distances[i]
else:
weight=1
weight*=current.weight()
if self.m_ClassType == Attribute.NOMINAL:
distribution[int(current.classValue())]+=weight
elif self.m_ClassType == Attribute.NUMERIC:
distribution[0]+=current.classValue()*weight
total+=weight
if total > 0:
Utils.normalize(distribution, total)
return distribution
def splitCritValue(self,
bags: Distribution,
totalNoInst: float = None,
numerator: float = None):
if totalNoInst is None and numerator is None:
numerator = self.oldEnt(bags) - self.newEnt(bags)
if Utils.equal(numerator, 0):
return float('inf')
denumerator = self.splitEnt(bags)
if Utils.equal(denumerator, 0):
return float('inf')
return denumerator / numerator
elif numerator is None:
res = 0
noUnkown = totalNoInst - bags.total()
if Utils.gr(bags.total(), 0):
for i in range(bags.numBags()):
res = res - self.lnFunc(bags.perBag(i))
res = res - self.lnFunc(noUnkown)
res = res + self.lnFunc(totalNoInst)
return res / math.log(2)
else:
denumerator = self.splitEnt(bags, totalNoInst)
if Utils.equal(denumerator, 0):
return 0
denumerator /= totalNoInst
return numerator / denumerator
def split(self,data:Instances)->List[Instances]:
subsetSize=[0]*self.m_numSubsets
for inst in data:
subset=self.whichSubset(inst)
if subset > -1:
subsetSize[subset]+=1
else:
weights=self.weights(inst)
for j in range(self.m_numSubsets):
if Utils.gr(weights[j], 0):
subsetSize[j]+=1
instances=[] #type:List[Instances]
for j in range(self.m_numSubsets):
instances.append(Instances(data,subsetSize[j]))
for inst in data:
subset=self.whichSubset(inst)
if subset > -1:
instances[subset].add(inst)
else:
weights=self.weights(inst)
for j in range(self.m_numSubsets):
if Utils.gr(weights[j], 0):
instances[j].add(inst)
instances[j].lastInstance().setWeight(float(weights[j]*inst.weight()))
return instances
def createMenu(self):
menu = QMenu()
showClassifierErrors = menu.addAction(
u"Visualize classifier errors") #type:QAction
showVisualizeTree = menu.addAction(u"Visualize tree") #type:QAction
selectedNames = [i.text() for i in self.m_History.selectedItems()]
o = None # type:List
if selectedNames is not None and len(selectedNames) == 1:
Utils.debugOut("history_name: ", selectedNames)
o = self.m_History.getNamedObject(selectedNames[0])
temp_vp = None # type:VisualizePanel
temp_grph = None # type:str
if o is not None:
for i in range(len(o)):
temp = o[i]
if isinstance(temp, VisualizePanel):
temp_vp = temp
elif isinstance(temp, str):
temp_grph = temp
if temp_vp is not None:
showClassifierErrors.setEnabled(True)
showClassifierErrors.triggered.connect(
self.classifierErrorTrigger(temp_vp))
else:
showClassifierErrors.setEnabled(False)
if temp_grph is not None:
showVisualizeTree.setEnabled(True)
showVisualizeTree.triggered.connect(
self.visualizeTreeTrigger(temp_grph))
else:
showVisualizeTree.setEnabled(False)
menu.exec_(QCursor.pos())
def computeAverageClassValues(self):
avgClassValues = [[]
for i in range(self.getInputFormat().numAttributes())
]
self.m_Indices = [[]
for i in range(self.getInputFormat().numAttributes())
]
for j in range(self.getInputFormat().numAttributes()):
att = self.getInputFormat().attribute(j)
if att.isNominal():
avgClassValues[j] = [0] * att.numValues()
counts = [0] * att.numValues()
for i in range(self.getInputFormat().numInstances()):
instance = self.getInputFormat().instance(i)
if not instance.classIsMissing(
) and not instance.isMissing(j):
counts[int(instance.value(j))] += instance.weight()
avgClassValues[j][int(instance.value(
j))] += instance.weight() * instance.weight()
sums = sum(avgClassValues[j])
totalCounts = sum(counts)
if Utils.gr(totalCounts, 0):
for k in range(att.numValues()):
if Utils.gr(counts[k], 0):
avgClassValues[j][k] /= counts[k]
else:
avgClassValues[j][k] = sums / totalCounts
self.m_Indices[j] = Utils.sortDouble(avgClassValues[j])
def addUndoPoint(self):
if self.m_Data is not None:
temp = tempfile.TemporaryFile()
pickle.dump(self.m_Data, temp)
temp.seek(0)
self.m_UndoList.append(temp)
self.state_changed_signal.emit()
Utils.debugOut("Now undo list len:", len(self.m_UndoList))
def dumpLabel(self,index:int,data:Instances):
text=""
text+=data.classAttribute().value(self.m_distribution.maxClass(index))
text+=" ("+str(Utils.roundDouble(self.m_distribution.perBag(index), 2))
if Utils.gr(self.m_distribution.numIncorrect(index), 0):
text+="/"+str(Utils.roundDouble(self.m_distribution.numIncorrect(index), 2))
text+=")"
return text
def signal_handler(signal, frame):
"""
Used to catch a keyboard signal like Ctrl+C in order to kill the kalliope program
:param signal: signal handler
:param frame: execution frame
"""
print "\n"
Utils.print_info("Ctrl+C pressed. Killing Kalliope")
sys.exit(0)
def rightSide(self, index: int, data: Instances):
text = ""
if data.attribute(self.m_attIndex).isNominal():
text += " = " + data.attribute(self.m_attIndex).value(index)
elif index == 0:
text += " <= " + Utils.doubleToString(self.m_splitPoint, 6)
else:
text += " > " + Utils.doubleToString(self.m_splitPoint, 6)
return text
def insertInstanceEvent(self, inst: Instances, row: int):
self.m_Pass = True
self.m_Table.insertRow(row)
self.m_Table.setItem(row, 0, QTableWidgetItem(str(row + 1)))
self.m_Table.setRawItem(inst, row)
self.adjustRowNo(row)
Utils.debugOut("insert after current instance count:",
self.model.getInstance().numInstances())
self.m_Pass = False
def maxBag(self):
max = 0
maxIndex = -1
for i in range(len(self.m_perBag)):
if Utils.gr(self.m_perBag[i], max) or Utils.equal(
self.m_perBag[i], max):
max = self.m_perBag[i]
maxIndex = i
return maxIndex
def check(self, minNoObj: float):
counter = 0
for i in range(len(self.m_perBag)):
if Utils.gr(self.m_perBag[i], minNoObj) or Utils.equal(
self.m_perBag[i], minNoObj):
counter += 1
if counter > 1:
return True
return False
def prob(self, classIndex: int, intIndex: int = None):
if intIndex is None:
if not Utils.equal(self.totaL, 0):
return self.m_perClass[classIndex] / self.totaL
return 0
else:
if Utils.gr(self.m_perBag[intIndex], 0):
return self.m_perClassPerBag[intIndex][
classIndex] / self.m_perBag[intIndex]
return self.prob(classIndex)
def prune(self):
if not self.m_isLeaf:
for i in range(len(self.m_sons)):
self.son(i).prune()
if Utils.gr(self.errorsForTree(),
self.errorsForLeaf()) or Utils.equal(
self.errorsForTree(), self.errorsForLeaf()):
self.m_sons = None
self.m_isLeaf = None
self.m_localModel = NoSplit(self.localModel().distribution())
def load_stt_plugin(self):
if self.stt is None:
self.stt_module_name = self.settings.default_stt_name
for stt_object in self.settings.stts:
if stt_object.name == self.stt_module_name:
stt_object.parameters["callback"] = self.callback
Utils.get_dynamic_class_instantiation('stt',
stt_object.name.capitalize(),
parameters=stt_object.parameters)
def evaluateClusterer(self, test: Instances, outputModel: bool):
i = loglk = unclusteredInstances = 0
cc = self.m_Clusterer.numberOfClusters()
self.m_numClusters = cc
instanceStats = [0] * cc
hasClass = test.classIndex() >= 0
clusterAssignments = []
filter = None #type:Filter
testRaw = copy.deepcopy(test)
testRaw.setClassIndex(test.classIndex())
if hasClass:
if testRaw.classAttribute().isNumeric():
raise Exception(unclusteredInstances)
filter = Remove()
filter.setAttributeIndices(str(testRaw.classIndex() + 1))
filter.setInvertSelection(False)
filter.setInputFormat(testRaw)
for inst in testRaw:
if filter is not None:
filter.input(inst)
filter.batchFinished()
inst = filter.output()
cnum = self.m_Clusterer.clusterInstance(inst)
clusterAssignments.append(cnum)
if cnum != -1:
instanceStats[cnum] += 1
sumNum = sum(instanceStats)
loglk /= sumNum
self.m_logL = loglk
self.m_clusterAssignments = []
# for i in clusterAssignments:
# print(",",i,end="")
# print()
for i in range(len(clusterAssignments)):
self.m_clusterAssignments.append(clusterAssignments[i])
numInstFieldWidth = int(
math.log(len(clusterAssignments)) / math.log(10) + 1)
if outputModel:
self.m_clusteringResult += str(self.m_Clusterer)
self.m_clusteringResult += "Clustered Instances\n\n"
clustFieldWidth = int((math.log(cc) / math.log(10)) + 1)
for i in range(cc):
if instanceStats[i] > 0:
self.m_clusteringResult+= Utils.doubleToString(i, clustFieldWidth, 0) \
+" " \
+ Utils.doubleToString(instanceStats[i], numInstFieldWidth, 0) \
+"(" + Utils.doubleToString((instanceStats[i] / sumNum * 100), 3, 0) \
+"%)\n"
if unclusteredInstances > 0:
self.m_clusteringResult += "\nUnclustered instances : " + str(
unclusteredInstances)
if hasClass:
self.evaluateClustersWithRespectToClass(test)
def useFilter(cls, data: Instances, filter: 'Filter'):
for i in range(data.numInstances()):
filter.input(data.instance(i))
filter.batchFinished()
newData = filter.getOutputFormat()
Utils.debugOut("Queue size:", filter.m_OutputQueue.qsize())
processed = filter.output()
while processed is not None:
newData.add(processed)
processed = filter.output()
return newData
def parsePost(self, post):
# View pretty json - http://jsonformatter.curiousconcept.com/
print(post)
pid = post["id"]
user = post["user"]["username"]
name = post["user"]["full_name"]
tags = post["tags"]
caption = post["caption"]
location = post["location"]
created_time = datetime.fromtimestamp(int(post["created_time"])).strftime('%Y-%m-%d %H:%M:%S')
images = post["images"]
photo = images["standard_resolution"]["url"]
profile_picture = post["user"]["profile_picture"]
likes = post["likes"]
comments = post["comments"]
text = ""
if caption:
text = caption["text"]
if location:
latitude = location["latitude"]
longitude = location["longitude"]
print("[*] Post ID: %s" % pid)
print('[*] User: %s' % user)
print('[*] Full name: %s' % name)
print('[*] Date of creation: %s' % created_time)
print('[*] Tags: %s' % tags)
print('[*] Latitude: %s - Longitude: %s' % (latitude, longitude))
print('[*] Photo: %s' % (photo))
print('[*] Text: %s' % text)
print('')
utils = Utils()
person = utils.get_user(user, name, profile_picture)
p = utils.save_post(pid, person, created_time, photo, text, latitude, longitude, "Instagram")
utils.save_tags(tags, p)
if len(likes["data"]) > 0:
c = utils.save_connection(p, 0)
for like in likes["data"]:
person_who_likes = utils.get_user(like["username"], like["full_name"], like["profile_picture"])
c.user.add(person_who_likes)
print('[+] Like from: %s' % (person_who_likes.login))
if comments["count"] > 0:
for comment in comments["data"]:
c = utils.save_connection(p, 1, comment["text"])
person_who_comments = utils.get_user(comment["from"]["username"], comment["from"]["full_name"], comment["from"]["profile_picture"])
c.user.add(person_who_comments)
print('[+] Comment from: %s' % person_who_comments.login)
print('')
return created_time
def __init__(self, target, logger, noprint=False, nolog=False, verbose=False):
self.URL = 'http://ip-api.com'
self.RequestURL = self.URL + '/json/{}'
self.BOLD = '\033[1m'
self.Proxy = None
self.UserAgentFile = None
self.UserAgents = None
self.Proxies = None
self.TargetsFile = None
self.ProxiesFile = None
self.Targets = None
self.NoPrint = noprint
self.Target = target
self.Logger = logger
self.Utils = Utils(nolog, verbose)
def getCurve(self, predictions: List[Prediction],
classIndex: int) -> Instances:
if len(predictions) == 0 or len(
predictions[0].distribution()) <= classIndex:
return None
totPos = totNeg = 0
probs = self.getProbabilities(predictions, classIndex)
for i in range(len(probs)):
pred = predictions[i]
if pred.actual() == Utils.missingValue():
continue
if pred.weight() < 0:
continue
if pred.actual() == classIndex:
totPos += pred.weight()
else:
totNeg += pred.weight()
insts = self.makeHeader()
sorted = Utils.sortDouble(probs)
tc = TwoClassStats(totPos, totNeg, 0, 0)
threshold = cumulativePos = cumulativeNeg = 0
for i in range(len(sorted)):
if i == 0 or probs[sorted[i]] > threshold:
tc.setTruePositive(tc.getTruePositive() - cumulativePos)
tc.setFalseNegative(tc.getFalseNegative() + cumulativePos)
tc.setFalsePositive(tc.getFalsePositive() - cumulativeNeg)
tc.setTrueNegative(tc.getTrueNegative() + cumulativeNeg)
threshold = probs[sorted[i]]
insts.add(self.makeInstance(tc, threshold))
cumulativePos = 0
cumulativeNeg = 0
if i == len(sorted) - 1:
break
pred = predictions[sorted[i]]
if pred.actual() == Utils.missingValue():
continue
if pred.weight() < 0:
continue
if pred.actual() == classIndex:
cumulativePos += pred.weight()
else:
cumulativeNeg += pred.weight()
if tc.getFalseNegative() != totPos or tc.getTrueNegative() != totNeg:
tc = TwoClassStats(0, 0, totNeg, totPos)
threshold = probs[sorted[-1]] + 10e-6
insts.add(self.makeInstance(tc, threshold))
return insts
def clusterInstance(self, instance: Instance):
dist = self.distributionForInstance(instance)
if dict is None:
raise Exception("Null distribution predicted")
if sum(dist) <= 0:
raise Exception("Unable to cluster instance")
return Utils.maxIndex(dist)
def truePositiveRate(self,classIndex:int):
correct=total=0
for j in range(self.m_NumClasses):
if j == classIndex:
correct+=self.m_ConfusionMatrix[classIndex][j]
total+=self.m_ConfusionMatrix[classIndex][j]
return Utils.division(correct, total)
def precision(self,classIndex:int):
correct=total=0
for i in range(self.m_NumClasses):
if i == classIndex:
correct+=self.m_ConfusionMatrix[i][classIndex]
total+=self.m_ConfusionMatrix[i][classIndex]
return Utils.division(correct, total)
def areaUnderROC(self,classIndex:int):
if self.m_Predictions is None:
return Utils.missingValue()
else:
tc=ThresholdCurve()
result=tc.getCurve(self.m_Predictions,classIndex)
return ThresholdCurve.getROCArea(result)
def classifyInstance(self,instance:Instance):
dist=self.distributionForInstance(instance)
if dist is None:
raise Exception("Null distribution predicted")
if instance.classAttribute().type() == Attribute.NOMINAL:
max=maxIndex=0
for i in range(len(dist)):
if dist[i]>max:
maxIndex=i
max=dist[i]
if max > 0:
return maxIndex
return Utils.missingValue()
elif instance.classAttribute().type() == Attribute.NUMERIC or instance.classAttribute().type() == Attribute.DATE:
return dist[0]
return Utils.missingValue()
def exceptions(e):
"""
logger = Logger()
logger.pid = Session.getPid()
logger.user = Session.getUserDisplay()
# ----------------------------------------------------
# Trace dans l'activité d'une erreur dans activity.log
logger.error('{APP_NAME} à rencontré une erreur'.format(APP_NAME=cfg._APP_NAME))
# ----------------------------------------------------
# Trace de l'exception dans un fichier à part
# import traceback
# traceback.print_exc()
logger.critical(
nom=exc_type.__name__,
message=str(exc_value),
trace=exc_tb
)
"""
# Recuperation ERREUR et trace dans Activité
exc_type, exc_value, exc_tb = sys.exc_info()
# print(exc_value)
# loggerAct.exception(e)
# Renvoi Erreur
if Utils.isAjaxRequest(request) is True:
return Render.jsonTemplate(
operation='OOPS !! Une erreur est arrivé',
message='{MESSAGE}'.format(MESSAGE=exc_value),
categorie="ERROR",
code=500)
return Render.htmlTemplate("errors/{}.html".format(str(500)),
data=None,
code=500)
def setValueAt(self, value, rowIndexList: List, columnIndex: int):
self.addUndoPoint()
for rowIndex in rowIndexList:
type = self.getType(columnIndex)
index = columnIndex - 1
inst = self.m_Data.instance(rowIndex)
att = inst.attribute(index)
if value is None:
inst.setValue(index, Utils.missingValue())
else:
if type == Attribute.NUMERIC:
inst.setValue(index, float(value))
elif type == Attribute.NOMINAL:
if att.indexOfValue(value) > -1:
inst.setValue(index, att.indexOfValue(value))
# print(att.indexOfValue(value))
else:
rowIndexList = []
break
elif type == Attribute.STRING:
inst.setValue(index, value)
elif type == Attribute.DATE:
#TODO
pass
self.update_instance_value_signal.emit(rowIndexList, columnIndex,
value)
def after_request(response):
"""Tracage des requetes URL."""
# Cela évite la duplication du registre dans le journal,
# puisque ce 500 est déjà connecté via @app.errorhandler.
if (response.status_code != 500
and not request.full_path.startswith('/static')
# and cfg._ENV[cfg._ENVIRONNEMENT]["LOG_WATCHER"] is True
):
# Construction du Message
text = "{} :: {} :: {} :: {} :: {} :: {} :: {} :: {}".format(
request.remote_addr,
Session.getUserDisplay()
if Session.getUserDisplay() is not None else "",
Session.getUserId() if Session.getUserId() is not None else "",
request.method, request.scheme, request.full_path,
response.status, '1' if Utils.isAjaxRequest(request) else '0')
# Enregistrement
# loggerWatch.info(text)
"""
file_object = open(cfg._ENV[cfg._ENVIRONNEMENT]["LOG_WATCHER_FILE"], "a+")
file_object.write("{} :: {} :: {} :: {} :: {} :: {} :: {} :: {}\n".format(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f'),
request.remote_addr,
Session.getUserDisplay() if Session.getUserDisplay() is not None else "",
Session.getUserId() if Session.getUserId() is not None else "",
request.method,
request.scheme,
request.full_path,
response.status
))
file_object.close()
"""
return response
def addUser():
if request.method == 'POST':
# Recuperation + traitement des données du formulaire
data = Utils.parseForm(dict(request.form))
# Cryptage des mots des passe + Decryptage des id
data["password"] = Crypt.encode(cfg._APP_SECRET_KEY, data["password"])
data["id"] = Crypt.decode(cfg._APP_SECRET_KEY, data["id"])
mdl = AuthUsersModel(cfg._BDD_PATH)
# Verification
if mdl.is_unique_username(data["username"]) is False:
return Render.jsonTemplate(
_OPERATION,
"Un utilisateur avec cet identifiant existe déjà",
categorie="WARNING")
# Traitement en BDD
try:
mdl.insertUser(data)
except SqliteAdapterException as errorSQL:
return Render.jsonTemplate(
_OPERATION,
'Ajout Utilisateur Impossible : {}'.format(str(errorSQL)),
categorie="ERROR")
else:
return Render.jsonTemplate(_OPERATION,
'Ajout Utilisateur',
categorie="SUCCESS")
else:
abort(400)
def updateParam():
if request.method == 'POST':
# Recuperation + traitement des données du formulaire
data = Utils.parseForm(dict(request.form))
# Initialisation de la Config
config = Configuration.from_filepath()
# Modification de la config
if 'time_zone' in data:
config.set_time_zone(data['time_zone'])
if 'hour_start' in data:
config.set_hour_start(data['hour_start'])
if 'hour_end' in data:
config.set_hour_end(data['hour_end'])
if 'filepathCredentials' in data:
config.set_filepathCredentials(data['filepathCredentials'])
if 'use_demo_account' in data:
config.set_use_demo_account(True if data['use_demo_account'] ==
'true' else False)
if 'spin_interval' in data:
config.set_spin_interval(data['spin_interval'])
# Envoi du message
return Render.jsonTemplate(_OPERATION,
'Enregistrement de la Configuration',
categorie="SUCCESS")
else:
abort(400)
def __init__(self, target, logger, noprint=False, nolog=False, verbose=False):
self.URL = 'http://ip-api.com'
self.RequestURL = self.URL + '/json/{}'
self.BOLD = '\033[1m'
self.Proxy = None
self.UserAgentFile = None
self.UserAgents = None
self.Proxies = None
self.TargetsFile = None
self.ProxiesFile = None
self.Targets = None
self.NoPrint = noprint
self.Target = target
self.Logger = logger
self.Utils = Utils(nolog, verbose)
class IpGeoLocationLib:
"""Retrieve IP Geolocation information from http://ip-api.com"""
def __init__(self, target, logger, noprint=False, nolog=False, verbose=False):
self.URL = 'http://ip-api.com'
self.RequestURL = self.URL + '/json/{}'
self.BOLD = '\033[1m'
self.Proxy = None
self.UserAgentFile = None
self.UserAgents = None
self.Proxies = None
self.TargetsFile = None
self.ProxiesFile = None
self.Targets = None
self.NoPrint = noprint
self.Target = target
self.Logger = logger
self.Utils = Utils(nolog, verbose)
def GetInfo(self, userAgent, targetsFile=None,
userAgentFile=None, proxy=False, proxiesFile=None,
exportToCSVFile=None, exportToXMLFile=None,
exportToTXTFile=None, googleMaps=False):
"""Retrieve information"""
self.UserAgent = userAgent
try:
#check proxies file and load it
if proxiesFile and os.path.isfile(proxiesFile) and os.access(proxiesFile, os.R_OK):
self.ProxiesFile = proxiesFile
self.Logger.Print('Loading Proxies from file {}..'.format(self.ProxiesFile))
self.__loadProxies()
#check user-agent strings file and load it
if userAgentFile and os.path.isfile(userAgentFile) and os.access(userAgentFile, os.R_OK):
self.UserAgentFile = userAgentFile
self.Logger.Print('Loading User-Agent strings from file {}..'.format(self.UserAgentFile))
self.__loadUserAgents()
#check targets file and load it
if targetsFile and os.path.isfile(targetsFile) and os.access(targetsFile, os.R_OK):
self.TargetsFile = targetsFile
self.Logger.Print('Loading targets from file {}..'.format(self.TargetsFile))
self.__loadTargets()
#check if proxy valid and configure connection
if proxy:
self.__configureProxy(proxy)
#retrieve information
results = None
if self.TargetsFile:
results = self.__retrieveGeolocations()
else:
results = self.__retrieveGeolocation(self.Target)
#export information
if exportToCSVFile and not os.path.exists(exportToCSVFile) and os.access(os.path.dirname(exportToCSVFile), os.W_OK):
self.__exportResultsToCSV(results, exportToCSVFile)
if exportToXMLFile and not os.path.exists(exportToXMLFile) and os.access(os.path.dirname(exportToXMLFile), os.W_OK):
self.__exportResultsToXML(results, exportToXMLFile)
if exportToTXTFile and not os.path.exists(exportToTXTFile) and os.access(os.path.dirname(exportToTXTFile), os.W_OK):
self.__exportResultsToTXT(results, exportToTXTFile)
#open location in Google Maps with default browser
if googleMaps and type(results) is IpGeoLocation:
self.Utils.openLocationInGoogleMaps(results)
return True
except UserAgentFileEmptyError:
self.Logger.PrintError("User-Agent strings file is empty!")
except InvalidTargetError:
self.Logger.PrintError('Please provide a valid Domain or IP address!')
except TargetsFileEmptyError:
self.Logger.PrintError('Targets file is empty!')
except UserAgentFileNotSpecifiedError:
self.Logger.PrintError('User-Agent strings file has not been provided!')
except TargetsFileNotSpecifiedError:
self.Logger.PrintError('Targets file has not been provided!')
except ProxyServerNotReachableError:
self.Logger.PrintError('Proxy server not reachable!')
except ProxiesFileNotSpecifiedError:
self.Logger.PrintError('Proxies file has not been provided!')
except ProxiesFileEmptyError:
self.Logger.PrintError('Proxies file is empty!')
except InvalidProxyUrlError:
self.Logger.PrintError('Proxy URL is not valid!')
except Exception as error:
self.Logger.PrintError('An unexpected error occurred {}!'.format(error))
return False
def __checkProxyUrl(self, url):
"""Check if proxy url is valid"""
url_checked = urlparse(url)
if (url_checked.scheme not in ('http', 'https')) | (url_checked.netloc == ''):
return False
return url_checked
def __configureProxy(self, proxy):
#proxy = self.__checkProxyUrl(proxy)
#if not proxy:
# raise MyExceptions.InvalidProxyUrlError()
self.Utils.checkProxyConn(self.URL, proxy.netloc)
self.Proxy = proxy
proxyHandler = request.ProxyHandler({'http':proxy.scheme + '://' + proxy.netloc})
opener = request.build_opener(proxyHandler)
request.install_opener(opener)
self.Logger.Print('Proxy ({}) has been configured.'.format(proxy.scheme + '://' + proxy.netloc))
def __exportResultsToCSV(self, objToExport, csvFile):
"""Export results to csv file"""
fileExporter = FileExporter()
self.Logger.Print('Saving results to {} CSV file.'.format(csvFile))
success = False
if type(objToExport) is IpGeoLocation:
success = fileExporter.ExportToCSV(objToExport, csvFile)
elif type(objToExport) is list:
success = fileExporter.ExportListToCSV(objToExport, csvFile)
if not success:
self.Logger.PrintError('Saving results to {} CSV file failed.'.format(csvFile))
def __exportResultsToXML(self, objToExport, xmlFile):
"""Export results to xml file"""
fileExporter = FileExporter()
self.Logger.Print('Saving results to {} XML file.'.format(xmlFile))
success = False
if type(objToExport) is IpGeoLocation:
success = fileExporter.ExportToXML(objToExport, xmlFile)
elif type(objToExport) is list:
success = fileExporter.ExportListToXML(objToExport, xmlFile)
if not success:
self.Logger.PrintError('Saving results to {} XML file failed.'.format(xmlFile))
def __exportResultsToTXT(self, objToExport, txtFile):
"""Export results to text file"""
fileExporter = FileExporter()
self.Logger.Print('Saving results to {} text file.'.format(txtFile))
success = False
if type(objToExport) is IpGeoLocation:
success = fileExporter.ExportToTXT(objToExport, txtFile)
elif type(objToExport) is list:
success = fileExporter.ExportListToTXT(objToExport, txtFile)
if not success:
self.Logger.PrintError('Saving results to {} text file failed.'.format(txtFile))
def __retrieveGeolocations (self):
"""Retrieve IP Geolocation for each target in the list"""
IpGeoLocObjs = []
for target in self.Targets:
IpGeoLocObjs.append(self.__retrieveGeolocation(target))
if len(self.Targets)>=150:
sleep(.500) #1/2 sec - ip-api will automatically ban any IP address doing over 150 requests per minute
return IpGeoLocObjs
def __retrieveGeolocation(self, target):
"""Retrieve IP Geolocation for single target"""
if not target:
query = 'My IP'
target=''
elif self.Utils.isValidIPAddress(target):
query = target
else:
ip = self.Utils.hostnameToIP(target)#domain?
if not ip:
raise InvalidTargetError()
query = target
target = ip
#pick random user-agent string
if self.UserAgentFile:
self.__pickRandomUserAgent()
#pick random proxy connection
if self.ProxiesFile:
self.__pickRandomProxy()
self.Logger.Print('Retrieving {} Geolocation..'.format(query))
req = request.Request(self.RequestURL.format(target), data=None, headers={
'User-Agent':self.UserAgent
})
response = request.urlopen(req)
if response.code == 200:
self.Logger.Print('User-Agent used: {}'.format(self.UserAgent))
encoding = response.headers.get_content_charset()
ipGeoLocObj = IpGeoLocation(query, json.loads(response.read().decode(encoding)))
self.Logger.Print('Geolocation information has been retrieved for {}({}).'.format(query, ipGeoLocObj.IP))
if not self.NoPrint:
self.Logger.PrintIPGeoLocation(ipGeoLocObj)
return ipGeoLocObj
return False
def __loadProxies(self):
"""Load proxies from file"""
if not self.ProxiesFile:
raise ProxiesFileNotSpecifiedError()
self.Proxies = [line.strip() for line in open(self.ProxiesFile, 'r') if line.strip()]
self.Logger.Print('{} Proxies loaded.'.format(len(self.Proxies)))
if len(self.Proxies) == 0:
raise ProxiesFileEmptyError()
def __loadUserAgents(self):
"""Load user-agent strings from file"""
if not self.UserAgentFile:
raise UserAgentFileNotSpecifiedError()
self.UserAgents = [line.strip() for line in open(self.UserAgentFile, 'r') if line.strip()]
self.Logger.Print('{} User-Agent strings loaded.'.format(len(self.UserAgents)))
if len(self.UserAgents) == 0:
raise UserAgentFileEmptyError()
def __loadTargets(self):
"""Load targets from file"""
if not self.TargetsFile:
raise TargetsFileNotSpecifiedError()
self.Targets = [line.strip() for line in open(self.TargetsFile, 'r') if line.strip()]
self.Logger.Print('{} Targets loaded.'.format(len(self.Targets)))
if len(self.Targets) == 0:
raise TargetsFileEmptyError()
def __pickRandomProxy(self):
"""Pick randomly a proxy from the list"""
if not self.Proxies or len(self.Proxies) == 0:
raise ProxiesFileEmptyError()
self.__configureProxy(random.choice(self.Proxies))
def __pickRandomUserAgent(self):
"""Pick randomly a user-agent string from the list"""
if not self.UserAgents or len(self.UserAgents) == 0:
raise UserAgentFileEmptyError()
self.UserAgent = random.choice(self.UserAgents)
