def getAllLinksOnPage(link, breadth, br, flag):
limitb = 0
newlinks = []
try:
br.open(link)
for ii in br.links():
if flag == 1:
if limitb < breadth:
u = urlparse.urljoin(ii.base_url, ii.url)
newlinks.append(u)
host = urlparse.urlparse(u).hostname #hostname
#print host
ips = socket.gethostbyname(host) #ip address
match = geolite2.lookup(ips)
country = match.country #country of the server
#print u
#print str(ips)+" "+str(country)
limitb += 1
else:
break
else:
u = urlparse.urljoin(ii.base_url, ii.url)
newlinks.append(u)
host = urlparse.urlparse(u).hostname #hostname
#print host
ips = socket.gethostbyname(host) #ip address
match = geolite2.lookup(ips)
country = match.country #country of the server
#print u
#print str(ips)+" "+str(country)
#limitb+=1
except Exception, e:
print "\nerror: " + str(e) + "\n"
def writeUniqueIPAddressWithCountry(self, IPAddressList,
uniqueIPAddressSet, filename):
print "Writing Unique IP Address with country..."
#create an array of ip address with country
uniqueIPAddressWithCountryList = []
#use geolite2 library to check if the unique ip address represents a country
for key in uniqueIPAddressSet:
match = geolite2.lookup(key)
if match is not None:
uniqueIPAddressWithCountryList.append(key)
#create a dictionary to store the number of hits for each unique ip address with country.
IPListDic = dict()
for key in uniqueIPAddressWithCountryList:
IPListDic[key] = 0
#loop the list with all IP address. Increase the count by 1 whenever the same IP address appears again
for key in IPAddressList:
value = IPListDic.get(key)
if value is not None:
value = IPListDic.get(key) + 1
IPListDic[key] = value
f = open(filename + "-unique ip country", "w")
f.writelines("IP Address, Country, No. Of Hits" + "\n")
#write the country, hits and unique ip addresses to a file
for key in IPListDic:
match = geolite2.lookup(key)
if match is not None and match.country is not None:
lineToWrite = key + "---" + match.country + "---" + str(
IPListDic.get(key))
f.writelines(lineToWrite + "\n")
continue
def _read_addr(fd):
"""
load next router address
"""
Entry._log.debug('read_addr')
addr = Address()
addr.cost = Entry._read_byte(fd)
addr.expire = Entry._read_time(fd)
addr.transport = Entry._read_string(fd)
addr.options = Entry._read_mapping(fd)
addr.firewalled = False
if addr.valid():
# This is a try because sometimes hostnames show up.
# TODO: Make it allow host names.
try:
addr.location = geolite2.lookup(addr.options.get('host', None))
except:
addr.location = None
# If the router is firewalled (i.e. has no 'host' mapping), then use the first introducer (of 3).
# In the future it might be worth it to do something else, but this helps for geopip information for now.
# http://i2p-projekt.i2p/en/docs/transport/ssu#ra
if not addr.location:
# If there are introducers then it's probably firewalled.
addr.firewalled = True
try:
addr.location = geolite2.lookup(addr.options.get('ihost0', None))
except:
addr.location = None
return addr
def experiment_smartor(history):
results_smartor = open("results_smartor.txt", "a")
relays = get_relays(controller)
entry = relays[0];
middle = relays[1];
exit = relays[2];
myIP = ipgetter.myip();
my_Address = geolite2.lookup(socket.gethostbyname(myIP))
for url in history:
dest_Address = geolite2.lookup(socket.gethostbyname(url))
if (dest_Address == None):
print("Couldn't get location of ", url)
continue
# Get list of fingerprints for exit nodes
exit_nodes = get_relays_fingerprint(num_relays, exit, dest_Address.location)
entry_nodes = get_relays_fingerprint(num_relays, entry, my_Address.location)
middleLocation = midpointCalculator(dest_Address.location, my_Address.location)
middle_nodes = get_relays_fingerprint(num_relays, middle, my_Address.location)
url = 'https://www.' + url
path_with_locations = get_best_circuit(url, controller, entry_nodes, middle_nodes, exit_nodes, 10)
if path_with_locations == -1:
continue
locations = [my_Address.location, dest_Address.location] + [x[1] for x in path_with_locations]
distance = totalDistance(locations)
best_path = [x[0] for x in path_with_locations]
print("best path ", best_path)
circuit_id = controller.new_circuit(best_path, await_build = True)
test = controller.get_circuit(circuit_id)
print 'Accessing url: ' + url
get_page(url, controller, test, results_smartor, distance)
def experiment_tor(history):
results_tor = open("results_tor.txt", "a")
myIP = ipgetter.myip();
my_Address = geolite2.lookup(socket.gethostbyname(myIP))
for url in history:
dest_Address = geolite2.lookup(socket.gethostbyname(url))
if (dest_Address == None):
print("Couldn't get location of ", url)
continue
url = 'https://www.' + url
test = controller.get_circuits()
for circuit in test:
if (len(circuit.path) > 2):
path = circuit.path
circ = circuit
break
print path
# test = path
res_list = [controller.get_network_status(x[0]).address for x in path] # Get ip addresses from fingerprints
# print res_list
locations_relay = [geolite2.lookup(x).location for x in res_list] # Do lookups
# print locations_relay
locations = [my_Address.location, dest_Address.location] + locations_relay
distance = totalDistance(locations)
time = scan_head(controller, circ, url)
if (time != -1):
results_tor.write(str(distance) + "," + str(time))
def getAllLinksOnPage(link, breadth, br, flag):
limitb = 0
newlinks = []
try:
br.open(link)
for ii in br.links():
if flag == 1:
if limitb < breadth:
u = urlparse.urljoin(ii.base_url, ii.url)
newlinks.append(u)
host = urlparse.urlparse(u).hostname # hostname
# print host
ips = socket.gethostbyname(host) # ip address
match = geolite2.lookup(ips)
country = match.country # country of the server
# print u
# print str(ips)+" "+str(country)
limitb += 1
else:
break
else:
u = urlparse.urljoin(ii.base_url, ii.url)
newlinks.append(u)
host = urlparse.urlparse(u).hostname # hostname
# print host
ips = socket.gethostbyname(host) # ip address
match = geolite2.lookup(ips)
country = match.country # country of the server
# print u
# print str(ips)+" "+str(country)
# limitb+=1
except Exception, e:
print "\nerror: " + str(e) + "\n"
def addLocation(x):
from geoip import geolite2 #
dictX = dict(x)
locSrcIp = geolite2.lookup(dictX['srcip'])
locDstIp = geolite2.lookup(dictX['dstip'])
try:
if locSrcIp and locSrcIp.location:
dictX['srclocation'] = {
'lat': locSrcIp.location[0],
'lon': locSrcIp.location[1]
}
else:
dictX['srclocation'] = {'lat': 48, 'lon': 22}
if locDstIp and locDstIp.location:
dictX['dstlocation'] = {
'lat': locSrcIp.location[0],
'lon': locSrcIp.location[1]
}
else:
dictX['dstlocation'] = {'lat': 48, 'lon': 22}
except AttributeError:
pass
except TypeError:
pass
return dictX
def _read_addr(fd):
"""
load next router address
"""
Entry._log.debug('read_addr')
addr = Address()
addr.cost = Entry._read_byte(fd)
addr.expire = Entry._read_time(fd)
addr.transport = Entry._read_string(fd)
addr.options = Entry._read_mapping(fd)
addr.firewalled = False
if addr.valid():
# This is a try because sometimes hostnames show up.
# TODO: Make it allow host names.
try:
addr.location = geolite2.lookup(addr.options.get('host', None))
except:
addr.location = None
# If the router is firewalled (i.e. has no 'host' mapping), then use the first introducer (of 3).
# In the future it might be worth it to do something else, but this helps for geopip information for now.
# http://i2p-projekt.i2p/en/docs/transport/ssu#ra
if not addr.location:
# If there are introducers then it's probably firewalled.
addr.firewalled = True
try:
addr.location = geolite2.lookup(
addr.options.get('ihost0', None))
except:
addr.location = None
return addr
def PrintResult(unique_client_ip_set, client_ip_record):
with open(os.getcwd()+"/result/[1] IP Address List.txt", "w") as f1:
f1.write("LIST OF UNIQUE IP ADDRESS\n")
for element in unique_client_ip_set:
f1.write(element+"\n")
with open(os.getcwd()+"/result/[2] IP Address Records.txt", "w") as f2:
for element in unique_client_ip_set:
if(geolite2.lookup(element) is None):
country = "N/A"
else:
country = geolite2.lookup(element).country
f2.write("IP ADDRESS: " + element + " --- COUNTRY: " + country + " --- HITS: " + str(client_ip_record[element][0]))
f2.write("\n")
with open(os.getcwd()+"/result/[3] IP Address Activity.txt", "w") as f3:
for element in unique_client_ip_set:
f3.write("\nIP ADDRESS: " + element + "\nTRANSACTION LIST:\n")
for activity in client_ip_record[element][1:]:
f3.write(activity.strip(' - ')+"\n")
with open(os.getcwd()+"/result/[4] SQL Injection.txt", "w") as f4:
f4.write("SQL INJECTION\n")
for element in unique_client_ip_set:
Flag = False
for activity in client_ip_record[element][1:]:
if(aq.detectSQLi(activity)):
Flag = True
f4.write("\nIP ADDRESS " + element + " --- REQUEST: ")
f4.write(activity.strip(' - '))
if(Flag):
f4.write("\n")
with open(os.getcwd()+"/result/[5] RFI.txt", "w") as f5:
f5.write("REMOTE FILE INCLUSIONS\n")
for element in unique_client_ip_set:
Flag = False
for activity in client_ip_record[element][1:]:
if(aq.detectRFI(activity)):
Flag = True
f5.write("\nIP ADDRESS " + element + " --- REQUEST: ")
f5.write(activity.strip(' - '))
if(Flag):
f5.write("\n")
with open(os.getcwd()+"/result/[6] WEB SHELL.txt", "w") as f6:
f6.write("WEB SHELL\n")
for element in unique_client_ip_set:
Flag = False
for activity in client_ip_record[element][1:]:
if(aq.detectWebShell(activity)):
Flag = True
f6.write("\nIP ADDRESS " + element + " --- REQUEST: ")
f6.write(activity.strip(' - '))
if(Flag):
f6.write("\n")
def get_country(request_ip):
math = geolite2.lookup(request_ip) if geolite2.lookup(
request_ip) is not None else geolite2.lookup_mine()
this_alpha = math.country
tz = str(math.timezone)
citycapital = tz[tz.index('/') + 1:]
for alpha in list(pycountry.countries):
if this_alpha.__contains__(alpha.alpha2):
this_alpha = alpha.name
return str(this_alpha + ' , ' + citycapital)
def country():
if request.method == 'GET':
match = geolite2.lookup(request.headers['X-Forwarded-For'])
return jsonify({"countries": get(match.country)})
elif request.method == 'POST':
match = geolite2.lookup(request.headers['X-Forwarded-For'])
if match is not None:
edit(match.country)
return jsonify({"message": "+1!"}), 200
else:
return jsonify({"message": "Origin IP country wasn't found."}), 500
def fetch_country_from_ip(iplist_data):
stats_dict = {}
count2 = 0
for ip in iplist_data:
if geolite2.lookup(ip) != None:
count2 += 1
country = country_codes[geolite2.lookup(ip).country]
if country not in stats_dict:
stats_dict[country] = 1
if country in stats_dict:
stats_dict[country] += 1
return [stats_dict, (len(iplist_data) - count2)]
def policy_is_valid_policy_region(self, fileid, remote_addr):
self.get_file_query(fileid)
match = geolite2.lookup(remote_addr)
if match is None:
return True
q = self.session.query(FileRegionsBlocked).filter_by(fileid=fileid).filter_by(region_code=match.country).all()
return len(q) == 0
def crawl_recursively(link, limitd, depth, breadth, br, flag):
if limitd == depth:
return
if link in crawled:
return
if limitd == 0:
host = urlparse.urlparse(link).hostname # hostname for the primary seed
ips = socket.gethostbyname(host) # ip address of its server
print host
match = geolite2.lookup(ips)
country = match.country # country of the server
print str(ips) + " " + str(country)
host = host.split(".")
host = host[-2] + "." + host[-1]
answers = dns.resolver.query(host, "NS")
for server in answers:
print str(server) + " " + socket.gethostbyname(str(server))
# print("\n"+str(limitd))
print link + "\n"
newLinks = getAllLinksOnPage(link, breadth, br, flag)
crawled.add(link) # seed
for link in newLinks:
crawl_recursively(link, limitd + 1, depth, breadth, br, flag)
return
def convert(ip):
match = geolite2.lookup(ip)
if match is None:
return 'N/a'
if match.country is None:
return 'N/a'
return match.country
def get_location_for_ip_addr(ip_addr, sqlite_db):
if ip_addr is None:
return None
cursor = sqlite_db.cursor()
query = "SELECT * FROM locations WHERE ip_addr='{}'".format(ip_addr)
cursor.execute(query)
locations = cursor.fetchall()
if len(locations) > 1:
raise Exception('Multiple locations found for single ip address')
if locations:
return locations[0]
location = geolite2.lookup(ip_addr)
query = "INSERT INTO locations (ip_addr, continent, country) VALUES ('{ip_addr}', '{continent}', '{country}');".format(
ip_addr=ip_addr,
continent=getattr(location, 'continent', None),
country=getattr(location, 'country', None),
)
cursor.execute(query)
sqlite_db.commit()
return get_location_for_ip_addr(ip_addr, sqlite_db)
def get_location_coordinates(self):
"""Get lat/lang for different cases eg: location="21st&Market,SF"
@return: latitude, longitude. In case of bounds query they are dicts
"""
log.debug("[NearbyFoodTruckHandler] Get location coordinates")
if self.query_parameter["location"]:
if self.query_parameter["location"] == "current":
match = geolite2.lookup(self.request.remote_ip)
latitude, longitude = match.location
return float(latitude), float(longitude)
else:
address, (latitude, longitude) = self.geolocator.geocode(self.query_parameter["location"])
return float(latitude), float(longitude)
elif self.query_parameter["point"]:
coordinates = self.query_parameter["point"].split(",")
latitude = coordinates[0]
longitude = coordinates[1]
return float(latitude), float(longitude)
else:
latitude = {}
longitude = {}
for idx, coordinate in enumerate(self.query_parameter["bounds"].split("|")):
latlang = coordinate.split(",")
latitude[idx] = float(latlang[0])
longitude[idx] = float(latlang[1])
return latitude, longitude
def geoip_country():
ip = flask.request.headers.get('X-Forwarded-For') or flask.request.remote_addr
try:
match = geolite2.lookup(ip)
return match.country if match else ''
except ValueError:
return ''
def flags():
con = lite.connect('/usr/share/nginx/html/flags.db')
dt = datetime.now().strftime('%Y%m%d')
d = {}
with gzip.open('/var/log/nginx/access.log-{0}.gz'.format(dt), 'r') as fin:
for line in fin:
if "mp3" in line:
ip = line.split('-')[0].rstrip()
match = geolite2.lookup(ip)
if match is not None:
c = match.country
if c in d and c:
d[c] += 1
else:
d[c] = 1
with con:
cur = con.cursor()
os.chdir('/usr/share/nginx/html/app/static/flags/')
countries = sorted(glob('*'))
for k, v in d.items():
key = countries.index('{0}.png'.format(k)) + 1
cur.execute('select total from flags where id = {0};'.format(key))
t = cur.fetchone()
total = v + t[0]
cur.execute('update flags set today = {0},total = {1} '
'where id = {2};'.format(v, total, key))
def customer_request():
print request.form
model = request.form.get('model', u'').encode('utf-8')
vin = request.form.get('vin', u'').encode('utf-8')
tel = request.form.get('tel', u'').encode('utf-8')
if not tel:
flash("Извините но не указан контакт. Заполните пожалуйста.")
return redirect(url_for('webprint'))
message = "Сообщение отослано. С вами свяжутся в ближайшее время {}. Спасибо".format(
tel)
match = geolite2.lookup(request.remote_addr)
email_message = "Bремя заявки={}\nIPaddress={}; {}\n"\
"Страна-{}\nВременная зона клиента-{}\nLat/Lon={}\n"\
"Модель={}\nVIN={}\nТелефон={}\n".format(
datetime.datetime.now().strftime("%Y-%B-%d %H:%M:%S"),
request.environ.get('REMOTE_ADDR'),
request.remote_addr,
match.country,
match.timezone,
match.location,
model, vin, tel)
subj = "Заявка с сайта bymotor.ru"
flash(message)
pylib.send_email(subj, email_message)
return redirect(url_for('webprint'))
def ip_lookup(self, ip):
# given ip, look up org, isp, lat and lon
# first, check if we have seen this ip before
if ip in self.iptable:
return self.iptable[ip]
try:
obj = IPWhois(ip, timeout=10) # times out after 10 seconds
results = obj.lookup(get_referral=True)
org = results['nets'][-1]['description']
isp = results['nets'][0]['description']
except (IPDefinedError, ASNLookupError, ASNRegistryError,
WhoisLookupError, HostLookupError, BlacklistError,
AttributeError) as e:
# log bad ip and error
logger.error('%s from IPWhois on IP %s, setting org & isp to None',
e, ip)
org = isp = None
except ValueError:
logger.error(
'Set org & isp to None, ValueError from IPWhois for IP %s', ip)
org = isp = None
# geolite2 returns NoneType if no match
try:
match = geolite2.lookup(ip)
if match:
if match.location:
if match.location[0]:
lat = match.location[0]
else:
lat = None
logger.warn(
'Set lat = None, geolite2 unable to find lat for IP %s',
ip)
if match.location[1]:
lon = match.location[1]
else:
lon = None
logger.warn(
'Set lon = None, geolite2 unable to find lon for IP %s',
ip)
else:
lat = lon = None
logger.warn(
'Set lat & lon = None, geolite2 unable to find lat/lon for IP %s',
ip)
else:
# log unable to find lat/lon for this ip
logger.warn(
'Set lat & lon = None, geolite2 unable to find lat/lon for IP %s',
ip)
lat = lon = None
except ValueError:
# log bad ip and error
logger.error(
'Set lat & lon = None, ValueError from geolite2 for IP %s', ip)
lat = lon = None
self.iptable[ip] = [org, lat, lon, isp]
return self.iptable[ip]
def post(self):
user = request.form
userJson = user.to_dict()
location = userJson["Location"]
activity = userJson["Activity"]
duration = userJson["Duration"]
ip_addrr = request.remote_addr
result = geolite2.lookup(ip_addrr)
latitude, longitude = 0.000, 0.000
if result is not None:
latitude, longitude = result.Location
if not duration:
duration = 0
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
cursor.execute("SELECT max(id) from Users") # hacky workaround for maxUserId
maxUserId = cursor.fetchone()[0]
row = (maxUserId+1, activity, location, duration, latitude, longitude)
cursor.execute("INSERT INTO Users VALUES(?, ?, ?, ?, ?, ?)", row)
conn.commit()
conn.close()
return jsonify({"Result":"Success"})
def geoip():
print C + "----------------------------------------------------------------"
print "MaxMind has provided a module that enables a user to conduct GeoIP reconaissance."
print "Users are able to find location data based on a IPv4 address"
print "----------------------------------------------------------------" + W
print "Loading..."
time.sleep(3)
location = raw_input("[>] What is the IP address? " )
try:
match = geolite2.lookup(location)
print O + "========================================"
gi = GeoIP.open("GeoLiteCity.dat", GeoIP.GEOIP_INDEX_CACHE | GeoIP.GEOIP_CHECK_CACHE)
print "Collected: " + str(gi.record_by_name(location))
print "========================================" + W
geoiptext = raw_input("[>] Would you like this saved to a text file? (Y/N) ")
if geoiptext == "y":
text_file = open("geoip_information", 'a')
text_file.write("GeoIP Information Results ")
text_file.write("\nIP Address: %s " % location)
text_file.write("\n" + str(gi.record_by_name(location)))
elif geoiptext == "n":
sys.exit(G + "Have a great one!" + W)
except ValueError:
print R + "That was not a valid IP Address! Care to try again?" + W
geoip()
def ipInfo(domain):
res = subprocess.check_output("dig %s"%(domain),shell = True)
res = res.split("\n")
res = filter(None,res)
ipDict = {}
try:
i = res.index(";; ANSWER SECTION:")
except ValueError:
ipDict['ip_list'] = 'NA'
ipDict['loc_list'] = 'NA'
return ipDict
else:
pass
ip_list = []
loc_list = []
i = i +1
while(res[i]!= ';; AUTHORITY SECTION:'):
ip = filter(None,res[i].split('\t')) [4]
ip_list.append(ip)
i = i+1
for ip in ip_list:
match = geolite2.lookup(ip)
if(match):
if(match.country in loc_list):
pass
else:
loc_list.append(match.country)
else:
loc_list.append('NA')
ipDict['ip_list'] = ip_list
ipDict['loc_list'] = loc_list
return ipDict
def get_location(ip_address):
match = geolite2.lookup(ip_address)
if match:
location_data = match.to_dict()
location_data['subdivisions'] = list(location_data['subdivisions'])
return location_data
raise ValueError("No results found for {}".format(ip_address))
def geoip_country():
ip = flask.request.headers.get('X-Forwarded-For') or flask.request.remote_addr
try:
match = geolite2.lookup(ip)
return match.country if match else ''
except ValueError:
return ''
def return_local_time(utchour):
app.logger.info(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + '] Request: GET /localhour/' + str(utchour))
if utchour == 24:
utchour = 0
if not 0 <= utchour <= 23:
app.logger.warning(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + '] Invalid utchour ' + str(utchour))
return str('{ "error": "invalid utchour ' + str(utchour) + '" }')
# Do GeoIP based on remote IP to determine TZ
try:
match = geolite2.lookup(request.remote_addr)
except Exception as e:
app.logger.error(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + '] Error: ' + str(e) + ' - whilst matching GeoIP data for IP')
return str('{ "error": "error looking up match for IP ' + str(request.remote_addr) + '" }')
# Check we got a match
if match is None:
app.logger.error(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + "] Failed to match IP to GeoIP data")
return str('{ "error": "no geoip match for IP ' + str(request.remote_addr) + '" }')
# From the match, try pulling timezone straight from geoip lookup
try:
local = timezone(match.timezone)
except UnknownTimeZoneError:
# If we can't directly find a timezone, get one based on the Country.
local = timezone(country_timezones(match.city)[0])
#local = timezone(country_timezones(match.country)[0])
except Exception as e:
return str('{ "error": "Error: ' + str(e) + ' - whilst getting timezone" }')
app.logger.info(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + '] Matched IP to timezone: ' + str(local))
local_dt = local.localize(datetime(datetime.today().year, datetime.today().month, datetime.today().day, utchour, 0, 0))
utc_dt = utc.normalize(local_dt.astimezone(utc))
app.logger.info(str(request.remote_addr) + ' [' + str(datetime.utcnow()) + '] Returning value: ' + str(utc_dt.hour) + ' for requested hour ' + str(utchour) + ' in Timezone ' + str(local))
return str('{ "hour": ' + str(utc_dt.hour) + ' }')
def ping(self):
opener = urllib2.build_opener(
urllib2.ProxyHandler({'http': self.remote}))
try:
t0 = time.time()
req = urllib2.Request('http://lumtest.com/myip')
r = opener.open(req, timeout=10)
ip = r.read().strip()
match = geolite2.lookup(ip)
ttl = time.time() - t0
r.close()
return {
'status': 'ok',
'result': {
'ip': ip,
'ttl': ttl,
'country': match.country,
'str': self.remote
}
}
except Exception, e:
return {
'status': 'error',
'result': repr(e),
'proxy': str(self.remote)
}
def ns_list(request):
page = int(request.GET.get("page",1))
page_size = 300
first,last = (page-1) * page_size, page * page_size
temp_ns_list = Ns.objects.order_by('ns_ip')[first:last]
count = Ns.objects.all().count()
locations = []
for ip in temp_ns_list:
match = geolite2.lookup(ip['ns_ip'])
latitude = 0
longitude = 0
if match and match.location:
(latitude, longitude) = match.location
temp_dict = {}
temp_dict['latitude'] = latitude
temp_dict['longitude'] = longitude
temp_dict['ip'] = ip['ns_ip']
locations.append(temp_dict)
paginator = Paginator(temp_ns_list, page_size, count)
ns_list = paginator.page(page)
context = {'ns_list': ns_list,'locations':locations, 'total': count}
return render(request, 'ns_list.html', context)
def drawGraph(groupSet): my_map = Basemap(projection='robin', lat_0=0, lon_0=0, resolution='l', area_thresh=1000.0) my_map.drawcoastlines() my_map.drawcountries() my_map.fillcontinents(color='gray') colorString = getAllTheColor() colorType = 0 for group in groupSet: lon = [] lat = [] for item in group: match = geolite2.lookup(item.getIp()) lat.append( match.location[0]) lon.append( match.location[1]) x,y = my_map(lon,lat) my_map.plot(x,y,color=colorString[colorType],marker='o',markersize=15) colorType = (colorType + 1) % len(colorString) plt.show()
def search_geoip(domain): print domain ip = socket.gethostbyname(domain) geoip_result = geolite2.lookup(ip) print geoip_result find_csv(geoip_result,domain)
def analysis(date,user,ip,passed): geo=geolite2.lookup(ip) country="unknown" gps="unknown" if geo is not None: country=geo.country gps=gps_to_str(geo.location) if passed: if not user in users_p: users_p[user]=0 users_p[user]+=1 if not ip in ip_p: ip_p[ip]=0 ip_p[ip]+=1 if not country in country_p: country_p[country]=0 country_p[country]+=1 if not gps in gps_p: gps_p[gps]=0 gps_p[gps]+=1 else: if not user in users_f: users_f[user]=0 users_f[user]+=1 if not ip in ip_f: ip_f[ip]=0 ip_f[ip]+=1 if not country in country_f: country_f[country]=0 country_f[country]+=1 if not gps in gps_f: gps_f[gps]=0 gps_f[gps]+=1
def geo_lookup(ip):
g = {}
try:
lookup = geolite2.lookup(ip)
except ValueError:
logger.info('Unable to lookup geo-info for %s' % ip)
pass
else:
if lookup:
geo = lookup.get_info_dict()
try:
g['geo.city_name'] = geo['city']['names']['en']
except:
pass
try:
g['geoip.continent_code'] = geo['continent']['names']['en']
except:
pass
try:
g['geoip.country_code'] = geo['country']['iso_code']
except:
pass
try:
g['geoip.country_name'] = geo['country']['names']['en']
except:
pass
try:
g['geoip.dma_code'] = geo['location']['metro_code']
except:
pass
try:
g['geoip.latitude'] = geo['location']['latitude']
except:
pass
try:
g['geoip.location'] = '%s, %s' % (geo['location']['latitude'],
geo['location']['longitude'])
except:
pass
try:
g['geoip.longitude'] = geo['location']['longitude']
except:
pass
try:
g['geoip.postal_code'] = geo['postal']['code']
except:
pass
try:
g['geoip.region_code'] = geo['subdivisions'][0]['iso_code']
except:
pass
try:
g['geoip.region_name'] = geo['subdivisions'][0]['names']['en']
except:
pass
try:
g['geoip.timezone'] = geo['location']['time_zone']
except:
pass
return g
def main():
params = cgi.FieldStorage()
out = {"ip": None, "country": None, "continent": None, "location": None}
header()
out["ip"] = cgi.escape(os.environ["REMOTE_ADDR"])
format = F_PLAIN
if (len(params) > 0):
for key in params:
if key == "format":
r_format = params[key].value
if r_format == "xml":
print("lol xml are you serious? get with the times")
elif r_format == "json":
format = F_JSON
elif key == "and":
extras = params[key].value.split(',')
if "geo" in extras:
from geoip import geolite2
match = geolite2.lookup(out["ip"].encode("UTF-8"))
if match is not None:
out["country"] = match.country
out["continent"] = match.continent
out["location"] = match.location
if format == F_PLAIN:
for item in out:
if out[item] is not None:
print(out[item])
elif format == F_JSON:
import json
print(json.dumps(dict(
(k, v) for k, v in out.items() if v is not None)))
def ipLocation(ip):
match = geolite2.lookup(ip)
if match is not None:
output = match.country
return output
else:
return "Unknown"
def HEADERanalyzer(line):
try:
xcounter = 0
IP = line.split(
)[options.
IPpos] # May need to be adjust, default 0 should work, combined is 1
Request = line.split('"')[1].lower()
Client = line.split('"')[-2]
logstring = str(
datetime.datetime.now()) + " " + IP + " Header: " + Client
m = re.search(blacklist, Client) # related services
i = re.search(whitelist, IP) # Whitelabeld IP's
if (m is not None) or (GETanalyzer(Request, IP)) or (Getrcursivecheck(
Request, IP)):
if (m is not None):
logstring += " Matched Rule: " + str(m.group(0))
if (i is None):
if not any(IP in s for s in recent):
if options.geoip:
match = geolite2.lookup(IP)
if match is not None:
logstring += " Country: " + match.country
print logstring
xcounter += 1
if not options.tryrun:
recent.append(IP)
else:
if options.enable_pf:
pfDrop(IP)
else:
iptablesDrop(IP)
return xcounter
except:
print "Unexpected error (Header):", sys.exc_info()
return 0
def test_geoip(self):
from lumberjack.postprocessors import geoip
from geoip import geolite2
self._disable_action_queue()
self.lj.action_queue._bulk = MagicMock()
ip = '128.141.43.1'
ip_lookup_data = geolite2.lookup(ip)
geopoint = {
'lat': ip_lookup_data.location[0],
'lon': ip_lookup_data.location[1]
}
self.logger.info({'ip': ip},
{'postprocessors': [geoip(field='ip')]})
self.lj.action_queue._flush()
self._check_source_in_queue({
'ip': ip,
'geoip': {
'country_code': ip_lookup_data.country,
'location': geopoint
}
})
def run():
"""
Run the main program
"""
arguments = Arguments()
args = arguments.parse(sys.argv[1:])
formatter = TextFormatter()
if not args["ip"]:
# Lookup my ip_address
# but Armin hardcoded url for checking ip address :)
# ip_address = geolite2.lookup_mine()
ip_address = PublicIPAddress().resolve()
else:
ip_address = args["ip"]
PublicIPAddress().validate(ip_address)
match = geolite2.lookup(ip_address)
if not match:
raise IPAddressException("Incorrect ip address")
weather_provider = OpenWeatherMap(formatter=formatter)
try:
conditions = weather_provider.now(lat=match.location[0], lon=match.location[1], units=args["units"])
except WeatherDataException as exc:
print >>sys.stderr, "Something bad happens({0}), " "Please try again later.".format(exc.message)
sys.exit(1)
puts(getattr(colored, get_temperature_color(conditions))(conditions))
def countryFromIP(ip): #get country from ip
geo = geolite2.lookup(ip)
country = "unknown"
if geo is not None:
country = geo.country
return country
def main():
address = raw_input("Enter IP or Domain: ")
ip = socket.gethostbyname(address)
match = geolite2.lookup(ip)
lat, lon = match.location
result = getplace(lat, lon)
print "The Address: ", result
def map_data():
attack_sources = helpers.query("SELECT IP as ip, COUNT(ID) as attempts, MIN(DateTime) as first_attempt, MAX(DateTime) as last_attempt FROM sshattempts GROUP BY IP ORDER BY COUNT(ID) DESC;")
total_attacks = helpers.query("SELECT count(*) FROM sshattempts;", one = True)
attack_summaries = []
for point in attack_sources:
origin = geolite2.lookup(point['ip'])
if origin is not None:
radius = ((point['attempts'] / total_attacks['count']) * 100)
fill = 'SML'
if radius > 40:
fill = 'BIG'
elif radius > 20:
fill = 'MED'
elif radius < 5:
radius = 5
attack_summaries.append({
'IP': point['ip'],
'latitude': origin.location[0],
'longitude': origin.location[1],
'count': point['attempts'],
'fillKey': fill,
'firstAttempt': point['first_attempt'].strftime('%Y-%m-%d %H:%M:%S'),
'lastAttempt': point['last_attempt'].strftime('%Y-%m-%d %H:%M:%S'),
'radius': radius
})
return Response(json.dumps(attack_summaries), mimetype='application/json')
def sentiments_endpoint(request):
"""
List all sentiments or create a new one.
"""
if request.method == 'POST':
data = JSONParser().parse(request)
data['ip_address'] = get_ip(request)
data['created'] = data.get('created') or datetime.datetime.now()
data['twitter_user'] = '******'
location_match = geolite2.lookup(data['ip_address'])
if location_match:
print(location_match.location)
data['latitude'], data['longitude'] = location_match.location
serializer = models.SentimentSerializer(data=data)
if serializer.is_valid():
serializer.save()
return JSONResponse(serializer.data, status=201)
return JSONResponse(serializer.errors, status=400)
elif request.method == 'GET':
max_items = request.GET.get('max_items') or 100
do_analyze = request.GET.get('analyze') or False
if do_analyze:
_do_analysis()
sentiments = models.Sentiment.objects.filter(latitude__isnull=False)[:max_items]
serializer = models.SentimentSerializer(sentiments, many=True)
return JSONResponse(serializer.data)
return JSONResponse([], status=400)
def ip_whois(ipaddress):
try:
whois_result = IPWhois(ipaddress).lookup()
except ValueError:
return render_template("error.html", error="does not appear to be a valid IP address.", error_subject=ipaddress)
except IPDefinedError as e:
e_split = str(e).split("'")[2:]
rfc = e_split[-2].replace("RFC ","rfc").split(",")[0] if "RFC" in e_split[-2] else None
return render_template("error.html", error="".join(e_split), error_subject=ipaddress, rfc=rfc)
supernet = None
for net in whois_result['nets']:
if net['cidr'] == whois_result['asn_cidr']:
whois_result.update(net)
supernet = net
extended_result = (supernet is not None)
if extended_result:
whois_result['nets'].remove(supernet)
try:
rdns_hostname = socket.gethostbyaddr(ipaddress)[0]
except socket.herror:
rdns_hostname = None
ip_geo = geolite2.lookup(ipaddress)
ip_location = ip_geo.location if ip_geo is not None else None
return render_template("ip.html", ipaddress=ipaddress, rdns_hostname=rdns_hostname, ip_location=ip_location, whois_result=whois_result, extended_result=extended_result)
def crawl_recursively(link, limitd, depth, breadth, br, flag):
if limitd == depth:
return
if link in crawled:
return
if limitd == 0:
host = urlparse.urlparse(
link).hostname #hostname for the primary seed
ips = socket.gethostbyname(host) #ip address of its server
print host
match = geolite2.lookup(ips)
country = match.country #country of the server
print str(ips) + " " + str(country)
host = host.split('.')
host = host[-2] + '.' + host[-1]
answers = dns.resolver.query(host, 'NS')
for server in answers:
print str(server) + " " + socket.gethostbyname(str(server))
#print("\n"+str(limitd))
print link + "\n"
newLinks = getAllLinksOnPage(link, breadth, br, flag)
crawled.add(link) #seed
for link in newLinks:
crawl_recursively(link, limitd + 1, depth, breadth, br, flag)
return
def retKML(ip):
try:
kml = ''
if ip == ni.ifaddresses('en1')[2][0]['addr']:
if MY_LOCATION == '':
MY_LOCATION = get_coordinates(MY_ADDRESS)
location = MY_LOCATION
kml = (
'<Placemark>\n'
'<name>%s,%s</name>\n'
'<Point>\n'
'<coordinates>%6f,%6f</coordinates>\n'
'</Point>\n'
'</Placemark>\n'
)%(ip, MY_ADDRESS, location[1], location[0])
else:
match = geolite2.lookup(ip)
country = match.country
timezone = match.timezone
subdivision = match.subdivisions
location = match.location
#print location
kml = (
'<Placemark>\n'
'<name>%s,%s,%s</name>\n'
'<Point>\n'
'<coordinates>%6f,%6f</coordinates>\n'
'</Point>\n'
'</Placemark>\n'
)%(ip, timezone, country, location[1], location[0])
return kml
except Exception, e:
return ''
def HEADERanalyzer(line):
try:
xcounter = 0
IP = line.split()[options.IPpos] # May need to be adjust, default 0 should work, combined is 1
Request = line.split('"')[1].lower()
Client = line.split('"')[-2]
logstring = str(datetime.datetime.now()) + " " + IP + " Header: " + Client
m = re.search(blacklist,Client) # related services
i = re.search(whitelist,IP) # Whitelabeld IP's
if ( m is not None) or ( GETanalyzer(Request,IP) ) or ( Getrcursivecheck(Request,IP) ):
if ( m is not None):
logstring += " Matched Rule: " + str(m.group(0))
if ( i is None ) :
if not any(IP in s for s in recent):
if options.geoip:
match = geolite2.lookup(IP)
if match is not None:
logstring += " Country: " + match.country
print logstring
xcounter += 1
if not options.tryrun:
recent.append(IP)
else:
if options.enable_pf:
pfDrop(IP)
else:
iptablesDrop(IP)
return xcounter
except:
print "Unexpected error (Header):", sys.exc_info()
return 0
def phase5(): # https://pythonhosted.org/python-geoip/
print "[*] Phase5 (Extract data)"
print "=> working on"
files = os.listdir('./Working/Unzip/')
with open('./Working/result.csv','r') as fi:
lines = fi.readlines()
with io.open('./Working/result.csv','w', encoding='utf8') as fo:
# with open('./Working/result.csv','w') as fo:
#fo.write(unicode('Serial Number,Upload Time,Name,Bank,Account Number,IP,Country\n'))
for i in range(36876):
csv,file = lines[i],'./Working/Unzip/' + files[i] + '/signCert.cert'
with open(file,'r') as fi:
cert = fi.readline().decode('cp949')
num, time, name, bank, account, ip = \
csv.split(',')[0], csv.split(',')[1], cert.split('=')[1].split('(')[0], cert.split('=')[2].split(',')[0], \
cert.split(')')[1].split(',')[0], csv.split(',')[5][:-1]
import subprocess
#try:
# ipinfo - 1,000 requests per day
#cn = subprocess.check_output('curl ipinfo.io/' + ip, shell=True).split('country": "')[1].split('"')[0]
try:
cn = geoip.lookup(ip).country
except:
cn = "None"
fo.write("%s,%s,%s,%s,%s,%s,%s,\n" %(num, time, unicode(name), bank, account, ip, cn))
def getGroupInfo(self,fileName = 'client_data_of_chn_1.txt',time=200,duration=100,classifyType = 'continent'):
print 'We get the Geographical Group Info from:',fileName,',from time slot:',time,\
',with duration:',duration,',and classifyType:',classifyType
self.GroupList = {}
readFile = open(fileName,'r')
for line in readFile:
split = line.split()
if len(split) != 19:
continue
if int(split[0]) < time:
continue
elif int(split[0]) > (time + duration):
break
match = geolite2.lookup(split[2])
if classifyType == 'continent':
value = match.continent
elif classifyType == 'country':
value = match.country
self.GroupList[int(split[1])] = value
readFile.close()
return self.GroupList
def get_random_ranges(self, num=1, max_ips=0, day_ranges=False):
if max_ips:
rranges = []
ips = 0
tries = 100
kill_loop = 0
while tries != 0:
r = random.choice(self.ranges)
if not r.strip():
continue
rcidr = r.split('/')[1]
count1 = 2**(32 - int(rcidr))
if ips + count1 < max_ips + (max_ips / 10):
tries -= 1
else:
break
if r not in rranges and not day_ranges:
rranges.append(r)
ips += count1
# config.max_ips += ips
if kill_loop > 1000:
break
elif ips < max_ips + (max_ips / 10) and tries == 1:
tries += 2
kill_loop += 1
elif r not in rranges and day_ranges:
check = ipaddress.ip_network(r)
r_ip = random.randrange(1, 200)
c_ip = str(check[r_ip])
try:
tm = geolite2.lookup(c_ip)
except TypeError:
print('''Python dependencies error:\n
~$ pip3 uninstall python-geoip python-geoip-python3\n ~$ pip3 install python-geoip-python3'''
)
sys.exit(0)
try:
time_src = pytz.timezone(tm.timezone)
except Exception as e:
logging.debug(e)
continue
time_pm = datetime.now(time_src)
check_done = time_pm.strftime("%H")
pm_0 = range(9)
pm_1 = ["{:02d}".format(i) for i in pm_0]
pm_1 += ["17", "18", "19", "20", "21", "22", "23"]
if [time for time in pm_1 if time in check_done]:
continue
else:
rranges.append(r)
ips += count1
config.max_ips += ips
# elif not rranges and tries == 1:
# tries += 1
if rranges:
config.random_countries.append(self.country)
else:
rranges = [random.choice(self.ranges)]
return rranges
def add_post(request):
if request.method == "GET":
client_ip, is_routable = get_client_ip(request)
country_code = None
if client_ip is None:
pass
# Unable to get the client's IP address
else:
# print(get_client_ip(request))
match = geolite2.lookup(client_ip)
if match is not None:
get_code = match.country
try:
country_code = CountryCode.objects.get(code=get_code)
except Exception as e:
print(e)
pass
return render(
request, 'baseapp/home.html', {
'country_item': country_item,
'all_country_item': all_country_item,
'world_item': world_item,
'cnn_item': cnn_item,
'youtube_item': youtube_item
})
else:
return render(request, 'baseapp/home.html')
def get_url_info(target):
data = geolite2.lookup(target.encode())
locator = Nominatim(user_agent="myGeocoder")
location = locator.reverse(data.location)
datas = {
"ip": str(data.ip),
"country_code": str(data.country),
"continent": str(data.continent),
"sub_div": str(data.subdivisions),
"timezone": str(data.timezone),
"lattitude": str(data.location[0]),
"longitude": str(data.location[1]),
"country": str(location.raw["address"]["country"]),
"county": str(location.raw["address"]["county"]),
"state": str(location.raw["address"]["state"]),
"postcode": str(location.raw["address"]["postcode"])
}
print(colors.blue +
"\n[+]------------ IP Information -----------------[+]\n")
for info in datas:
if info:
print(colors.yellow + info + " : " + datas[info])
else:
pass
print("\nGoogle Map : https://www.google.com/maps/search/" +
datas["lattitude"] + "," + datas["longitude"] + "\n")
print(colors.blue +
"\n[+] ------------------- Completed -------------------\n\n")
def get_geo_from_ip(ip_addr): try: from geoip import geolite2 return geolite2.lookup(ip_addr) except ImportError: return except ValueError: return
def get_coords_from_ip(ip=None):
if ip is None:
ip = _get_local_ip()
match = geolite2.lookup(ip)
if match is not None:
return match.location
else:
return None
def ip_location(ip):
match = geolite2.lookup(ip)
try:
country = match.country
return country
except Exception,e:
logging.info(ip)
return
def add_new_game(ip):
print "Ip address: " + ip
match = geolite2.lookup(ip)
print match.country
global INITIAL_ID
INITIAL_ID += 1
GAMES[INITIAL_ID] = game(match.country)
return INITIAL_ID
def ip2country():
url = 'http://eth0.me'
res = requests.get(url, timeout = 5)
if res.text is not None:
match = geolite2.lookup(res.text.strip())
if match is not None:
return match.country
return 'CN'
def search_ip(request):
ip = request.GET['question']
if 'position' not in request.GET:
date_position = 0
else:
date_position = int(request.GET['position'])
if date_position < 0:
date_position = 0
try:
reversed_dns = socket.gethostbyaddr(ip)
except socket.herror:
reversed_dns = 'Unknown host'
try:
lat, long = geolite2.lookup(ip).location
except AttributeError:
lat, long = None, None
try:
http80 = HTTP(HTTP80.objects.filter(ip=ip).order_by('-date')[date_position].data)
except IndexError:
http80 = None
try:
http443 = HTTP(HTTP443.objects.filter(ip=ip).order_by('-date')[date_position].data)
except IndexError:
http443 = None
try:
http8000 = HTTP(HTTP8000.objects.filter(ip=ip).order_by('-date')[date_position].data)
except IndexError:
http8000 = None
try:
http8080 = HTTP(HTTP8080.objects.filter(ip=ip).order_by('-date')[date_position].data)
except IndexError:
http8080 = None
try:
https = HTTPS(HTTPS443.objects.filter(ip=ip).order_by('-date')[date_position].data)
except IndexError:
https = None
return render(request, 'graphs/search.html',
{'ip': ip,
'reverse': reversed_dns[0],
'lat': lat,
'long': long,
'date_position': date_position,
'http80': http80,
'http443': http443,
'http8000': http8000,
'http8080': http8080,
'https': https
})
def get_location(ip):
if ip is None:
return "Unknown"
match = geolite2.lookup(ip)
if match is None:
return "Unknown"
return match.country
def geoLocateIp(ipAddress):
try:
ipResult = geolite2.lookup(ipAddress)
allData = ipResult.get_info_dict()
countries = re.search(r"en': u'.+?',",re.search(r"country(.*)",str(allData)).group())
ipCountry = countries.group().replace("en': u'", '').replace("',", '')
return ipCountry
except:
return 'ERROR - "%s" is not a valid public ip address.' % ipAddress
