def setup_data(self):
"""
This method uses the classes pandas dataframe and saves back to it.
It first initializes the connection with geolite2, then for each IP in the data, it finds out the country and
city of that IP and stores this as a new column onto our pandas dataframe. After it has does this we close the
connection to geolite2.
After that will call httpagentparser which converts our user_agent_string into its browser family and it's OS,
this is done for each row and is saved as new columns onto our pandas dataframe.
"""
reader = geolite2.reader()
self.df["country"] = self.df.apply(
lambda k: (self.none_check(reader.get(self.check_ip(k["IP"])),
"IP", "country")
if k["IP"] not in ["-", None] else None),
axis=1)
self.df["city"] = self.df.apply(
lambda k:
(self.none_check(reader.get(self.check_ip(k["IP"])), "IP", "city")
if k["IP"] not in ["-", None] else None),
axis=1)
geolite2.close()
self.df["browser_family"] = self.df.apply(
lambda k: self.none_check(
httpagentparser.detect(k["user_agent_string"], {}).get(
"browser"), "user_agent_str", "browser")
if k["IP"] not in ["-", None] else None,
axis=1)
self.df["os_family"] = self.df.apply(lambda k: self.none_check(
httpagentparser.detect(k["user_agent_string"], {}).get("os"),
"user_agent_str", "os") if k["IP"] not in ["-", None] else None,
axis=1)
def test_none_check(self):
"""
This test check whether the none_check returns what it is suppose to depending on the input parameters.
"""
reader = geolite2.reader()
# Asserting that the correct country comes out
ip_right = reader.get("92.238.71.10")
assert (self.etl.none_check(ip_right, "IP",
"country") == "United Kingdom")
# Asserting that when there is no key with that name or similar the method returns none
assert (self.etl.none_check(ip_right, "IP", "browser") == None)
geolite2.close()
# Asserting the none_check gets the name correctly
user_agent_right = httpagentparser.detect(
"Mozilla/5.0 (iPad; CPU OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53",
{}).get("browser")
assert (self.etl.none_check(user_agent_right, "user_agent_str",
"browser") == "Safari")
# Asserting that when a none value is passed for both method types, then none is returned back
value_none = None
assert (self.etl.none_check(value_none, "IP", "country") == None)
assert (self.etl.none_check(value_none, "user_agent_str",
"browser") == None)
# Asserting that a differnet method type has been chosen, it will reutrned back none
assert (self.etl.none_check(value_none, "mutliply", "country") == None)
def add_countries(df):
print("This might take a while, please wait...")
geo = geolite2.reader()
df['src_country'] = df['src_ip'].apply(get_country, geo=geo)
df['dst_country'] = df['dst_ip'].apply(get_country, geo=geo)
geolite2.close()
return df
def geoJSON(self):
""" Generates the relevent GeoJSON data needed for the map """
jsonList = list()
# Load GeoIP
reader = geolite2.reader()
for ip in self.ipList:
# Get GeoIP information for IP
try:
ipgeo = reader.get(ip)
lat = ipgeo["location"]["latitude"]
lon = ipgeo["location"]["longitude"]
except TypeError:
# Some IPs don't have geolocation information for some reason,
# and can't be included on the map, so skip them.
print("No geoip information for", ip)
pass
# Obfuscate IP (this can used for public map pages)
if self.hideIP == True:
ip = obfuscateIPAddress(ip)
jsonList.append(geoJSONTemplate(ip, self.service, [lon,lat], self.color))
geolite2.close() # Close geoip db
return jsonList
def get_country_by_ip(ip_address):
reader = geolite2.reader()
geo_data = reader.get(ip_address)
geolite2.close()
if geo_data and 'country' in geo_data and 'iso_code' in geo_data['country']:
country_iso_code = geo_data['country']['iso_code']
if country_iso_code in countries:
return Country(country_iso_code)
def get_country_by_ip(ip_address):
reader = geolite2.reader()
geo_data = reader.get(ip_address)
geolite2.close()
if geo_data and 'country' in geo_data and 'iso_code' in geo_data['country']:
country_iso_code = geo_data['country']['iso_code']
if country_iso_code in countries:
return Country(country_iso_code)
def get_location_from_ip(ip):
result = None
try:
result = geolite2.reader().get(ip)
except:
result = None
finally:
geolite2.close()
return result
def get_country_by_ip(ip):
reader = geolite2.reader()
geo_data = reader.get(ip)
geolite2.close()
if geo_data and "country" in geo_data and "iso_code" in geo_data["country"]:
country_iso_code = geo_data["country"]["iso_code"]
if country_iso_code in countries:
return Country(country_iso_code)
return None
def geoip(ip):
"""
Returns geo IP info of the given IP.
"""
reader = geolite2.reader()
geo_info = reader.get(ip)
geo_info = geo_info or {}
geo_info.update({'ip': ip})
geolite2.close()
return geo_info
def main(data):
data['city'] = []
data['country'] = []
ip_array = data['IP']
reader = geolite2.reader()
print('Starting IP conversion to city, country')
for ip in ip_array:
ips = ip.split(', ')
ip_valid = isipvalid(ips)
if ip_valid is not None and len(ip_valid) == 1:
location = reader.get(ip_valid[0])
if location is not None and 'country' in location.keys(
) and 'city' in location.keys():
data['country'].append(location['country']['names']['en'])
data['city'].append(location['city']['names']['en'])
elif location is not None and 'country' in location.keys():
data['country'].append(location['country']['names']['en'])
data['city'].append('Unknown')
elif location is not None and 'city' in location.keys():
data['city'].append(location['city']['names']['en'])
data['country'].append('Unknown')
else:
data['country'].append('Unknown')
data['city'].append('Unknown')
elif ip_valid is not None and len(ip_valid) > 1:
location_0 = reader.get(ip_valid[0])
location_1 = reader.get(ip_valid[1])
if location_0 is not None and 'country' in location_0.keys(
) and 'city' in location_0.keys():
data['country'].append(location_0['country']['names']['en'])
data['city'].append(location_0['city']['names']['en'])
elif location_1 is not None and 'country' in location_1.keys(
) and 'city' in location_1.keys():
data['country'].append(location_1['country']['names']['en'])
data['city'].append(location_1['city']['names']['en'])
elif location_1 is not None and 'country' in location_1.keys():
data['country'].append(location_1['country']['names']['en'])
data['city'].append('Unknown')
elif location_1 is not None and 'city' in location_1.keys():
data['city'].append(location_1['city']['names']['en'])
data['country'].append('Unknown')
else:
data['country'].append('Unknown')
data['city'].append('Unknown')
else:
data['country'].append('Unknown')
data['city'].append('Unknown')
geolite2.close()
return data
def get_time(dashed_ip):
ip = dashed_ip.replace("-", ".")
reader = geolite2.reader()
geo_info = reader.get(ip)
geolite2.close()
print(geo_info)
tz = geo_info['location']['time_zone']
time_obj = pytz.timezone(tz)
date_time = datetime.now(time_obj)
time = date_time.strftime("%H:%M:%S")
return jsonify({'time': time})
def ipaddr_forecast():
ipaddr = str(ipgetter.myip())
reader = geolite2.reader()
loc = reader.get(ipaddr)
geolite2.close()
latlong = str(loc["location"]["latitude"]) + "," + str(
loc["location"]["longitude"])
ep = get_endpoint_data(latlong)
return epdata_to_forecast(ep)
def get_country_from_url(url):
try:
hostname = urlparse(url)[1]
ip = socket.gethostbyname(hostname)
result = geolite2.reader().get(ip)
country_iso_code = result['country']['iso_code']
except:
country_iso_code = "unknown"
finally:
geolite2.close()
return country_iso_code
def turn_ip_address(ip = None):
if ip == None:
return
try:
reader = geolite2.reader()
line = reader.get(ip)
if line == None:
return None
return (line['country']['names']['en'],line['continent']['names']['en'],line['location']['latitude'],line['location']['longitude'])
finally:
geolite2.close()
def CountryByIP(ip):
match = geolite2.reader()
info = match.get(ip)
geolite2.close()
#print (info)
if info is None:
return ('Unknown')
elif info.get('country') is None:
return ("Unknown")
else:
return (info['country']['names']['en'])
def print_packets(pcap):
"""Print out information about each packet in a pcap
Args:
pcap: dpkt pcap reader object (dpkt.pcap.Reader)
"""
data = {}
# header = "time_stamp,ip.source,ip.destination,port.source, port.destination, ip.protocol, length.packet, iso.country"
# for row in reader:
# data[row['time_stamp']] = [row["ip.length"],row["ip.identification"],row["ip.offset"],row["ip.ttl"],row["ip.checksum"],row["ip.source"],row["ip.destination"],row["tcp.sequence"],row["tcp.acknowledge"],row["tcp.offset"],row["tcp.flags_x"],row["tcp.checksum"],row["tcp.urgent_point"],row["tcp_flags_binary_string_x"],row["traffic_status_x"],row["threat_type"],row["time_index"],row["seconds_in_day"],row["sin_time"],row["cos_time"],row["well_known_src_port"],row["well_known_dst_port"],row["registered_src_port"],row["registered_dst_port"],row["ephemeral_src_port"],row["ephemeral_dst_port"],row["tcp.flags_y"],row["tcp_flags_binary_string_y"],row["traffic_status_y"],row["country"],row["country_iso"],row["alpha_2_code"],row["same_source"],row["same_destination_portion"],row["diff_port_portion"],row["same_port_portion"],row["same_dest_same_port_portion"],row["same_dest_diff_port_portion"],row["diff_dest_same_port_portion"]]
reader = geolite2.reader()
# For each packet in the pcap process the contents
for timestamp, buf in pcap:
# Unpack the Ethernet frame (mac src/dst, ethertype)
eth = dpkt.ethernet.Ethernet(buf)
#print('Ethernet Frame: ', mac_addr(eth.src), mac_addr(eth.dst), eth.type)
# Make sure the Ethernet data contains an IP packet
if not isinstance(eth.data, dpkt.ip.IP):
#print('Non IP Packet type not supported %s\n' % eth.data.__class__.__name__)
continue
# Print out the timestamp in UTC
#print('Timestamp: ', str(datetime.datetime.utcfromtimestamp(timestamp)))
print('Timestamp: ', str(timestamp))
# Now unpack the data within the Ethernet frame (the IP packet)
# Pulling out src, dst, length, fragment info, TTL, and Protocol
ip = eth.data
# Pull out fragment information (flags and offset all packed into off field, so use bitmasks)
do_not_fragment = bool(ip.off & dpkt.ip.IP_DF)
more_fragments = bool(ip.off & dpkt.ip.IP_MF)
fragment_offset = ip.off & dpkt.ip.IP_OFFMASK
sourceip = inet_to_str(ip.src)
destip = inet_to_str(ip.dst)
sourcematch = reader.get(sourceip)
destmatch = reader.get(sourceip)
# Print out the info
print('Protocol %d: %s (%d) -> %s (%d) \n%s -> %s \nMisc: len=%d ttl=%d DF=%d MF=%d offset=%d' % \
(ip.p, sourceip, ip.data.sport, destip, ip.data.dport, sourcematch['country']['iso_code'], destmatch['country']['iso_code'], ip.len, ip.ttl, do_not_fragment, more_fragments, fragment_offset))
print('')
geolite2.close()
def popularcountry():
country_list = []
cursor.execute("SELECT IP FROM testtable WHERE Category LIKE 'cart?goods%' ")
ip_tulpe = cursor.fetchall()
ip_list = to_list(ip_tulpe)
for ip in ip_list:
country_list.append(country(ip))
print('Посетители этой страны совершают больше всего действий на сайте: ' + str(freq(country_list)))
geolite2.close()
def main(args):
"""
Main for geoplot.py
:param args:
:return:
"""
events = {}
src_patt = re.compile("SRC=(.+?)\s")
dst_patt = re.compile("DST=(.+?)\s")
print(args)
rd = geolite2.reader()
ipl = rd.get(args.DST_IP)
dlon = ipl['location']['longitude']
dlat = ipl['location']['latitude']
with open(args.LOG_FILE, 'r') as infile:
for line in infile.readlines():
try:
dst = dst_patt.findall(line)[0]
except IndexError:
continue
if args.DST_IP == dst:
src = src_patt.findall(line)[0]
if src in events:
events[src] += 1
else:
events[src] = 1
fig = plt.figure()
m = Basemap(resolution='l',
projection='mill', lon_0=0)
m.drawcoastlines()
m.fillcontinents()
m.drawcountries()
maxwidth = 10
largest = sorted(events.items(), key=lambda x: x[1], reverse=True)[0][1]
for evt in events:
ipl = rd.get(evt)
if ipl:
lat = ipl['location']['latitude']
long = ipl['location']['longitude']
m.plot(long, lat, 'ro', latlon=True, zorder=3,
markersize=events[evt] * (maxwidth / largest))
m.plot(dlon, dlat, 'go', latlon=True, zorder=2, markersize=4)
geolite2.close()
plt.show()
def process_request(self, request):
user_time_zone = request.session.get('user_time_zone')
try:
if not user_time_zone:
user_ip = get_real_ip(request)
if user_ip:
reader = geolite2.reader()
ip_details = reader.get(user_ip)
user_time_zone = ip_details['location']['time_zone']
geolite2.close()
if user_time_zone:
request.session['user_time_zone'] = user_time_zone
timezone.activate(pytz.timezone(user_time_zone))
except:
timezone.deactivate()
def index():
ip = request.remote_addr
from geolite2 import geolite2
reader = geolite2.reader()
if (str(ip) != '127.0.0.1') and ('192.168' not in str(ip)) and (
'China' not in str(reader.get(ip))) and ('Beijing' not in str(
reader.get(ip))):
geolite2.close()
return render_template('403.html', ip=ip)
else:
geolite2.close()
name = session.get('name')
if name:
name = name[:1]
return render_template('index.html', name=name)
def set_geo(form=None):
if form is not None:
user = User.find_by('username', form['username'])
user.modif('lat', form['lat'])
user.modif('long', form['long'])
api_key = str(os.environ['GOOGLE_API_KEY'])
geolocator = GoogleV3(api_key=api_key)
location = geolocator.reverse(form['lat'] + ", " + form['long'])
user.modif('location', str(location[4]))
else:
user = User.find_by('username', session['user'])
ip = request.environ['REMOTE_ADDR']
reader = geolite2.reader()
match = reader.get('62.210.33.87')
geolite2.close()
user.modif('lat', str(match['location']['latitude']))
user.modif('long', str(match['location']['longitude']))
user.modif('location', str(match['city']['names']['fr']))
user.save()
return "", 200
def queryDns(list, coordinates, timeout, host, output_file):
resolver_count = 0
address_count = 0
ip_db = geolite2.reader()
resolver = dns.resolver.Resolver()
resolver.lifetime = timeout
file = open(output_file, "w")
for ip in list:
try:
resolver.nameservers = [ip]
dns_query_response = resolver.query(host, 'A')
f_entry = "ip: " + str(ip)
resolver_count = resolver_count + 1
loc = ip_db.get(ip)
if loc is not None:
coordinates.append(
tuple([
loc['location']['latitude'],
loc['location']['longitude']
]))
f_entry = f_entry + ("; latitude: ") + str(
loc['location']['latitude']) + ("; longitude: ") + str(
loc['location']['longitude']) + "\n"
address_count = address_count + 1
else:
f_entry = f_entry + ("; Could not resolve location\n")
file.write(f_entry)
except:
continue
geolite2.close()
file.write("\nTotal number of ips used for verification: " +
str(len(list)))
file.write(("\nTotal number of open DNS resolvers: ") +
str(resolver_count) +
("\nTotal number of location resolvable open DNS resolvers: ") +
str(address_count))
file.write("\nTotal number of location unresolved open DNS resolvers: " +
str(resolver_count - address_count))
file.write("\nTotal number of ips found not to be open DNS resolvers: " +
str(len(list) - resolver_count))
file.close()
def getpeerinfo():
peers = normalize_result(rpc_connect().getpeerinfo())
peers = json.loads(json.loads(peers.response[0]))
peer_list = []
for peer in peers:
ip, separator, port = peer["addr"].rpartition(':')
port = int(port) # convert to integer
ip = ip_address(ip.strip("[]")) # convert to `IPv4Address` or `IPv6Address`
reader = geolite2.reader()
match = reader.get(str(ip))
geolite2.close()
out = {}
out['client'] = peer['subver']
out['ip'] = peer['addr']
out['last_seen'] = datetime.datetime.fromtimestamp(int(peer['lastrecv'])).strftime('%Y-%m-%d %H:%M:%S')
out['connected_since'] = datetime.datetime.fromtimestamp(int(peer['conntime'])).strftime('%Y-%m-%d %H:%M:%S')
out['ping'] = str(peer['pingtime'] * 1000) + "ms"
if match is not None:
out['location']=match
peer_list.append(out)
return jsonify(peer_list)
def getChinaIP(ip='127.0.0.1'):
reader = geolite2.reader()
ip_info = reader.get(ip)
geolite2.close()
print(ip_info)
return True if ip_info['country']['iso_code'] == 'CN' else False
def publish(request):
WITHOUT_ADVERTISING = 0
WITH_ADVERTISING = 1
BOTH_IN_TWO_WINDOWS = 2
shorter_url_name_service = request.GET.get('suns', "")
shortened_url_protocol = request.GET.get('ptc', "")
shortened_url_netloc = request.GET.get('nl', "")
shortened_url_path = request.GET.get('pth', "")
shortened_url = shortened_url_protocol+"://"+shortened_url_netloc+shortened_url_path
# publication_mode = int(request.GET.get('pm', "2"))
publication_mode = request.GET.get('pm', "2")
reader = geolite2.reader()
remote_ip = request.META.get('HTTP_X_FORWARDED_FOR', "")
result = reader.get(remote_ip)
if result:
country = result['country']['names']['es']
else:
country = "N/A"
geolite2.close()
# pp = pprint.PrettyPrinter(indent=4)
# print("\nMeta:\n")
# pp.pprint(request.META)
# print("\n\nResul:\n")
# pp.pprint(result)
content = None
try:
content = Content.objects.get(shortened_url=shortened_url)
with transaction.atomic():
# Se actualiza el atributo last_time_visited al ejecutar .save()
content.save()
visit = Visit.objects.create(content=content,
referer=request.META.get('HTTP_REFERER', ""),
remote_ip=request.META.get('HTTP_X_FORWARDED_FOR', ""),
# remote_host=request.META.get('REMOTE_HOST', ""),
remote_host=request.META.get('HTTP_FORWARDED', ""),
http_agent=request.META.get('HTTP_USER_AGENT', ""),
country=country)
except Content.DoesNotExist:
with transaction.atomic():
adfly_api = AdflyApi()
result = adfly_api.expand(shortened_url)
original_url = result['data'][0]['url']
# url_request = urls.reverse('publish')
content = Content.objects.create(url=original_url,
shorter_url_name_service=shorter_url_name_service,
shortened_url=shortened_url)
visit = Visit.objects.create(content=content,
referer=request.META.get('HTTP_REFERER', ""),
remote_ip=request.META.get(
'HTTP_X_FORWARDED_FOR', ""),
remote_host=request.META.get(
'HTTP_FORWARDED', ""),
http_agent=request.META.get('HTTP_USER_AGENT', ""),
country=country)
except Exception as e:
print("Error en view: " + str(e))
print("||||||||||||||||||||||||||||||||||||||||||||||||||||\n")
print(traceback.format_exc())
# raise Http404("Houston!, we have a problem...")
# https://overiq.com/django/1.10/showing-404-errors-in-django/
return HttpResponseNotFound("Houston!, we have a problem...")
visits = Visit.objects.filter(content=content).count()
if visits > 5:
return render(request, 'welcome/publish.html', {
'inspection_facebook_time':False,
'publication_mode':publication_mode,
'content':content
})
else:
return render(request, 'welcome/publish.html', {
'inspection_facebook_time':True,
'content': content
})
def lookup_geoip(ip_list):
geoip_data = {}
reader = geolite2.reader()
has_data = 0
no_data = 0
print("Looking up GeoIP (location) data...", end="", flush=True)
# If ip_list is empty, stop doing work
if not ip_list:
count_results("GeoIP", has_data, no_data)
geoip_data.update({
"No valid IP addresses.": [
"None", "None", "None", "None", "None", "None", "None", "None",
"None", "None"
]
})
return geoip_data
for ip in ip_list:
data_list = []
try:
match = reader.get(ip)
if match is None:
geoip_data.update({
"No data": [
"No data", "No data", "No data", "No data", "No data",
"No data", "No data", "No data", "No data", "No data"
]
})
no_data += 1
continue
# return geoip_data
# City -> {'city': {'names': {'en': value}}
data_list.append(match.get('city', {}).get('names', {}).get('en'))
# Accuracy radius -> {'location': {'longitude': value}}
data_list.append(match.get('location', {}).get('accuracy_radius'))
# Latitude -> {'location': {'latitude': value}}
data_list.append(match.get('location', {}).get('latitude'))
# Longitude -> {'location': {'longitude': value}}
data_list.append(match.get('location', {}).get('longitude'))
# Postal code -> {'postal': {'code': value}}
data_list.append(match.get('postal', {}).get('code'))
# Metro code -> {'location': {'metro_code': value}}
data_list.append(str(match.get('location', {}).get('metro_code')))
# State or subdivision -> {'subdivisions': [{'names': {'en': value}}]} {key: list[key2: {key: value}]}
if match.get('subdivisions'):
data_list.append(
match.get('subdivisions')[0].get('names', {}).get('en'))
else:
data_list.append(None)
# Time zone -> {'location': {'time_zone': value}}
data_list.append(match.get('location', {}).get('time_zone'))
# Country -> {'country': {'names': {'en': value}}
data_list.append(
match.get('country', {}).get('names', {}).get('en'))
# Continent -> {'continent': {'names': {'en': }}
data_list.append(
match.get('continent', {}).get('names', {}).get('en'))
geoip_data.update({ip: data_list})
has_data += 1
print('.', end="", flush=True)
except ValueError:
geoip_data.update({
"No data": [
"No data", "No data", "No data", "No data", "No data",
"No data", "No data", "No data", "No data", "No data"
]
})
count_results("GeoIP", has_data, no_data)
geolite2.close()
return geoip_data
def onlineStats(self):
print('Analysing online web logs...')
linesProcessed = 0
path = './online/*'
files = glob.glob(path)
startDate = '27 Sept 2015'
endDate = '26 Sept 2016'
pdfCount = 0
pdfCountMyanmar = 0
pdfMyanmarPercentage = 0
ipList = []
monthTotals = {}
myanmarMonthTotals = {}
pageViews = 0
britishCouncil = 0
# Loop through log files
for file in files:
#print(os.path.basename(path))
# Open the first file
with open(file) as fp:
for line in fp:
if line is None:
continue
linesProcessed = linesProcessed+1
print('Processing log file row: '+str(linesProcessed))
if '200' in line and not any(x in line for x in ('.zip', '.pdf', '.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.mp4', '.ico', '.txt', '.php', 'bot', 'spider')):
pageViews = pageViews+1
lineData = self.apache2_logrow(line)
#print('.', end='')
# If requested file is a PDF add to the count
if "pdf" in lineData[4] or "zip" in lineData[4]:
#print(lineData[4])
pdfCount = pdfCount+1
#print('*', end='')
# Add IP to list
ipList.append(lineData[0])
logDate = lineData[3].split('/')
logMonth = logDate[1]
logYear = logDate[2].split(':')
logYear = logYear[0]
# All month totals
if logYear not in monthTotals:
monthTotals[logYear] = {}
if logMonth in monthTotals[logYear]:
monthTotals[logYear][logMonth] = monthTotals[logYear][logMonth]+1
else:
monthTotals[logYear][logMonth] = 1
# Myanmar month totals
reader = geolite2.reader()
geo = reader.get(lineData[0])
if geo is None:
continue
if 'country' in geo:
country = geo['country']['iso_code']
if 'MM' in country:
if logYear not in myanmarMonthTotals:
myanmarMonthTotals[logYear] = {}
if logMonth in myanmarMonthTotals[logYear]:
myanmarMonthTotals[logYear][logMonth] = myanmarMonthTotals[logYear][logMonth]+1
else:
myanmarMonthTotals[logYear][logMonth] = 1
if 'britishcouncil' in lineData[4]:
print(lineData[4])
britishCouncil = britishCouncil+1
# Work out the Myanmar IPs
ipCountries = {}
for ip in ipList:
#print(ip)
if ip is not '::1':
reader = geolite2.reader()
geo = reader.get(ip)
if geo is None:
continue
if 'country' in geo:
country = geo['country']['iso_code']
countryName = geo['country']['names']['en']
#print(country)
#quit()
#print(type(country))
if 'MM' in country:
#print(country)
pdfCountMyanmar = pdfCountMyanmar+1
if countryName in ipCountries:
ipCountries[countryName] = ipCountries[countryName]+1
else:
ipCountries[countryName] = 1
geolite2.close()
# Work out the Myanmar downloads as a percentage
if pdfCountMyanmar > 0:
pdfCountMyanmar1percent = pdfCount/100
pdfMyanmarPercentage = pdfCountMyanmar/pdfCountMyanmar1percent
print('\nWeb log analysis complete:')
print('\t Log lines processed: {}'.format(linesProcessed))
print('\t {} to {}'.format(startDate, endDate))
print('\t Total page views: {}'.format(pageViews))
print('\t Total PDF and ZIP downloads {}'.format(pdfCount))
print('\t Total PDF and ZIP downloads from Myanmar {}'.format(pdfCountMyanmar))
print('\t British Council downloads {}'.format(britishCouncil))
print('\t Myanmar percentage {}%'.format(round(pdfMyanmarPercentage, 2)))
print('\t Downloads by country: ')
sorted_ipCountries = OrderedDict(sorted(ipCountries.items(), key=itemgetter(1), reverse=True))
for x in sorted_ipCountries:
print('\t\t - {} : {}'.format(x, ipCountries[x]))
pp = pprint.PrettyPrinter(indent=3)
pp.pprint(monthTotals)
for year in monthTotals:
for month in monthTotals[year]:
print(year+', '+month+', '+str(monthTotals[year][month]))
# Create CSV
with open('onlinestats.csv', 'w', newline='') as csvfile:
onlineCSV = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
onlineCSV.writerow(['Time period ', startDate+' to '+endDate])
onlineCSV.writerow(['Total page views', pageViews])
onlineCSV.writerow(['Total downloads (PDF and ZIP files)', pdfCount])
onlineCSV.writerow(['British Council downloads', britishCouncil])
pmp = str(round(pdfMyanmarPercentage, 2))
onlineCSV.writerow(['Total downloads from Myanmar', str(pdfCountMyanmar)+' ('+str(pmp+'%')+')'])
onlineCSV.writerow(['Total downloads by month:'])
onlineCSV.writerow(['Year', 'Month', 'Downloads'])
od = collections.OrderedDict(sorted(monthTotals.items()))
for year, month in od.items():
odm = collections.OrderedDict(sorted(month.items()))
for mon, value in odm.items():
onlineCSV.writerow([year,mon,value])
"""
for year in monthTotals:
for month in monthTotals[year]:
onlineCSV.writerow([year, month, monthTotals[year][month]])
"""
onlineCSV.writerow(['Myanmar downloads by month:'])
onlineCSV.writerow(['Year', 'Month', 'Downloads'])
od = collections.OrderedDict(sorted(myanmarMonthTotals.items()))
for year, month in od.items():
odm = collections.OrderedDict(sorted(month.items()))
for mon, value in odm.items():
onlineCSV.writerow([year,mon,value])
"""
for year in myanmarMonthTotals:
for month in myanmarMonthTotals[year]:
onlineCSV.writerow([year, month, myanmarMonthTotals[year][month]])
"""
onlineCSV.writerow(['Country', 'Downloads'])
sorted_ipCountries = OrderedDict(sorted(ipCountries.items(), key=itemgetter(1), reverse=True))
for x in sorted_ipCountries:
onlineCSV.writerow([x, ipCountries[x]])
# Create document
'''
elif jsonData['eventid'] == "cowrie.login.failed":
try:
loc = reader.get(jsonData['src_ip'])['country']['names']['fr']
keys = (jsonData['session'], jsonData['username'],
jsonData['password'], int(ts), jsonData['src_ip'], loc)
c.execute("insert into failed values (?,?,?,?,?,?)", keys)
except:
continue
elif jsonData['eventid'] == "cowrie.command.input":
try:
keys = (int(ts), jsonData['src_ip'], jsonData['input'],
jsonData['session'])
c.execute("insert into command values (?,?,?,?)", keys)
except:
continue
geolite2.close()
conn.commit()
conn.close()
print("Starting graphs generation.")
#Données du jour
pie_graph(
"select loc, count(*) as nb from failed where datetime(timestamp, 'unixepoch') >= date('"
+ str(datetime.date.today()) + "') group by loc order by nb desc", 'pays',
'failed.png', 5)
pie_graph(
"select loc, count(*) as nb from success where datetime(timestamp, 'unixepoch') >= date('"
+ str(datetime.date.today()) + "') group by loc order by nb desc", 'pays',
'success.png', 5)
pie_graph(
"select password, count(*) as nb from failed where datetime(timestamp, 'unixepoch') >= date('"
def whois_ip(ip):
reader = geolite2.reader()
answer = reader.get(ip)
geolite2.close()
return answer
def process_page(self, buf):
"""
Process the page record and add any matching edits to elasticsearch.
Args:
buf (str): The page record XML representation.
"""
revisions = {
0: '',
}
prev_revision = ''
try:
# Load the string into an ETree Element
page = ET.fromstring(buf)
# Necessary as parent / children may not appear in order for diffing later.
for revision in page.iter('revision'):
cur_id = revision.find('id').text
cur_text = revision.find('text').text
revisions[cur_id] = cur_text
# Iterate over revisions
for revision in page.iter('revision'):
cur_id = revision.find('id').text
cur_timestamp = revision.find('timestamp').text
cur_text = revision.find('text').text
if not revision.find('parentid') == None:
parent_id = revision.find('parentid').text
else:
parent_id = 0
if not revision.find('comment') == None:
cur_comment = revision.find('comment').text
else:
cur_comment = ''
for contributor in revision.iter('contributor'):
for anonedit in contributor.iter('ip'):
for gov_org, org_ranges in self.ip_ranges.iteritems():
for cur_range in org_ranges:
if IPAddress(
anonedit.text) in IPNetwork(cur_range):
reader = geolite2.reader()
geo_data = reader.get(anonedit.text)
geolite2.close()
edit_document = {
'timestamp':
datetime.now(),
'edit_date':
cur_timestamp,
'title':
page.find('title').text,
'ip':
anonedit.text,
'ip-range-owner':
gov_org,
'document':
cur_text,
'language':
self.file['language'],
'diff':
self.unidiff(revisions[parent_id],
cur_text),
'location': {
"lat":
geo_data['location']['latitude'],
"lon":
geo_data['location']['longitude'],
}
}
print self.es['con'].index(
index=self.es['edit_index'],
doc_type=self.es['edit_doctype'],
id=cur_id,
body=edit_document)
try:
prev_revision = revision.find('text').text
except:
pass
except:
print "Parsing Buffer Failed!"
del revisions
def country(self):
reader = geolite2.reader()
t = reader.get(self.ip)
geolite2.close()
return t["country"]["iso_code"]
