示例#1
0
#       include your user improvement in the next release; users are not obligated to do this, but we would enjoy this courtesy tremendously.
#
#  18)  Sections 0) a) 0) b) and 1) a) are sufficient for use; however sections 1) b) through 1) h) are presented to clarify 1 a) and to enforce non-discrimination and non-exclusion of use.
#       For example some people may choose to redefine the meaning of the words "person" "human being" or "sentient individual" to exclude certain types of people.
#       This would be deemed unacceptable and is specifically rejected by the enumeration presented.  If the wording presented is problematic please contact us and suggest a change,
#       and it will be taken into consideration.
#################################################################################################################################################################################
################################################################ BEGINNING OF PROGRAM #########################################################################################

import sys
import os
import GeoIP
import pygeoip

path = "/home/dawnfire/bin"
os.chdir(path)  # change to dir with dat files

gcunt = pygeoip.GeoIP('GeoIP.dat')
gcit = pygeoip.GeoIP('GeoLiteCity.dat')
gasn = pygeoip.GeoIP('GeoIPASNum.dat')
gorg = pygeoip.GeoIP('GeoIPOrg.dat')

for arg in sys.argv[1:]:
    print(arg, gcunt.country_code_by_addr(arg),
          gcunt.country_name_by_addr(arg))
#   print(gorg.org_by_addr(arg))
#   print(gasn.org_by_addr(arg))
#   print(gcit.record_by_addr(arg))
quit()
################################################################ END OF PROGRAM ################################################################################################
示例#2
0
        res = dict()
        for ip in iplist:
            if ip in ("127.0.0.1", self.ip_addr):
                res[ip] = 100
            else:
                res[ip] = 0
        return res


# # TODO: check if pygeoip is installed!
geo_ip_found = False
try:
    import pygeoip
    try:
        path = "resources/GeoLiteCity.dat"
        gi_city = pygeoip.GeoIP(path)
        #gi_isp = pygeoip.GeoIP("resources/GeoIPISP.dat")
        geo_ip_found = True
    except:
        print >> sys.stderr, "Missing the required geoip database for GeoIPPolicy (if desired) %s, you can find it here: http://geolite.maxmind.com/download/geoip/database/" % path
except:
    print >> sys.stderr, "pygeoip is not installed! GeoIPPolicy cannot be used...(see http://code.google.com/p/pygeoip/ to download it)"


class GeoIPPolicy(SameHostPolicy):
    ''' Rank the same host with a positive value, all other with 0.
    '''
    def __init__(self, own_ip=None):
        SameHostPolicy.__init__(self, own_ip)
        if not geo_ip_found:
            raise Exception("Geo ip is not installed!")
示例#3
0
# TODO: in Jamie's algorithm, for adding adversary we should change the default adversary's bw

import os
import pickle, csv, pygeoip, random
import numpy as np
from collections import namedtuple, defaultdict
import math
from binascii import hexlify
from os import urandom
import re
from collections import namedtuple, defaultdict
import copy
from sets import Set
import logging

rawdata = pygeoip.GeoIP('../requirements/GeoLiteCity.dat')
ai = pygeoip.GeoIP('../requirements/GeoIPASNum.dat')
PATH_TO__SAVE = './files'
PATH_TO__DATA = '../requirements/consensuses-2015'  # './data-2013'
PATH_AS_DATASET = '../datasets'

CREATE_THRESHOLD = 40000
MINIMUM_SETS = 50
DELETE_THRESHOLD = 20000
BREAK_SET_BW_THRESHOLD = 2 * CREATE_THRESHOLD
CREATE_SUB_SET_BANDWIDTH_THRESHOLD = CREATE_THRESHOLD
PROCESSES = 4  # number of processes in multiprocesses module
DEFUALT_ADVERSARY_BW = 50
NUMBER_OF_ADVERSARIES = 1000
AFTER_DAY = '2015-01-01'
ATTACK = 2
示例#4
0
    def setUp(self):
        set_default_values(self)
        with codecs.open(CITY_DB_PATH, 'rb', 'latin_1') as db_file:
            contents = db_file.read()

        self.gic = pygeoip.GeoIP(db_contents=contents)
示例#5
0
import pygeoip
gi = pygeoip.GeoIP('GeoLiteCity.dat')


def printRecord(tgt):
    rec = gi.record_by_name(tgt)
    city = rec['city']
    # region = rec['repipgion_name']
    country = rec['country_name']
    longi = rec['longitude']
    lat = rec['latitude']
    print '[*] Target: ' + tgt + ' Geo-located. '
    print '[+] ' + str(city) + ', ' + str("region") + ', ' + str(country)
    print '[+] Latitude: ' + str(lat) + ', Longitute: ' + str(longi)


tgt = '112.25.223.108'
printRecord(tgt)
示例#6
0
 def __init__(self, paths):
     if paths and pygeoip is None:
         raise TypeError("pygeoip module is not installed")
     self.dbs = [pygeoip.GeoIP(x, pygeoip.MMAP_CACHE) for x in paths]
示例#7
0
import dpkt
import socket
import pygeoip
import optparse

gi = pygeoip.GeoIP('/opt/geoIP/GeoLiteCity.dat')

def retGeoStr(ip):
    try:
        rec = gi.record_by_name(ip)
        city = rec['city']
        country = rec['country_code3']

        if city != '':
            geoLoc = city + ', ' + country
        else:
            geoLoc = country

        return geoLoc
    
    except Exception, e:
        return 'Unregistered'

def printPcap(pcap):
    for (ts, buf) in pcap:
        try:
            eth = dpkt.ethernet.Ethernet(buf)
            ip = eth.data
            src = socket.inet_ntoa(ip.src)
            dst = socket.inet_ntoa(ip.dst)
            print '[+] Src: ' + src + ' --> Dst: ' + dst
示例#8
0
#!/usr/bin/env python3

import sys
import argparse
from collections import defaultdict
from datetime import timedelta, date, datetime
import gzip
import json
import re
import apache_log_parser
import pygeoip
import os

gi = pygeoip.GeoIP(os.path.join(os.path.dirname(__file__), 'GeoIP.dat'))

ignoreips = ['77.161.34.157']  #proycon@home, kobus@home,
internalips = ['127.0.0.1', '131.174.30.3',
               '131.174.30.4']  #localhost, spitfire, applejack
internalblocks = ['131.174.']
line_parser = apache_log_parser.make_parser(
    "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\"")


def ininternalblock(ip):
    for internalblock in internalblocks:
        if ip.startswith(internalblock):
            return True
    return False


class PythonObjectEncoder(json.JSONEncoder):
示例#9
0
import pygeoip as geo
from collections import OrderedDict

gi = geo.GeoIP('GeoLiteCity.dat')
hit_dict = {}
#what's the output format? 
with open ('request_ip_addresses.txt', 'r') as source:
  #how big can the input be? speed vs space
  #clarify delimiter
  lines = source.read().split('\n')
  for item in lines:
      #get the record
#      print(item)
      try:
        hit_dict[item] += 1
      except KeyError:
        hit_dict[item] = 1


sorted_hit_dict = sorted(hit_dict.items(), key=lambda x:x[1])
item_count = 0
for item in reversed(sorted_hit_dict):
    requestRecord = gi.record_by_addr(item[0])
    lat = requestRecord['latitude']
    lon = requestRecord['longitude']
    city = requestRecord['city']
    print("%s,%s,%s,%s,%s" %(item[0],item[1],lat,lon,city))
    item_count += 1
    if item_count == 100:
        break
  
示例#10
0
# coding=UTF-8

import dpkt
import socket
import pygeoip
import optparse

gi = pygeoip.GeoIP('/home/wei/myPy/GeoLiteCity.dat')


def retKMl(ip):
    rec = gi.record_by_name(ip)
    try:
        longitude = rec['longitude']
        latitude = rec['latitude']
        kml = ('<Placemark>\n'
               '<name>%s</name>\n'
               '<Point>\n'
               '<coordinates>%6f,%6f</coordinates>\n'
               '</Point>\n'
               '</Placemark>\n') % (ip, longitude, latitude)
        return kml
    except Exception, e:
        return ''


def plotIPs(pcap):
    kmlPts = ''
    for (ts, buf) in pcap:
        try:
            eth = dpkt.ethernet.Ethernet(buf)
示例#11
0
import pygeoip

gip = pygeoip.GeoIP("GeoLiteCity.dat")  #Download this file on github
res = gip.record_by_addr(
    'IP address')  #Anan zaku shigar da IP address din target dinku

for k, v in res.items():
    print("%s : %s" % (k, v))
示例#12
0
文件: dshell.py 项目: zxsted/Dshell
    def __init__(self, **kwargs):
        self.name = 'unnamed'
        self.description = ''
        self.longdescription = ''
        self.filter = ''
        self.author = 'xx'
        self.decodedbytes = 0
        self.count = 0
        '''dict of options specific to this decoder in format
                'optname':{configdict} translates to --decodername_optname'''
        self.optiondict = {}

        # out holds the output plugin. If None, will inherit the global output
        self.out = None
        # format is the format string for this plugin, if None, uses global
        self.format = None

        # capture options
        self.l2decoder = dpkt.ethernet.Ethernet  # decoder to use if raw mode
        # strip extra layers before IP/IPv6? (such as PPPoE, IP-over-IP, etc..)
        self.striplayers = 0

        self._DEBUG = False

        # can we chain a decoder off the output of this one?
        self.chainable = False
        self.subDecoder = None  # decoder to pass output to for chaining

        # set flags to indicate if handlers are present
        if 'packetHandler' in dir(self):
            self.isPacketHandlerPresent = True
        else:
            self.isPacketHandlerPresent = False
        if 'connectionHandler' in dir(self):
            self.isConnectionHandlerPresent = True
        else:
            self.isConnectionHandlerPresent = False
        if 'blobHandler' in dir(self):
            self.isBlobHandlerPresent = True
        else:
            self.isBlobHandlerPresent = False

        # for connection tracking, if applicable
        self.connectionsDict = {}
        self.cleanupts = 0

        # instantiate and save references to lookup function
        try:
            self.geoccdb = [
                pygeoip.GeoIP(os.environ['DATAPATH'] + '/GeoIP/GeoIP.dat',
                              pygeoip.MEMORY_CACHE).country_code_by_addr
            ]
            try:
                self.geoccdb.append(
                    pygeoip.GeoIP(
                        os.environ['DATAPATH'] + '/GeoIP/GeoIPv6.dat',
                        pygeoip.MEMORY_CACHE).country_code_by_addr)
            except:
                pass
        except:
            self.geoccdb = None

        try:
            self.geoasndb = [
                pygeoip.GeoIP(os.environ['DATAPATH'] + '/GeoIP/GeoIPASNum.dat',
                              pygeoip.MEMORY_CACHE).org_by_addr
            ]
            try:
                self.geoasndb.append(
                    pygeoip.GeoIP(
                        os.environ['DATAPATH'] + '/GeoIP/GeoIPASNumv6.dat',
                        pygeoip.MEMORY_CACHE).org_by_addr)
            except:
                pass
        except:
            self.geoasndb = None

        # import kw args into class members
        if kwargs:
            self.__dict__.update(kwargs)
示例#13
0
def stats_format_users(project_id, users, anon_users, auth_users, geo=False):
    """Format User Stats into JSON."""
    userStats = dict(label="User Statistics", values=[])
    userAnonStats = dict(label="Anonymous Users", values=[], top5=[], locs=[])
    userAuthStats = dict(label="Authenticated Users", values=[], top5=[])

    userStats['values'].append(
        dict(label="Anonymous", value=[0, users['n_anon']]))
    userStats['values'].append(
        dict(label="Authenticated", value=[0, users['n_auth']]))

    for u in anon_users:
        userAnonStats['values'].append(dict(label=u[0], value=[u[1]]))

    for u in auth_users:
        userAuthStats['values'].append(dict(label=u[0], value=[u[1]]))

    # Get location for Anonymous users
    top5_anon = []
    top5_auth = []
    loc_anon = []
    # Check if the GeoLiteCity.dat exists
    geolite = current_app.root_path + '/../dat/GeoLiteCity.dat'
    if geo:  # pragma: no cover
        gic = pygeoip.GeoIP(geolite)
    for u in anon_users:
        if geo:  # pragma: no cover
            loc = gic.record_by_addr(u[0])
        else:
            loc = {}
        if loc is None:  # pragma: no cover
            loc = {}
        if (len(loc.keys()) == 0):
            loc['latitude'] = 0
            loc['longitude'] = 0
        top5_anon.append(dict(ip=u[0], loc=loc, tasks=u[1]))

    for u in anon_users:
        if geo:  # pragma: no cover
            loc = gic.record_by_addr(u[0])
        else:
            loc = {}
        if loc is None:  # pragma: no cover
            loc = {}
        if (len(loc.keys()) == 0):
            loc['latitude'] = 0
            loc['longitude'] = 0
        loc_anon.append(dict(ip=u[0], loc=loc, tasks=u[1]))

    for u in auth_users:
        sql = text('''SELECT name, fullname from "user" where id=:id;''')
        results = session.execute(sql, dict(id=u[0]))
        for row in results:
            fullname = row.fullname
            name = row.name
        top5_auth.append(dict(name=name, fullname=fullname, tasks=u[1]))

    userAnonStats['top5'] = top5_anon[0:5]
    userAnonStats['locs'] = loc_anon
    userAuthStats['top5'] = top5_auth

    return dict(users=userStats,
                anon=userAnonStats,
                auth=userAuthStats,
                n_anon=users['n_anon'],
                n_auth=users['n_auth'])
示例#14
0
 def setUp(self):
     self.gi = pygeoip.GeoIP(COUNTRY_DB_PATH)
     self.gi6 = pygeoip.GeoIP(COUNTRY_V6_DB_PATH)
示例#15
0
from django.shortcuts import render
import dpkt
import socket
import pygeoip
from django.template import Template, Context
from django.template.loader import get_template
from django.http import HttpResponse
import json

gi = pygeoip.GeoIP('/opt/GeoIP/Geo.dat')


def printRecord(tgt):
    rec = gi.record_by_name(tgt)
    city = rec['city']
    country = rec['country_name']
    long = rec['longitude']
    lat = rec['latitude']
    return (tgt, lat, long, city, country)


def printPcap(pcap):
    uniqueIP = set()
    for (ts, buf) in pcap:
        try:
            eth = dpkt.ethernet.Ethernet(buf)
            ip = eth.data
            src = socket.inet_ntoa(ip.src)
            dst = socket.inet_ntoa(ip.dst)
            uniqueIP.add(src)
        except:
示例#16
0
import pygeoip
import pycountry
import time
from datetime import datetime
import logging
from conf import settings

logger = logging.getLogger(__name__)

geov4 = pygeoip.GeoIP(settings.GEOIP4_DB)
geov6 = pygeoip.GeoIP(settings.GEOIP6_DB)


def get_alpha2_code(ip):
    try:
        return geov4.country_code_by_addr(ip)
    except pygeoip.GeoIPError:
        return geov6.country_code_by_addr(ip)


def get_alpha3_code(alpha2):
    country = pycountry.countries.get(alpha2=alpha2)
    return country.alpha3


def add_country(request):
    """Add ISO 3166-1 alpha-3 code to country field of request dict."""
    ip = request.get('ip_address')
    alpha2 = get_alpha2_code(ip)
    alpha3 = get_alpha3_code(alpha2)
    request['country'] = alpha3
示例#17
0
 def setUp(self):
     self.gi = pygeoip.GeoIP(ORG_DB_PATH)
示例#18
0
from netaddr import IPAddress, IPRange, IPSet
from sortedcontainers import SortedDict

from logger import get_logger

logger = get_logger('winnower')

# from http://en.wikipedia.org/wiki/Reserved_IP_addresses:
reserved_ranges = IPSet([
    '0.0.0.0/8', '100.64.0.0/10', '127.0.0.0/8', '192.88.99.0/24',
    '198.18.0.0/15', '198.51.100.0/24', '203.0.113.0/24', '233.252.0.0/24'
])
gi_org = SortedDict()
base_path = os.path.dirname(__file__)
full_path = base_path + '/data/GeoIP.dat'
geo_data = pygeoip.GeoIP(full_path, pygeoip.MEMORY_CACHE)


def load_gi_org(filename):
    with open(filename, 'rb') as f:
        org_reader = csv.DictReader(f, fieldnames=['start', 'end', 'org'])
        for row in org_reader:
            gi_org[row['start']] = (IPRange(row['start'], row['end']),
                                    unicode(row['org'], errors='replace'))

    return gi_org


def org_by_addr(address):
    as_num = None
    as_name = None
示例#19
0
文件: proxy.py 项目: weiyuchen/qrassh
 def getCountryCode(self):
     path, file = os.path.split(__file__)
     file_name = os.path.join(path, "geo_ip.dat")
     print("Load geo_ip.dat from " + file_name)
     geo_ip = pygeoip.GeoIP(file_name)
     return geo_ip.country_code_by_addr(self.clientIp)
示例#20
0
import datetime
import os

try:
    import pygeoip
except ImportError:
    print "Install pygeoip"
    print "\tpip install pygeoip"
    pass

from django.db import models

gi = pygeoip.GeoIP(os.path.join('DionaeaFR/static', 'GeoIP.dat'),
                   pygeoip.MEMORY_CACHE)


class Connection(models.Model):
    connection = models.IntegerField(primary_key=True,
                                     blank=True,
                                     verbose_name='ID')

    connection_type = models.TextField(blank=True, verbose_name='State')

    connection_transport = models.TextField(blank=True,
                                            verbose_name='Protocol')

    connection_protocol = models.TextField(blank=True, verbose_name='Service')

    connection_timestamp = models.IntegerField(blank=True, verbose_name='Date')

    connection_root = models.IntegerField(blank=True, verbose_name='Root')
示例#21
0
#logging.captureWarnings(True)
## https://github.com/shazow/urllib3/issues/497
import requests.packages.urllib3
requests.packages.urllib3.disable_warnings()


from netaddr import IPAddress
def is_ip(address):
    try:
        ip = IPAddress(address)
        return True
    except:
        return False

import pygeoip
gi1 = pygeoip.GeoIP(GEODB_DIR + '/GeoIPASNum.dat', pygeoip.MEMORY_CACHE)
gi2 = pygeoip.GeoIP(GEODB_DIR + '/GeoLiteCity.dat', pygeoip.MEMORY_CACHE)

def org_by_addr(address):
    if is_ip(address):
    	logging.debug("geolocate " + str(address))
        gias = gi1.org_by_addr(address)
        logging.debug("geo: " + str(gias))
	as_num, sep, as_name = str(gias).partition(' ')
        as_num = as_num.replace("AS", "")
        return as_num, as_name
    else:
	return None, None

def cc_by_addr(address):
    if is_ip(address):
示例#22
0
        dns_servers = kwargs.pop('dns_servers')
        dns_tcpover = kwargs.pop('dns_tcpover', [])
        dns_timeout = kwargs.pop('dns_timeout', 2)
        super(self.__class__, self).__init__(*args, **kwargs)
        self.dns_servers = list(dns_servers)
        self.dns_tcpover = tuple(dns_tcpover)
        self.dns_intranet_servers = [x for x in self.dns_servers if is_local_addr(x)]
        self.dns_blacklist = set(dns_blacklist)
        self.dns_timeout = int(dns_timeout)
        self.dns_cache = ExpireCache(max_size=65536)
        self.dns_trust_servers = set(['8.8.8.8', '8.8.4.4', '2001:4860:4860::8888', '2001:4860:4860::8844'])
<<<<<<< HEAD
        for dirname in ('.', '/usr/share/GeoIP/', '/usr/local/share/GeoIP/'):
            filename = os.path.join(dirname, 'GeoIP.dat')
            if os.path.isfile(filename):
                geoip = pygeoip.GeoIP(filename)
                for dnsserver in self.dns_servers:
                    if ':' not in dnsserver and geoip.country_name_by_addr(parse_hostport(dnsserver, 53)[0]) not in ('China',):
                        self.dns_trust_servers.add(dnsserver)
                break
=======
        if pygeoip:
            for dirname in ('.', '/usr/share/GeoIP/', '/usr/local/share/GeoIP/'):
                filename = os.path.join(dirname, 'GeoIP.dat')
                if os.path.isfile(filename):
                    geoip = pygeoip.GeoIP(filename)
                    for dnsserver in self.dns_servers:
                        if ':' not in dnsserver and geoip.country_name_by_addr(parse_hostport(dnsserver, 53)[0]) not in ('China',):
                            self.dns_trust_servers.add(dnsserver)
                    break
>>>>>>> origin/master
示例#23
0
 def setUp(self):
     set_default_values(self)
     self.gic = pygeoip.GeoIP(CITY_DB_PATH)
示例#24
0
def do_mirrors(path, context):

    # normalize OS path separators to fwd slash
    path = path.replace(os.path.sep, '/')

    # if we are on a downloads page
    #if '/downloads/index.html' in path:
    if '/downloads/' in path:

        # create mirrors dict
        mirrors = {}
        # instantiate geoip object, load db
        gi = pygeoip.GeoIP('./plugins/gimp_mirrors/GeoIP.dat')

        # load mirrors2.json file from content
        try:
            with open('./content/downloads/mirrors2.json') as data_file:
                data = json.load(data_file)

                for record in data:
                    #print "####"
                    #print record

                    try:
                        country = gi.country_name_by_name(record)
                        #print country

                        if country in mirrors:
                            mirrors[country] += data[record]
                        else:
                            mirrors[country] = data[record]

                    except:
                        print "cannot resolve record: ", record

                try:
                    with open(path, 'r') as f:
                        s = f.read()
                        #print s.decode('utf-8')
                        s = s.decode('utf-8').replace(u"<!-- MIRRORS -->",
                                                      html_output(mirrors))

                    with open(path, 'w') as f:
                        f.write(s.encode('utf-8'))

                except:
                    print "Trouble with reading/writing path."

        except IOError:
            print "Cannot open /content/downloads/mirrors.json!"
            try:
                with open(path, 'r') as f:
                    s = f.read()
                    #print s.decode('utf-8')
                    s = s.decode('utf-8').replace(
                        u"<!-- MIRRORS -->",
                        u"<p><small>Cannot open mirrors2.json</small></p>")

                with open(path, 'w') as f:
                    f.write(s.encode('utf-8'))

            except:
                print "Trouble with reading/writing path."
示例#25
0
#!/usr/bin/env python
# coding: utf-8

from __future__ import print_function
import hashlib
import requests
import json
import pygeoip
import time
import os
from virus_total_apis import PublicApi as VirusTotalPublicApi

rawdata = pygeoip.GeoIP('GeoLiteCity/GeoLiteCity.dat')
API_KEY = '<** INSERT API KEY HERE **>'
vt = VirusTotalPublicApi(API_KEY)

md5_ok = []


def ipquery(ip):
    data = rawdata.record_by_name(ip)
    country = data['country_name']
    city = data['city']
    result = str(str(city) + ', ' + str(country))
    return (result)


def md5(fname):
    hash_md5 = hashlib.md5()
    with open(fname, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
示例#26
0
#!/usr/bin/python
# -*- coding: utf-8 -*-
import os
import pygeoip
import optparse

from utils import print_ip_details

if __name__ == '__main__':
    parser = optparse.OptionParser(
        "usage %prog --ip <IP Address> [-d <database file>] ")
    parser.add_option('--ip',
                      dest='ip',
                      type='string',
                      help='specify ip address')
    parser.add_option(
        '-d',
        dest='database',
        type='string',
        default="GeoLiteCity.dat",
        help='specify database location address [default: %default]')

    (options, args) = parser.parse_args()

    if not options.ip:
        print(parser.usage)
        exit(0)

    client = pygeoip.GeoIP(os.path.join(os.getcwd(), options.database))
    print_ip_details(client, options.ip)
示例#27
0
#!/usr/bin/python
# -*- coding: utf-8 -*-
import dpkt
import socket
import pygeoip
import optparse
gi = pygeoip.GeoIP('D:\Violent-Python-Examples\Chapter-4\GeoLiteCity.dat')


def retKML(ip):
    rec = gi.record_by_name(ip)
    try:
        longitude = rec['longitude']
        latitude = rec['latitude']
        kml = ('<Placemark>\n'
               '<name>%s</name>\n'
               '<Point>\n'
               '<coordinates>%6f,%6f</coordinates>\n'
               '</Point>\n'
               '</Placemark>\n') % (ip, longitude, latitude)
        return kml
    except:
        return ''


def plotIPs(pcap):
    kmlPts = ''
    for (ts, buf) in pcap:
        try:
            eth = dpkt.ethernet.Ethernet(buf)
            ip = eth.data
 def lookUpIP(self, ip):
     gi = pygeoip.GeoIP('./src/GeoLiteCity.dat')
     return gi.record_by_addr(ip)
示例#29
0
from collections import Counter
from collections import OrderedDict
import os
import re
import datetime

fileName = "glastopf.log"
outFile = "glastopf.csv"
allLog = "all.csv"
gCfreqFile = "gcf.csv"
gRfreqFile = "grf.csv"
gResourceFile = "gresrcf.csv"
gDailyHitsFile = "gdailyhits.csv"
logPath = "glastopf/logs/"
csvPath = "glastopf/csv/"
gip = pygeoip.GeoIP("GeoIP.dat", pygeoip.MEMORY_CACHE)
ignoreLine = [
    "Initializing Glastopf", "Connecting to main database", "Glastopf started",
    "Bootstrapping dork database", "Generating initial dork pages",
    "Stopping Glastopf"
]

activityList = []


def write_list_of_dicts_to_csv(fileName, list_of_dicts):
    os.chdir('glastopf/csv')
    with open(fileName, "wb") as out_file:
        fieldnames = sorted(list(set(k for d in list_of_dicts for k in d)))
        writer = csv.DictWriter(out_file,
                                fieldnames=fieldnames,
示例#30
0
def salvar(request):
    if request.method == 'POST':
        if 'rotulo' in request.POST:
            rotulo = request.POST['rotulo']
            #Atualizando objeto Dado...
            #try:
            #    objeto = models.Dado.objects.get(user=request.user)
            #except models.Dado.DoesNotExist:
            #    objeto = None
            #    #print ("o objeto nao existia!")

            #if objeto == None:
            #print ("objeto foi none, instanciando um novo...")
            objeto = models.Dado()
            #print ("objeto foi instanciado, agora setando o user...")
            #print ("setando o user...")
            objeto.user = request.user
            # print request.user
            objeto.filename = request.session['filename']
            # print request.session['filename']
            objeto.tamanho = request.session['tamanho']
            # print request.session['tamanho']
            texto = request.session['texto']
            objeto.texto = texto
            # print request.session['texto']
            objeto.numsil = request.session['numsil']
            # print request.session['numsil']
            objeto.silabas = request.session['silabas']
            # print request.session['silabas']
            objeto.vel = request.session['vel']
            # print request.session['vel']
            ip = get_client_ip(request)
            # print ip
            if ip == "::1":
                ip = "127.0.0.1"
            print "ip do cliente -", ip
            objeto.ipaddr = ip
            import pygeoip
            gi = pygeoip.GeoIP(
                '/home/zuao/git/matebot/matebotweb/GeoLiteCity.dat')
            if ip == '127.0.0.1' or ip.split(".")[:2] == ['192', '168']:
                geo = {
                    'city': u'Recife',
                    'region_code': u'30',
                    'area_code': 0,
                    'time_zone': 'America/Recife',
                    'dma_code': 0,
                    'metro_code': None,
                    'country_code3': 'BRA',
                    'latitude': -8.050000000000011,
                    'postal_code': None,
                    'longitude': -34.900000000000006,
                    'country_code': 'BR',
                    'country_name': 'Brazil',
                    'continent': 'SA'
                }
            else:
                geo = gi.record_by_addr(ip)
            geotxt = geo['city'] + ", " + geo['country_name']
            print "geo info -", geotxt
            objeto.geo = geotxt
            objeto.latitude = geo['latitude']
            objeto.longitude = geo['longitude']
            print "rotulo -", rotulo
            objeto.rotulo = rotulo
            print "idade -", request.session['idade']
            objeto.idade = request.session['idade']
            print "sexo -", request.session['sexo']
            objeto.sexo = request.session['sexo']

            #analise sentimento no texto
            from textblob import TextBlob as tb
            tblob = tb(texto)
            if tblob.detect_language() != "en":
                try:
                    tblob2 = tb(tblob.translate(to='en').string)
                except:
                    # no caso da traducao falhar passamos o mesmo texto original mesmo (provavelmente a analise de sentimento vai dar 0)
                    # nao é ideal mas melhor que nao tratar a excecao
                    tblob2 = tblob
            polarity = tblob2.sentiment.polarity
            print "polarity -", polarity
            objeto.polarity = polarity

            #print ("salvando objeto...")
            try:
                objeto.save(force_insert=True)
            except Exception, e:
                print "Nao pode escrever no db -", str(e)

            return HttpResponse()  # if everything is OK