def ip2country(request): ip = str(request.REQUEST.get('ip')) geoip = GeoIP(settings.GEOIP, MEMORY_CACHE) c = geoip.country_name_by_addr(ip) c+="; " whois = os.popen("whois %s 2>&1" % ip) file.close for ln in whois: ''' inetnum: 134.36.0.0 - 134.36.255.255 descr: University of Dundee descr: Dundee DD1 4HN descr: Scotland netname: DUNDEE-UNIV descr: University of Dundee country: GB ''' if ln.startswith("inetnum") or ln.startswith("netname") or ln.startswith("descr"): c+=ln.split(":")[1].strip()+"; " if ln.startswith("country"): c+=ln.split(":")[1].strip()+"." break if len(c) > 400: break return HttpResponse(c)
class GraphManager(object): """ Generates and processes the graph based on packets """ def __init__(self, packets, layer=3, geo_ip=os.path.expanduser('~/GeoIP.dat')): self.graph = DiGraph() self.layer = layer self.geo_ip = None self.data = {} try: self.geo_ip = GeoIP(geo_ip) except: logging.warning("could not load GeoIP data") if self.layer == 2: edges = map(self._layer_2_edge, packets) elif self.layer == 3: edges = map(self._layer_3_edge, packets) elif self.layer == 4: edges = map(self._layer_4_edge, packets) else: raise ValueError( "Other layers than 2,3 and 4 are not supported yet!") for src, dst, packet in filter(lambda x: not (x is None), edges): if src in self.graph and dst in self.graph[src]: self.graph[src][dst]['packets'].append(packet) else: self.graph.add_edge(src, dst) self.graph[src][dst]['packets'] = [packet] for node in self.graph.nodes(): self._retrieve_node_info(node) for src, dst in self.graph.edges(): self._retrieve_edge_info(src, dst) def get_in_degree(self, print_stdout=True): unsorted_degrees = self.graph.in_degree() return self._sorted_results(unsorted_degrees, print_stdout) def get_out_degree(self, print_stdout=True): unsorted_degrees = self.graph.out_degree() return self._sorted_results(unsorted_degrees, print_stdout) @staticmethod def _sorted_results(unsorted_degrees, print_stdout): sorted_degrees = OrderedDict( sorted(list(unsorted_degrees), key=lambda t: t[1], reverse=True)) for i in sorted_degrees: if print_stdout: print(sorted_degrees[i], i) return sorted_degrees def _retrieve_node_info(self, node): self.data[node] = {} if self.layer >= 3 and self.geo_ip: if self.layer == 3: self.data[node]['ip'] = node elif self.layer == 4: self.data[node]['ip'] = node.split(':')[0] node_ip = self.data[node]['ip'] try: country = self.geo_ip.country_name_by_addr(node_ip) self.data[node]['country'] = country if country else 'private' except: # it seems like we are not dealing with valid IPs... # best effort approach: skip del self.data[node] #TODO layer 2 info? def _retrieve_edge_info(self, src, dst): edge = self.graph[src][dst] if edge: packets = edge['packets'] edge['layers'] = set( list( itertools.chain( *[set(GraphManager.get_layers(p)) for p in packets]))) edge['transmitted'] = sum(len(p) for p in packets) edge['connections'] = len(packets) @staticmethod def get_layers(packet): return list(GraphManager.expand(packet)) @staticmethod def expand(x): yield x.name while x.payload: x = x.payload yield x.name @staticmethod def _layer_2_edge(packet): return packet[0].src, packet[0].dst, packet @staticmethod def _layer_3_edge(packet): if packet.haslayer(IP): return packet[1].src, packet[1].dst, packet @staticmethod def _layer_4_edge(packet): if any(map(lambda p: packet.haslayer(p), [TCP, UDP])): src = packet[1].src dst = packet[1].dst _ = packet[2] return "%s:%i" % (src, _.sport), "%s:%i" % (dst, _.dport), packet def draw(self, filename=None): graph = self.get_graphviz_format() for node in graph.nodes(): if node not in self.data: # node might be deleted, because it's not legit etc. continue node.attr['shape'] = 'circle' node.attr['fontsize'] = '10' node.attr['width'] = '0.5' if 'country' in self.data[str(node)]: country_label = self.data[str(node)]['country'] if country_label == 'private': node.attr['label'] = str(node) else: node.attr['label'] = "%s (%s)" % (str(node), country_label) if not (country_label == 'private'): node.attr['color'] = 'blue' node.attr['style'] = 'filled' #TODO add color based on country or scan? for edge in graph.edges(): connection = self.graph[edge[0]][edge[1]] edge.attr['label'] = 'transmitted: %i bytes\n%s ' % ( connection['transmitted'], ' | '.join(connection['layers'])) edge.attr['fontsize'] = '8' edge.attr['minlen'] = '2' edge.attr['penwidth'] = min( connection['connections'] * 1.0 / len(self.graph.nodes()), 2.0) graph.layout(prog='dot') graph.draw(filename) def get_graphviz_format(self, filename=None): agraph = networkx.drawing.nx_agraph.to_agraph(self.graph) # remove packet information (blows up file size) for edge in agraph.edges(): del edge.attr['packets'] if filename: agraph.write(filename) return agraph
import re import sys from pygeoip import GeoIP g = GeoIP('GeoIP.dat') r = re.compile(r'20. ([0-9]+) "') d = {} while True: line = sys.stdin.readline() if not line: break match = r.search(line) if match: ip = line.split(' ')[0] country = g.country_name_by_addr(ip) if not country: country = 'Unknown' d[country] = int(match.group(1)) + d.get(country, 0) ds = d.items() ds.sort(lambda x, y: cmp(y[1], x[1])) for k, v in ds: if v > (1024 * 1024 * 1024): print k, '%0.2fGb' % (v / (1024 * 1024 * 1024.0))
class QueryManager(object): """ Implements handling multiple threads used to speed up serverqueries """ def __init__(self): """ Constructor - It starts with some basic initialisations and spawns a coordinator thread which creates more threads to perform the master server query and also the status updates for the servers. """ self.serverqueue = Queue() self.messageque = Queue() self.pulsemessageque = Queue() self.threadcount = 0 self.servercount = 0 self.processedserver = 0 self.filterdcount = 0 self.gui_lock = None self.geo_lock = None coord = Thread(target=self.coordinator) coord.daemon = True coord.start() dbname = Globals.geoip_dir+ '/GeoIP.dat' self.pygeoip = GeoIP(dbname, pygeoip.const.MMAP_CACHE) self.abort = False def start_serverlist_refresh(self, liststore, tab): """ Refreshes the Serverlist of a tab @param liststore - the liststore which contains the servers to be refreshed @param tab - the tab requesting the refresh """ self.tab = tab self.filter = None iter = liststore.iter_children(None) while iter: server = liststore.get_value(iter, 8) self.serverqueue.put(server) iter = liststore.iter_next(iter) self.servercount = self.serverqueue.qsize() gobject.idle_add(tab.clearServerList) self.messageque.put('serverlist_loaded') def startMasterServerQueryThread(self, filter, tab): """ Starts the masterserver query. @param filter - filter to apply @param tab - tab requesting the serverlist """ self.tab = tab self.filter = filter tab.clearServerList() #this message will cause the coordinator to start querying the master #server self.messageque.put('start_master_server_query') def startRecentServersLoadingThread(self, tab): """ Starts loading the recent servers list @param tab - tab rquesting the recent servers """ fm = FileManager() self.tab = tab self.filter = None serverdict = fm.getRecentServers() for key in serverdict: self.serverqueue.put(serverdict[key]) self.servercount = len(serverdict) #notify the coordinator thread, that the serverlist is loaded self.messageque.put('serverlist_loaded') def startFavoritesLoadingThread(self, tab): """ Starts loading the favorites @param tab - the tab requesting the favoriteslist """ fm = FileManager() self.tab = tab self.filter = None serverlist = fm.getFavorites().values() for server in serverlist: self.serverqueue.put(server) self.servercount = len(serverlist) #notify the coordinator thread, that the serverlist is loaded self.messageque.put('serverlist_loaded') def lookup_server(self, server, tab): """ Starts the lookup of a certain server. @param server - the server to be looked up @param tab - the requesting tab """ self.tab = tab self.filter = None self.serverqueue.put(server) self.servercount = 1 self.messageque.put('serverlist_loaded') def coordinator(self): """ Method that runs as coordinator thread. Spawning additional threads based on string messages in the messagueue Messages accepted: start_master_server_query - is the start signal. will cause the coordinator to spawn two new threads. The first is to pulse the progressbar on self.tab every 0.1 seconds. The second thread performs the master server query, when this thread finishes puts the message serverlist_loaded into the messagequeue indicating that the query was succesfull and the serverqueue is filled with servers serverlist_loaded - spawns 10 worker threads that will perform the get_status request of the servers in the serverqueue finished - if the status of all servers has been retreived the last thread puts this message in the messagueue. calls the serverlist_loading_finished method on self.tab and terminates the coordinator thread """ # main thread loop Log.log.debug('Thread:Coordinator started...') while True: try: message = self.messageque.get() if message == 'start_master_server_query': Log.log.info('Thread:Coordinator - start_master_server_' \ +'query signal received') # spawn the pulse progressbar thread pt = Thread(target=self.pulse_progressbar_thread) pt.setDaemon(True) pt.start() #spawns the master server query thread pt = Thread(target=self.master_server_query_thread) pt.setDaemon(True) pt.start() elif message == 'serverlist_loaded': Log.log.info('Thread:Coordinator - received serverlist' \ +'_loaded signal. Queuesize is ' \ + str(self.serverqueue.qsize())) #stop the pulsing of the progressbar self.pulsemessageque.put('stop_pulse') #start 10 worker threads retreiving the status of #the servers in the serverqueue for i in range(10): name = 'Worker_' + str(i+1) t = Thread(target=self.get_server_status_thread, name=name) t.setDaemon(True) t.start() elif message == 'finished': #finish tasks :) Log.log.info('Thread:Coordinator - received the ' \ + 'finished signal') self.gui_lock = threading.RLock() with self.gui_lock: gobject.idle_add(self.tab.serverlist_loading_finished) break elif message == 'all_aborted': #all_aborted tasks :) Log.log.info('Thread:Coordinator - received the ' \ + 'all_aborted signal') self.gui_lock = threading.RLock() with self.gui_lock: gobject.idle_add(self.set_progressbar_aborted) gobject.idle_add(self.tab.serverlist_loading_finished) break except Empty: True def master_server_query_thread(self): """ This method is running as a thread to retreive a list of servers from the master server. """ query = Q3ServerQuery() empty = self.filter.show_empty full = self.filter.show_full #query the urban terror master server serverlist = query.getServerList('master.urbanterror.info' ,27900 ,empty ,full) #put all servers in the serverqueue for server in serverlist: self.serverqueue.put(server) self.servercount = len(serverlist) #notify the coordinator thread, that the serverlist is loaded self.messageque.put('serverlist_loaded') def get_server_status_thread(self): """ This method will run as worker thread to retreive the status of the servers in the serverqueue """ #increment thread count. #the counter will be decreased on exit and compared to 0 #so the last thread can notify the coordinator that all threads finished #their work self.gui_lock = threading.RLock() with self.gui_lock: self.threadcount+=1 Log.log.debug('Thread:' + threading.current_thread().name + \ ' started') # main thread loop while True: try: self.gui_lock = threading.RLock() with self.gui_lock: if self.abort: self.threadcount -= 1 Log.log.info('Thread:' + threading.current_thread().name + \ ' exiting due to abort signal') if self.threadcount == 0: #last thread reached Log.log.info('Thread:' + threading.current_thread().name + \ ' notifying the coordinator thread that all threads ' \ + 'was aborted') self.messageque.put('all_aborted') break server = self.serverqueue.get(False) #perform the statusrequest query = Q3ServerQuery() server = query.getServerStatus(server) #add the server to the gui self.gui_lock = threading.RLock() with self.gui_lock: self.set_location(server) self.processedserver+=1 gobject.idle_add(self.set_progressbar_fraction) if None == self.filter or \ self.filter.does_filter_match_server(server): gobject.idle_add(self.tab.addServer, server) else: self.filterdcount+=1 # server is not added but filterd except Empty: #no more threads in the queue break thread execution self.gui_lock = threading.RLock() with self.gui_lock: self.threadcount -= 1 Log.log.debug('Thread:' + threading.current_thread().name + \ ' finishes working and exiting') if self.threadcount == 0: #last thread reached Log.log.info('Thread:' + threading.current_thread().name + \ ' notifying the coordinator thread that the queue ' \ + 'processing is finished') self.messageque.put('finished') break def pulse_progressbar_thread(self): """ This method runs as a background thread that pulse the progressbar of self.tab every 0.1 seconds """ while True: try: message = self.pulsemessageque.get(True, 0.1) if message == 'stop_pulse': break except Empty: self.gui_lock = threading.RLock() with self.gui_lock: gobject.idle_add(self.pulse_progressbar) def set_progressbar_fraction(self): """ Sets the progressbar fraction. Uses the total servercount and the processed servercount values to calculate the fraction """ if not self.abort: fraction = float(self.processedserver) / float(self.servercount) bartext = None if 1.0 == fraction: bartext = 'finished getting server status - displaying ' \ + str((self.servercount-self.filterdcount)) + \ ' servers (' + str(self.filterdcount) + ' filtered)' self.tab.statusbar.progressbar.set_fraction(0.0) else: bartext = 'fetching server status (' + str(self.processedserver) + \ ' / ' + str(self.servercount) + ') - ' + \ str(self.filterdcount) + ' servers filtered' self.tab.statusbar.progressbar.set_fraction(fraction) self.tab.statusbar.progressbar.set_text(bartext) def pulse_progressbar(self): """ Pulse the progressbar, called by the thread using gobject.idle_add """ self.tab.statusbar.progressbar.set_text('fetching serverlist from master server') self.tab.statusbar.progressbar.pulse() def set_progressbar_aborted(self): """ Sets the text of the progressbar to the aborted message and resets fraction """ self.tab.statusbar.progressbar.set_text('task aborted') self.tab.statusbar.progressbar.set_fraction(0.0) def abort_current_task(self): """ Stops the processing of the queue by setting a abort flag. """ self.gui_lock = threading.RLock() with self.gui_lock: self.abort = True def set_location(self, server): """ Determine location of a server based on the ip adress of the server and set it at the server object Extra threading lock used because there was some strange effects without it. @param - the server object """ self.geo_lock = threading.RLock() with self.geo_lock: #location = country(server.getHost()) location = self.pygeoip.country_code_by_addr(server.getHost()) locname = self.pygeoip.country_name_by_addr(server.getHost()) server.set_location(location) server.set_location_name(locname)
def index(): res_data = {} verbose = False prettyprint = False try: client_ip = request.remote_route[0] except: client_ip = request.environ['REMOTE_ADDR'] pass args = request.query_string.split('&') if 'v' in args: verbose = True if 'pp' in args: prettyprint = True res_data['ip'] = client_ip try: ua = request.environ['HTTP_USER_AGENT'] user_agent = user_agents.parse(ua) res_data['user_agent'] = str(user_agent) except Exception as e: user_agent = False pass if user_agent and not verbose: if user_agent.browser.family == 'Other': response.set_header('Content-type', 'text/plain') return res_data['ip'] + '\r\n' try: ip = GeoIP(config.get('app', 'geoip_database')) res_data['country'] = ip.country_name_by_addr(client_ip) res_data['city'] = ip.record_by_addr(client_ip) res_data['asn'] = ip.asn_by_addr(client_ip) except Exception as e: pass try: name = reversename.from_address(client_ip) answers = resolver.query(name, 'PTR') res_data['hostname'] = [] for answer in answers: res_data['hostname'].append(str(answer).rstrip('.')) except Exception as e: pass if user_agent and verbose: if user_agent.browser.family == 'Other': response.set_header('Content-type', 'application/json') if prettyprint: return json.dumps( res_data, sort_keys=True, indent=4, separators=(',', ': ') ) + '\r\n' return json.dumps(res_data) return template( 'index', ua_info=res_data, page_title='Your IP-address is: ' + res_data['ip'], verbose=verbose )
def hit(request): stable_omero_downloads = 'http://downloads.openmicroscopy.org/latest-stable/omero' agent = None try: agt = request.META.get('HTTP_USER_AGENT', '') if agt is not None and agt.startswith("OMERO."): try: agent = Agent.objects.get(agent_name=agt) except Agent.DoesNotExist: return HttpResponseRedirect(UPGRADE_CHECK_URL) except: logger.error(traceback.format_exc()) return HttpResponseRedirect(UPGRADE_CHECK_URL) else: return HttpResponseRedirect(UPGRADE_CHECK_URL) except: logger.error(traceback.format_exc()) return HttpResponseRedirect(UPGRADE_CHECK_URL) logger.debug("Agent %s" % agent) agent_version = '' update = None try: agent_version = request.REQUEST.get('version') ver = Version.objects.get(pk=1) if agent_version is not None: try: regex = re.compile("^.*?[-]?(\\d+[.]\\d+([.]\\d+)?)[-]?.*?$") agent_cleaned = regex.match(agent_version).group(1) agent_split = agent_cleaned.split(".") local_cleaned = regex.match(ver.version).group(1) local_split = local_cleaned.split(".") rv = (agent_split < local_split) except: rv = True if rv: update = 'Please upgrade to %s. See %s for the latest version.' % (ver, stable_omero_downloads) else: update = 'Please upgrade to %s. See %s for the latest version.' % (ver, stable_omero_downloads) except: logger.debug(traceback.format_exc()) logger.debug("Agent version %s" % agent_version) ip = None try: real_ip = None try: # HTTP_X_FORWARDED_FOR can be a comma-separated list of IPs. The # client's IP will be the first one. # http://code.djangoproject.com/ticket/3872 real_ip = request.META['HTTP_X_FORWARDED_FOR'] logger.debug("HTTP_X_FORWARDED_FOR: %s" % real_ip) real_ip = real_ip.split(",")[-1].strip() except KeyError: real_ip = request.META.get('REMOTE_ADDR') if real_ip is not None: try: ip = IP.objects.get(ip=real_ip) except IP.DoesNotExist: latitude = None longitude = None country = None geoip = GeoIP(settings.GEODAT, STANDARD) gir = geoip.record_by_addr(real_ip) if gir is not None: latitude = gir["latitude"] longitude = gir["longitude"] geoip = GeoIP(settings.GEOIP, MEMORY_CACHE) country = geoip.country_name_by_addr(real_ip) logger.debug("IP: %s, latitude: '%s', longitude: '%s'" % (real_ip, latitude, longitude)) ip = IP(ip=real_ip, latitude=latitude, longitude=longitude, country=country) ip.save() except Exception, x: logger.debug(traceback.format_exc()) raise x
class GraphManager(object): """ Generates and processes the graph based on packets """ def __init__(self, packets, layer=3, geo_ip=os.path.expanduser('~/GeoIP.dat')): self.graph = DiGraph() self.layer = layer self.geo_ip = None self.data = {} try: self.geo_ip = GeoIP(geo_ip) except: logging.warning("could not load GeoIP data") if self.layer == 2: edges = map(self._layer_2_edge, packets) elif self.layer == 3: edges = map(self._layer_3_edge, packets) elif self.layer == 4: edges = map(self._layer_4_edge, packets) else: raise ValueError("Other layers than 2,3 and 4 are not supported yet!") for src, dst, packet in filter(lambda x: not (x is None), edges): if src in self.graph and dst in self.graph[src]: self.graph[src][dst]['packets'].append(packet) else: self.graph.add_edge(src, dst, {'packets': [packet]}) for node in self.graph.nodes(): self._retrieve_node_info(node) for src, dst in self.graph.edges(): self._retrieve_edge_info(src, dst) def get_in_degree(self, print_stdout=True): unsorted_degrees = self.graph.in_degree() return self._sorted_results(unsorted_degrees, print_stdout) def get_out_degree(self, print_stdout=True): unsorted_degrees = self.graph.out_degree() return self._sorted_results(unsorted_degrees, print_stdout) @staticmethod def _sorted_results(unsorted_degrees, print_stdout): sorted_degrees = OrderedDict(sorted(unsorted_degrees.items(), key=lambda t: t[1], reverse=True)) for i in sorted_degrees: if print_stdout: print(sorted_degrees[i], i) return sorted_degrees def _retrieve_node_info(self, node): self.data[node] = {} if self.layer >= 3 and self.geo_ip: if self.layer == 3: self.data[node]['ip'] = node elif self.layer == 4: self.data[node]['ip'] = node.split(':')[0] node_ip = self.data[node]['ip'] country = self.geo_ip.country_name_by_addr(node_ip) self.data[node]['country'] = country if country else 'private' #TODO layer 2 info? def _retrieve_edge_info(self, src, dst): edge = self.graph[src][dst] if edge: packets = edge['packets'] edge['layers'] = set(list(itertools.chain(*[set(GraphManager.get_layers(p)) for p in packets]))) edge['transmitted'] = sum(len(p) for p in packets) edge['connections'] = len(packets) @staticmethod def get_layers(packet): return list(GraphManager.expand(packet)) @staticmethod def expand(x): yield x.name while x.payload: x = x.payload yield x.name @staticmethod def _layer_2_edge(packet): return packet[0].src, packet[0].dst, packet @staticmethod def _layer_3_edge(packet): if packet.haslayer(IP): return packet[1].src, packet[1].dst, packet @staticmethod def _layer_4_edge(packet): if any(map(lambda p: packet.haslayer(p), [TCP, UDP])): src = packet[1].src dst = packet[1].dst _ = packet[2] return "%s:%i" % (src, _.sport), "%s:%i" % (dst, _.dport), packet def draw(self, filename=None, figsize=(50, 50)): graph = self.get_graphviz_format() for node in graph.nodes(): node.attr['shape'] = 'circle' node.attr['fontsize'] = '10' node.attr['width'] = '0.5' if 'country' in self.data[str(node)]: country_label = self.data[str(node)]['country'] if country_label == 'private': node.attr['label'] = str(node) else: node.attr['label'] = "%s (%s)" % (str(node), country_label) if not (country_label == 'private'): node.attr['color'] = 'blue' node.attr['style'] = 'filled' #TODO add color based on country or scan? for edge in graph.edges(): connection = self.graph[edge[0]][edge[1]] edge.attr['label'] = 'transmitted: %i bytes\n%s ' % (connection['transmitted'], ' | '.join(connection['layers'])) edge.attr['fontsize'] = '8' edge.attr['minlen'] = '2' edge.attr['penwidth'] = min(connection['connections'] * 1.0 / len(self.graph.nodes()), 2.0) graph.layout(prog='dot') graph.draw(filename) #TODO do we need a .dot file export? def get_graphviz_format(self, filename=None): agraph = networkx.to_agraph(self.graph) if filename: agraph.write(filename) return agraph