def dologs(): """Process the last weeks worth of logs, build a digested version""" # format: time, type, version, client, id, lat, long # where # time = time of the call # type = init (new), revisit (init with prev), travel (a travel time request) # version = client version of nextquery # id = a unique id (for tracking multiple use by clients, without recording ip address, etc.) # lat, long = only present if this is a travel request. weekago = totimestamp(datetime.now() - timedelta(days=7)) countaccess = 0 counterror = 0 countnew = 0 output = StringIO.StringIO() print >> output, "Time,Type,NFVersion,ID,Lat,Long\n" for record in logservice.fetch(start_time=weekago): urlbits = record.resource.split('/') if len(urlbits) < 2: continue calltype = urlbits[1] version = ("" if len(urlbits)<3 else urlbits[2]) param = ("" if len(urlbits)<4 else urlbits[3]) if calltype in ("_ah", "tasks"): continue # we don't care about these if calltype == "init" and param != "": calltype = "revisit" if calltype == "init": countnew+=1 countaccess+=1 outputtime = datetime.fromtimestamp(record.start_time).strftime("%Y-%m-%d %X %a") print >>output, outputtime, ",", calltype, ",",version, ",", getId(record.ip), if calltype == "traveltimes": print >>output, ",", param else: print >>output, ",," for record in logservice.fetch(start_time=weekago,minimum_log_level=logservice.LOG_LEVEL_WARNING): urlbits = record.resource.split('/') if urlbits[1] in ("init","traveltimes","tasks"): counterror+=1 print >> output, "\nAccesses: ", countaccess print >> output, "Unique: ", len(ipmap.keys()) print >> output, "New: ", countnew print >> output, "Errors: ", counterror return output.getvalue()
def getapplogs(start_offset, how_many): end_time = time.time() i=0 s='' end_offset=start_offset+how_many for req_log in logservice.fetch(end_time=end_time, offset=None, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): for app_log in req_log.app_logs: if i in range(start_offset,end_offset): lmessage=app_log.message if (True == lmessage.startswith('%lg%')): lmessage=re.sub('%lg%','',lmessage) s=("""%s %s""" %(lmessage,s)) elif (i>end_offset): break i+=1 if (''==s ): print 'No logging message specified<br />' return s
def getapplogs(start_offset, how_many): end_time = time.time() i = 0 s = '' end_offset = start_offset + how_many for req_log in logservice.fetch( end_time=end_time, offset=None, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): for app_log in req_log.app_logs: if i in range(start_offset, end_offset): lmessage = app_log.message if (True == lmessage.startswith('%lg%')): lmessage = re.sub('%lg%', '', lmessage) s = ("""%s %s""" % (lmessage, s)) elif (i > end_offset): break i += 1 if ('' == s): print 'No logging message specified<br />' return s
def get_logs(log_ids, log_level): for request_log in logservice.fetch(minimum_log_level=log_level, include_incomplete=True, include_app_logs=True, request_ids=log_ids): d = {name: getattr(request_log, name) for name, val in request_log.__class__.__dict__.iteritems() if isinstance(val, property) and not name.startswith('_') } d['start_time'] = datetime.datetime.fromtimestamp( request_log.start_time) if request_log.end_time: d['end_time'] = datetime.datetime.fromtimestamp( request_log.end_time) d['duration'] = (d['end_time'] - d['start_time']).total_seconds() d['app_logs'] = [{ 'time': datetime.datetime.fromtimestamp(app_log.time), 'level': app_log.level, 'message': app_log.message } for app_log in d['app_logs'] if app_log.level >= log_level ] yield d
def do(self, proc): proc = str( proc ) try: offset = self.request.get('offset') or None if offset: offset = urlsafe_b64decode(str(offset)) except: offset = None end_time = time() count = 5 show_next = False last_offset = None i = 1 for req_log in logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): ip = str( req_log.ip ) rq = str( req_log ) meth = str( req_log.method ) rec = str( req_log.resource ) if rec == '/celtic.png': self.response.headers['Content-Type'] = 'image/png' a = open( 'celtik.png', 'rb' ) for e in a: self.response.out.write( e ) if not rec == '/celtic.png': logging.info( 'Bad Request: %s %s' % ( proc, rec ) )
def get(self): end_time = time.time() start_time = end_time - 6 * 60 * 60 # 6 hours before now- same as cronjob interval. html = '' report_needed = False for req_log in logservice.fetch(start_time=start_time, end_time=end_time, minimum_log_level=logservice.LOG_LEVEL_ERROR, include_app_logs=True): report_needed = True html += '%s - %s <br/>\n' % (req_log.resource, req_log.method) html += '%s <br/>\n' % req_log.ip html += '<br/>\n' for app_log in req_log.app_logs: html += 'Date: %s' % datetime.datetime.fromtimestamp(app_log.time).strftime('%D %T UTC') html += '<b><pre>%s</pre></b><br/>\n' % app_log.message html += '<hr>\n' if report_needed: mail.send_mail(sender="*****@*****.**", to='*****@*****.**', subject='Whaler Errors', body=html, html=html) self.response.out.write(html)
def get(self): logging.info('Starting Main handler') # Get the incoming offset param from the Next link to advance through # the logs. (The first time the page is loaded, there won't be any offset.) try: offset = self.request.get('offset') or None if offset: offset = base64.urlsafe_b64decode(str(offset)) except TypeError: offset = None # Set up end time for our query. end_time = time.time() # Count specifies the max number of RequestLogs shown at one time. # Use a boolean to initially turn off visiblity of the "Next" link. count = 5 show_next = False last_offset = None # Iterate through all the RequestLog objects, displaying some fields and # iterate through all AppLogs beloging to each RequestLog count times. # In each iteration, save the offset to last_offset; the last one when # count is reached will be used for the link. i = 0 for req_log in logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): self.response.out.write('<br /> REQUEST LOG <br />') self.response.out.write( 'IP: %s <br /> Method: %s <br /> Resource: %s <br />' % (req_log.ip, req_log.method, req_log.resource)) self.response.out.write( 'Date: %s<br />' % #datetime.datetime.fromtimestamp(req_log.end_time).strftime('%D %T UTC')) str(req_log.end_time)) last_offset= req_log.offset i += 1 for app_log in req_log.app_logs: self.response.out.write('<br />APP LOG<br />') self.response.out.write( 'Date: %s<br />' % #datetime.datetime.fromtimestamp(app_log.time).strftime('%D %T UTC')) str(req_log.end_time)) self.response.out.write('<br />Message: %s<br />' % app_log.message) if i >= count: show_next = True break # Prepare the offset URL parameters, if any. if show_next: query = self.request.GET query['offset'] = base64.urlsafe_b64encode(last_offset) next_link = urllib.urlencode(query) self.response.out.write('<a href="/rest/logHandler?%s">Next</a>' % next_link)
def do_not_run_test_logservice_fetch(request_id): """This test fails at logservice.fetch""" logging.info('TESTING') found_log = False for req_log in logservice.fetch(request_ids=[request_id], include_app_logs=True): for app_log in req_log.app_logs: if app_log.message == 'TESTING': found_log = True assert found_log
def get_logs(offset=None): # Logs are read backwards from the given end time. This specifies to read # all logs up until now. end_time = time.time() logs = logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True) return logs
def get_logs(offset=None): # Logs are read backwards from the given end time. This specifies to read # all logs up until now. end_time = time.time() logs = logservice.fetch( end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True) return logs
def get_logs(offset=None, log_level=1): """ 依目前的時間取得記錄 """ LOG_LEVELS = [ logservice.LOG_LEVEL_DEBUG, logservice.LOG_LEVEL_INFO, logservice.LOG_LEVEL_WARNING, logservice.LOG_LEVEL_ERROR, logservice.LOG_LEVEL_CRITICAL ] end_time = time.time() return logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=LOG_LEVELS[log_level], include_app_logs=True)
def countapplogs(): end_time = time.time() count=0 for req_log in logservice.fetch(end_time=end_time, offset=None, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): for app_log in req_log.app_logs: lmessage=app_log.message if (True == lmessage.startswith('%lg%')): count+=1 return count
def countapplogs(): end_time = time.time() count = 0 for req_log in logservice.fetch( end_time=end_time, offset=None, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): for app_log in req_log.app_logs: lmessage = app_log.message if (True == lmessage.startswith('%lg%')): count += 1 return count
def readclienthistory(stream): """Write client access data from logs""" # format: time, req-type, ip, lat, long # where req-type = {1=new_init, 2=re_init, 3=travel_time} # and lat, long are empty unless this is a travel request for record in logservice.fetch(): request_type = record.resource # URL string if '/init' in request_type: isnew = (1 if re.search('/[0-9]',request_type) == None else 2) stream.write("%d, %d, %s,,\n" % (record.start_time, isnew, record.ip)) elif '/travel' in request_type: mob = re.search('/([0-9.-]+),([0-9.-]+)',request_type) (l1,l2) = ((mob.group(1),mob.group(2)) if mob != None else ('','')) stream.write("%d, 3, %s, %s, %s\n" % (record.start_time, record.ip, l1, l2)) else: pass # ignore all other message types
def _get_log_messages(self, offset, count, min_level=logservice.LOG_LEVEL_DEBUG): logs = [] index = 1 for log in logservice.fetch(end_time=time.time(), offset=None, minimum_log_level=min_level, include_app_logs=True): if log.offset == offset: break log.timestamp = datetime.datetime.fromtimestamp(log.start_time) for app_log in log.app_logs: app_log.timestamp = datetime.datetime.fromtimestamp(app_log.time) app_log.level_name = LOG_LEVELS[app_log.level] logs.append(log) index += 1 if index > count: break return logs
def get(self): if not isUserAdmin(self): self.session[LOGIN_NEXT_PAGE_KEY] = self.URL self.redirect("/") return now = int(time.mktime(time.gmtime())) # Fetch 24 hours of logs start = now - 3600 * 24 logs = fetch(start_time=start, end_time=now, minimum_log_level=logservice.LOG_LEVEL_INFO, include_incomplete=False, include_app_logs=True) messages = [] for log in logs: for appLog in log.app_logs: message = unicode(appLog.message.strip(codecs.BOM_UTF8), 'utf-8') if message.startswith("2013"): messages.append(message) template_values = { 'messages':messages } template = jinja_environment.get_template('templates/log/logLines.html') self.printPage("Logs", template.render(template_values), False, False)
def get_logs(log_ids, log_level): for request_log in logservice.fetch(minimum_log_level=log_level, include_incomplete=True, include_app_logs=True, request_ids=log_ids): d = { name: getattr(request_log, name) for name, val in request_log.__class__.__dict__.iteritems() if isinstance(val, property) and not name.startswith('_') } d['start_time'] = datetime.datetime.fromtimestamp( request_log.start_time) if request_log.end_time: d['end_time'] = datetime.datetime.fromtimestamp( request_log.end_time) d['duration'] = (d['end_time'] - d['start_time']).total_seconds() d['app_logs'] = [{ 'time': datetime.datetime.fromtimestamp(app_log.time), 'level': app_log.level, 'message': app_log.message } for app_log in d['app_logs'] if app_log.level >= log_level] yield d
def get(self): end_date = self.request.get('end_date', None) end_time = self.request.get('end_time', None) start_date = self.request.get('start_date', None) start_time = self.request.get('start_time', None) count = self.request.get('count', None) offset = None end_time = time.time() if end_time is None else float(end_time) count = None if count is None else int(count) if end_date: end_time = time.mktime(time.strptime(end_date, "%Y%m%d")) if start_date: start_time = time.mktime(time.strptime(start_date, "%Y%m%d")) class MagicEncoder(json.JSONEncoder): def default(self, obj): use_dict = [logservice.AppLog, logservice.log_service_pb.RequestLog, logservice.log_service_pb.LogLine, logservice.log_service_pb.LogOffset] skip = [logservice.RequestLog] if any(isinstance(obj, x) for x in use_dict): return obj.__dict__ elif any(isinstance(obj, x) for x in skip): o = {} attributes = [ 'api_mcycles', 'app_engine_release', 'app_id', 'app_logs', 'combined', 'cost', 'end_time', 'finished', 'host', 'http_version', 'instance_key', 'ip', 'latency', 'mcycles', 'method', 'module_id', 'nickname', 'offset', 'pending_time', 'referrer', 'replica_index', 'request_id', 'resource', 'response_size', 'start_time', 'status', 'task_name', 'task_queue_name', 'url_map_entry', 'user_agent', 'version_id', 'was_loading_request'] for a in attributes: f = getattr(obj, a) if f: if callable(f): o[a] = f() else: o[a] = f return o elif 'Lock' in str(type(obj)): return {} else: return json.JSONEncoder.default(self, obj) #wish json could stream, till then this will hog RAM log_records = [] log_records_gen = logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True) counter = 0 for idx, log_record in enumerate(log_records_gen): if start_time and log_record.start_time < start_time: continue log_records.append(log_record) if count and counter >= count: break counter += 1 self.response.headers['Content-Type'] = 'application/json' self.response.out.write(json.dumps(log_records, indent=True, cls=MagicEncoder))
def index(): loglines = [] req_logs = logservice.fetch(minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True, include_incomplete=True) req_logs= list(itertools.islice(req_logs, 0, 10)) return render_template('hello.html', req_logs=req_logs)
def fetch_logs(self, time_period, save_to_file=False, send_to_es=False, send_to_udp=False): f = lambda: (self.username, self.password) logger.info("Successfully authenticated") logger.info("Fetching logs from %s to %s (GAE TZ)" % (time_period['start_human'], time_period['end_human'])) end = time_period['end'] start = time_period['start'] start_human = time_period['start_human'] intervals = _split_time_period(start, end) i = 0 for interval in intervals: try: start, end, start_human = interval index_name = start_human.strftime('%Y.%m.%d') dest = '%s-%s.log' % (app_name, start_human.strftime('%Y-%m-%d')) logger.info("Interval : %s - %s %s" % (start, end, start_human)) save_recovery_log(start) lines = [] offset = None remote_api_stub.ConfigureRemoteApiForOAuth( self.app_name + '.appspot.com', '/remote_api') for req_log in logservice.fetch( end_time=end, start_time=start, minimum_log_level=logservice.LOG_LEVEL_INFO, version_ids=self.version_ids, include_app_logs=True, include_incomplete=True, offset=offset): logger.debug("Retrieved - %s" % req_log.combined) lines.append(self._prepare_json(dest, req_log)) i = i + 1 if i % 100 == 0: logger.info("Fetched %d req logs so far" % i) # end fetch if lines: if send_to_es: self.redis_transports.send_to_es( index_name, dest, lines) else: self.redis_transports.callback(dest, lines) #if send_to_udp: # self.redis_transports.send_to_udp(dest, lines, self.udp_host, self.udp_port) if save_to_file: f = file(os.path.join(save_to_file, dest), 'a') f.write('\n'.join([x['line'] for x in lines])) f.close() if send_to_es: logger.info("Save to es %s", len(lines)) else: logger.info("Save to redis %s", len(lines)) # end interval except KeyboardInterrupt: return except: logger.error("Something went wrong", exc_info=True) continue logger.info("Retrieved %d logs. Done." % i) return ""
def get(self): logging.info('Starting Main handler') # Get the incoming offset param from the Next link to advance through # the logs. (The first time the page is loaded, there won't be any offset.) start_time_set=False try: offset = self.request.get('offset') or None if offset: offset = base64.urlsafe_b64decode(str(offset)) except TypeError: offset = None try: start_time = self.request.get('start_time') or None if start_time: start_time = float(base64.urlsafe_b64decode(str(start_time))) start_time_set=True except TypeError: start_time = None start_time_set=False try: filter = str(self.request.get('filter')) or None except TypeError: filter = None # Set up end time for our query. # Count specifies the max number of RequestLogs shown at one time. # Use a boolean to initially turn off visiblity of the "Next" link. count = 1000 show_next = True last_offset = 5000 dt=datetime.datetime.now() tt=dt.timetuple() year=tt[0] month=tt[1] ttt=time.strptime((("01 %s %s") % (month,year)), "%d %m %Y") if not start_time_set: end_time = time.time() start_time = time.mktime(ttt) else: dt2=datetime.datetime.utcfromtimestamp(float(start_time)) tt2=dt2.timetuple() year2=tt2[0] month2=tt2[1] month2=month2+1 if month2==13: month2=1 year2=year2+1 ttt2=time.strptime((("01 %s %s") % (month2,year2)), "%d %m %Y") end_time=time.mktime(ttt2) dt3=datetime.datetime.utcfromtimestamp(float(start_time)) tt3=dt3.timetuple() year3=tt3[0] month3=tt3[1] month3=month3-1 if month3==0: month3=12 year3=year3-1 ttt3=time.strptime((("01 %s %s") % (month3,year3)), "%d %m %Y") start_time_next=time.mktime(ttt3) # Iterate through all the RequestLog objects, displaying some fields and # iterate through all AppLogs beloging to each RequestLog count times. # In each iteration, save the offset to last_offset; the last one when # count is reached will be used for the link. i = 0 for req_log in logservice.fetch(start_time=start_time,end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=False): referrer=req_log.referrer ip=req_log.ip if filter and referrer and not string.find(referrer, filter) == -1: # self.response.out.write("<br /> REQUEST LOG <br />") # self.respons self.response.out.write("""<a href=\"/logs3?filter=%s\">IP</a> %s <br />""" % (ip, req_log.combined)) i += 1 else: if not filter: self.response.out.write("""<a href=\"/logs3?filter=%s\">IP</a> %s <br />""" % (ip, req_log.combined)) i += 1 # self.response.out.write("""IP: %s <br /> Method: %s <br /> # Resource: %s <br />""" % (req_log.ip, # req_log.method, req_log.resource)) # self.response.out.write("Date: "+datetime.datetime.fromtimestamp(req_log.end_time).strftime('%D %T UTC') +"<br />") last_offset= req_log.offset for app_log in req_log.app_logs: self.response.out.write("<br />APP LOG<br />") self.response.out.write("Date: "+datetime.datetime.fromtimestamp(app_log.time).strftime('%Y-%m-%d %H:%M:%S UTC') +"<br />") self.response.out.write("<br />Message: "+app_log.message+"<br />") if i >= count: show_next = True break # Prepare the offset URL parameters, if any. if show_next: query = self.request.GET query['offset'] = base64.urlsafe_b64encode(last_offset) query['start_time'] = base64.urlsafe_b64encode(("%s")%(start_time_next)) next_link = urllib.urlencode(query) self.response.out.write("<a href=\"/logs2?"+next_link+"\">Next</a>") self.response.out.write("<br />")
def _offload_logs(offload_run_key): to_be_saved = list() write_queue = Queue() def writer(): current_date = None gcs_fh = None while True: packet = write_queue.get() if packet is None: if gcs_fh: gcs_fh.close() return # Exit thread date, buf, offset, to_be_saved = packet if current_date is None or current_date != date: if gcs_fh: gcs_fh.close() current_date = date gcs_fh = _get_bucket_file_handle(date) gcs_fh.write(buf.getvalue()) if offset is not None: offload_run = db.get(offload_run_key) offload_run.offset = offset to_be_saved.append(offload_run) if to_be_saved: db.put(to_be_saved) done = False offload_run = db.get(offload_run_key) offset = offload_run.offset thread = threading.Thread(target=writer) thread.daemon = True thread.start() try: buf_date = None buf = StringIO() start = now() for requestLog in logservice.fetch(offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_incomplete=False, include_app_logs=True): if offset is None: # This is the first request # ==> Store the request id offset_settings = OffloadSettings.get_instance() offset_settings.until_request_id = requestLog.request_id to_be_saved.append(offset_settings) elif requestLog.request_id == offload_run.until_request_id: if buf_date: write_queue.put((buf_date, buf, None, to_be_saved)) to_be_saved = list() db.delete(offload_run_key) # This job is done done = True break offset = requestLog.offset date = datetime.datetime.fromtimestamp(requestLog.start_time).date() if not buf_date: buf_date = date elif date != buf_date: if buf.tell() > 0: write_queue.put((buf_date, buf, offset, to_be_saved)) to_be_saved = list() buf = StringIO() buf_date = date elif buf.tell() > MAX_BUF_SIZE: write_queue.put((buf_date, buf, offset, to_be_saved)) to_be_saved = list() buf = StringIO() elif now() - start > 9 * 60: # Deffered deadline = 10 minutes write_queue.put((buf_date, buf, offset, to_be_saved)) break for appLog in requestLog.app_logs: if appLog.message and appLog.message.startswith(OFFLOAD_HEADER): buf.write(appLog.message[OFFLOAD_HEADER_LENGTH + 1:]) buf.write("\n") else: if buf.tell() > 0: write_queue.put((buf_date, buf, None, to_be_saved)) done = True finally: write_queue.put(None) # Exit writer thread thread.join() if not done: offload_logs(offload_run_key)
def fetch_logs(time_period, recovery_log, username, password, app_name, version_ids, offset=None, dest="/tmp/gae_log.log", append=False): f = lambda: (username, password) try: remote_api_stub.ConfigureRemoteApi(None, REMOTE_API_PATH, f, app_name, secure=True) except ConfigurationError: # Token expired? logger.exception( "Token validation failed. Probably expired. Will retry") remote_api_stub.ConfigureRemoteApi(None, REMOTE_API_PATH, f, app_name, secure=True) logger.info("Successfully authenticated") version_ids = version_ids logger.info("Fetching logs from %s to %s (GAE TZ)" % (time_period['start_human'], time_period['end_human'])) end = time_period['end'] start = time_period['start'] intervals = _split_time_period(start, end) # TODO - move to classes instead of globals global last_time_period last_time_period = time_period i = 0 if append: mode = 'a' else: mode = 'w' f = open(dest, mode) try: for interval in intervals: start, end = interval logger.info("Interval : %s - %s" % (start, end)) for req_log in logservice.fetch( end_time=end, start_time=start, minimum_log_level=logservice.LOG_LEVEL_INFO, version_ids=version_ids, include_app_logs=True, include_incomplete=True, offset=offset): logger.debug("Retrieved - %s" % req_log.combined) i = i + 1 if i % 100 == 0: logger.info("Fetched %d req logs so far" % i) f.write(_prepare_json(req_log)) f.write('\n') # keeping track in case - if need to resume global last_offset last_offset = req_log.offset # end fetch # end interval except: logger.exception("Something went wrong") save_recovery_info() logger.info("Retrieved %d logs. Done." % i) f.close() return ""
def fetch_logs(self, time_period, save_to_file=False, send_to_es=False, send_to_udp=False): f = lambda: (self.username, self.password) logger.info("Successfully authenticated") logger.info("Fetching logs from %s to %s (GAE TZ)" % (time_period['start_human'], time_period['end_human'])) end = time_period['end'] start = time_period['start'] start_human = time_period['start_human'] intervals = _split_time_period(start, end) i = 0 for interval in intervals: try: start, end, start_human = interval index_name = start_human.strftime('%Y.%m.%d') dest = '%s-%s.log' % (app_name, start_human.strftime('%Y-%m-%d')) logger.info("Interval : %s - %s %s" % (start, end, start_human)) save_recovery_log(start) lines = [] offset = None remote_api_stub.ConfigureRemoteApiForOAuth(self.app_name + '.appspot.com', '/remote_api') for req_log in logservice.fetch(end_time=end, start_time=start, minimum_log_level=logservice.LOG_LEVEL_INFO, version_ids=self.version_ids, include_app_logs=True, include_incomplete=True, offset=offset): logger.debug("Retrieved - %s" % req_log.combined) lines.append(self._prepare_json(dest, req_log)) i = i + 1 if i % 100 == 0: logger.info("Fetched %d req logs so far" % i) # end fetch if lines: if send_to_es: self.redis_transports.send_to_es(index_name, dest, lines) else: self.redis_transports.callback(dest, lines) #if send_to_udp: # self.redis_transports.send_to_udp(dest, lines, self.udp_host, self.udp_port) if save_to_file: f = file(os.path.join(save_to_file, dest), 'a') f.write('\n'.join([x['line'] for x in lines])) f.close() if send_to_es: logger.info("Save to es %s", len(lines)) else: logger.info("Save to redis %s", len(lines)) # end interval except KeyboardInterrupt: return except: logger.error("Something went wrong", exc_info=True) continue logger.info("Retrieved %d logs. Done." % i) return ""
def get(self): logging.info('Starting Main handler') # Get the incoming offset param from the Next link to advance through # the logs. (The first time the page is loaded, there won't be any offset.) try: offset = self.request.get('offset') or None if offset: offset = base64.urlsafe_b64decode(str(offset)) except TypeError: offset = None # Set up end time for our query. end_time = time.time() # Count specifies the max number of RequestLogs shown at one time. # Use a boolean to initially turn off visiblity of the "Next" link. count = 500 show_next = True last_offset = None url_map_entry = None # Iterate through all the RequestLog objects, displaying some fields and # iterate through all AppLogs beloging to each RequestLog count times. # In each iteration, save the offset to last_offset; the last one when # count is reached will be used for the link. i = 0 for req_log in logservice.fetch(end_time=end_time, offset=offset, minimum_log_level=logservice.LOG_LEVEL_INFO, include_app_logs=True): self.response.out.write("<br /> REQUEST LOG <br />\n") # self.respons # self.response.out.write("Date: " + datetime.datetime.fromtimestamp(req_log.end_time).strftime('%D %T UTC') + " <br />\n") self.response.out.write("Date: " + datetime.datetime.fromtimestamp(req_log.end_time).strftime('%H:%M:%S %Y-%m-%d UTC') + " <br />\n") # if req_log.url_map_entry: # self.response.out.write((""" <a href=\"%s\">%s</a><br />\n""" % (req_log.url_map_entry, req_log.offset))) # else: # self.response.out.write(" <br />\n") self.response.out.write("""%s <br />\n""" % (req_log.combined)) # self.response.out.write("""IP: %s <br /> Method: %s <br /> # Resource: %s <br />""" % (req_log.ip, # req_log.method, req_log.resource)) # self.response.out.write("Date: "+datetime.datetime.fromtimestamp(req_log.end_time).strftime('%D %T UTC') +"<br />") last_offset= req_log.offset i += 1 for app_log in req_log.app_logs: self.response.out.write("APP LOG<br />\n") # self.response.out.write("Date: "+datetime.datetime.fromtimestamp(app_log.time).strftime('%D %T UTC') +"<br />\n") self.response.out.write("Date: "+datetime.datetime.fromtimestamp(app_log.time).strftime('%H:%M:%S %Y-%m-%d UTC') +"<br />\n") self.response.out.write("Message: "+app_log.message+"<br />\n") messagetxt = app_log.message messarray = messagetxt.split("; ") for messvalue in messarray: messvalue2=messvalue.split(": ") mess = {} if len(messvalue2)>1: [key,val]=messvalue2[:2] mess[key]=val if 'link' in mess: statslink = ("<a href=\"%s\">%s</a>" % (mess['link'],mess['link'])) self.response.out.write("<br />STATS DETAILS: %s<br />" % (statslink)) if i >= count: show_next = True break self.response.out.write("<br />\n") # Prepare the offset URL parameters, if any. if show_next: query = self.request.GET query['offset'] = base64.urlsafe_b64encode(last_offset) next_link = urllib.urlencode(query) self.response.out.write("<a href=\"/logs?"+next_link+"\">Next</a>") self.response.out.write("<br />\n")
def fetch_logs(time_period, recovery_log, username, password, app_name, version_ids, offset=None, dest="/tmp/gae_log.log",append=False): f = lambda : (username, password) try: remote_api_stub.ConfigureRemoteApi(None, REMOTE_API_PATH, f, app_name) except ConfigurationError: # Token expired? logger.exception("Token validation failed. Probably expired. Will retry") remote_api_stub.ConfigureRemoteApi(None, REMOTE_API_PATH, f, app_name) logger.info("Successfully authenticated") version_ids = version_ids logger.info("Fetching logs from %s to %s (GAE TZ)" % (time_period['start_human'],time_period['end_human'])) end = time_period['end'] start = time_period['start'] intervals = _split_time_period(start,end) # TODO - move to classes instead of globals global last_time_period last_time_period = time_period i = 0 if append: mode = 'a' else: mode = 'w' f = open(dest, mode) try: for interval in intervals: start, end = interval logger.info("Interval : %s - %s" % (start,end)) for req_log in logservice.fetch(end_time=end, start_time=start, minimum_log_level=logservice.LOG_LEVEL_INFO, version_ids=version_ids, include_app_logs=True, include_incomplete=True, offset=offset): logger.debug("Retrieved - %s" % req_log.combined) i = i + 1 if i % 100 == 0: logger.info("Fetched %d req logs so far" % i) f.write(_prepare_json(req_log)) f.write('\n') # keeping track in case - if need to resume global last_offset last_offset = req_log.offset # end fetch # end interval except: logger.exception("Something went wrong") save_recovery_info() logger.info("Retrieved %d logs. Done." % i) f.close() return ""