def find_meta_data(self, path, *args): # initial setup path = self.get_file_path() + path dictionary = dict() for arg in args: dictionary[arg] = "" # read_file_line_by_line() with open(path) as file: for line in file: if line.startswith(";"): self.examine_line(dictionary, line, ",") else: file.close() break # self.__reverse_read_file() file = FileReadBackwards(path) for line in file: if line.startswith(";") and ":" in line: self.examine_line(dictionary, line, ":") else: file.close() break return dictionary
def continue_index(filename): f = FileReadBackwards(filename) last_line = f.__iter__().__next__() try: print(last_line) last_line = last_line.strip().replace('"', '').strip("}").strip("{").split( ":", 1) # because f**k. return int(last_line[0]) except: print( "Ups. There has been a problem with continue file. Please fix this shit." ) sys.exit(-1)
def read_log(entryType, nEntries): fo = FileReadBackwards(LOG_FILE_PATH, encoding="utf-8") entriesRead= 0 entries = [] for line in fo: entry_dict = log_entry_decode(line) if int(entry_dict['EntryID']) == entryType.value: entriesRead += 1 entries.append(entry_dict) if entriesRead == nEntries: break fo.close() return entries
def __init__(self, path='/var/log/pwnagotchi.log'): self.path = path self.last_session = None self.last_session_id = '' self.last_saved_session_id = '' self.duration = '' self.duration_human = '' self.deauthed = 0 self.associated = 0 self.handshakes = 0 self.peers = 0 self.last_peer = None self._peer_parser = re.compile( 'detected unit (.+)@(.+) \(v.+\) on channel \d+ \(([\d\-]+) dBm\) \[sid:(.+) pwnd_tot:(\d+) uptime:(\d+)\]' ) lines = [] with FileReadBackwards(self.path, encoding="utf-8") as fp: for line in fp: line = line.strip() if line != "" and line[0] != '[': continue lines.append(line) if SessionParser.START_TOKEN in line: break lines.reverse() self.last_session = lines self.last_session_id = hashlib.md5(lines[0].encode()).hexdigest() self.last_saved_session_id = self._get_last_saved_session_id() self._parse_stats()
def readPlayList(): retnext = 0 global currTrack global homedir global playList global direction nextTrack = '' if direction == 'resume' and currTrack != '': nextTrack = currTrack return (nextTrack) if direction == 'previous': hplayList = FileReadBackwards(homedir + 'playList' + playList + '.txt', encoding="utf-8") else: hplayList = open(homedir + 'playList' + playList + '.txt', 'r') for f in hplayList: f = f.rstrip('\n') if retnext == 1: nextTrack = f break if currTrack == "": nextTrack = f break if currTrack == f: retnext = 1 return (nextTrack)
def log_get_most_recent_lines(limit): """Get lines from the back of the log.""" lines = [] with FileReadBackwards(LOGFILE_PATH, encoding="utf-8") as frb: for line in itertools.islice(frb, limit): lines.append(line) return lines[::-1]
def gather_plot_stats(TEST,DURATION,CYCLE_PERIOD,DEVICE_TYPE,PREDICTOR,OUTPUT_DIR,DATE): checker = { "powerPred.overall_fp_rate":False , "powerPred.overall_hit_rate":False} HOME = os.environ['HOME'] test_name = TEST +'_'+DURATION+'_' +CYCLE_PERIOD+'_'+DEVICE_TYPE+'_'+PREDICTOR print(test_name) path = HOME + '/' + OUTPUT_DIR + '/gem5_out/' + test_name + '/stats.txt' storage = {} for k in checker.keys(): storage[k] = 0 with FileReadBackwards(path, encoding="utf-8") as frb: while True: line = frb.readline() if not line: break for k in checker.keys(): if k in line: checker[k] = True if (line.split()[1]) == 'nan': val = 0 else: val = float(line.split()[1]) storage[k] = val if reduce(lambda a, b: a and b, checker.values()): break return storage
def match_error_string(file_path, time_a, time_b, match_string): """Searches a string in a log file for all the lines where date is between time_a and time_b :param logs_dir: path of the directory where log file is located :param time_a: time which defines the starting point of the time frame :param time_b: time which defines the ending point of the time frame :param match_string: string to be search in any log lines :returns: If the string is found it returns True. False is returned otherwise """ with FileReadBackwards(file_path, encoding="utf-8") as frb: for line in frb: match = re.match(HEALTHCHECK_LOGREGEXP, line) if not match: continue columns = line.split(' ') str_date = columns[0] + ' ' + columns[1] dt_line = datetime.strptime(str_date, HEALTHCHECK_DATEFORMAT) if dt_line < time_a or dt_line > time_b: break if line.find(match_string) > 0: return True return False
async def last_upgrade(request): timpestamp = 0 founded = False with FileReadBackwards("/var/log/apt/history.log", encoding="utf-8") as f: while True: line = f.readline() if not line: break current_config = line.split(': ') if current_config[0] == 'End-Date': timestamp = time.mktime(datetime.datetime.strptime(current_config[1].rstrip(), "%Y-%m-%d %H:%M:%S") .timetuple()) elif current_config[0] == 'Commandline': cmd = current_config[1].rstrip() match = re.match(r'^apt(-get)? (.* )?(dist-)?upgrade( -y)?$', cmd) if match is not None: founded = True break if founded: res = {"data": {"utc-timestamp": int(timestamp)}} else: res = {"error": True, "msg": "No info about upgrade"} return web.json_response(res)
def parse(self, skip=False): if skip: logging.debug("skipping parsing of the last session logs ...") else: logging.debug("parsing last session logs ...") lines = [] if os.path.exists(self.path): with FileReadBackwards(self.path, encoding="utf-8") as fp: for line in fp: line = line.strip() if line != "" and line[0] != '[': continue lines.append(line) if LastSession.START_TOKEN in line: break lines.reverse() if len(lines) == 0: lines.append("Initial Session") self.last_session = lines self.last_session_id = hashlib.md5(lines[0].encode()).hexdigest() self.last_saved_session_id = self._get_last_saved_session_id() self._parse_stats() self.parsed = True
def get_last_update_date(self): with FileReadBackwards('/var/log/apt/history.log') as apt_history_file: for block in self._read_apt_blocks(apt_history_file): result = self._try_extract_upgrade_date(block) if result: return result.timestamp() return 0
def get_days(): with FileReadBackwards("res/WUTBEAVRS-1.DEP", encoding="utf-8") as data: for line in data: if line.strip() == 79 * '=': break line = data.readline() return line.split()[2]
def parse_stats(stat_file): epoch = [] stats = {} with FileReadBackwards(stat_file, encoding="utf-8") as sf: for line in sf: if line.strip() == "": continue elif "End Simulation Statistics" in line: stats = {} elif "Begin Simulation Statistics" in line: epoch.append(stats) else: stat = [] sstr = re.sub('\s+', ' ', line).strip() if('-----' in sstr): continue elif(sstr == ''): continue elif(sstr.split(' ')[1] == '|'): # Ruby Stats l = [] for i in sstr.split('|')[1:]: l.append(i.strip().split(' ')) stat.append("ruby_multi") stat.append(l) else: stat.append("single") stat.append(sstr.split(' ')[1]) stats["stats."+sstr.split(' ')[0]] = stat print("Read "+str(len(epoch))+" Epochs") return epoch
def getPeriodLog(path, lastNHour=24): now = datetime.datetime.now() last = now - datetime.timedelta(hours=lastNHour) try: with FileReadBackwards(path, encoding="utf-8") as frb: arr = [] lastLine = "" lastValid = False while True: l = frb.readline() if lastLine: if lastValid: date_parsed = getDateFromLine(lastLine) if date_parsed and date_parsed > last: arr.append(lastLine) lastLine = l else: lastLine = l + "\n" + lastLine else: lastLine = l lastValid = isValidLogLine(l) if not l: break except FileNotFoundError: print(f"Log file not found: {path}") else: return list(reversed(arr))
def showLog(self): parsed_path = urlparse(self.path) filename = os.path.dirname(os.path.abspath(__file__)) + parsed_path.path query = parse_qs(parsed_path.query) # print("\n\n!!!start open==", filename, query, query['start']) data = "" start = 0 end = 100 cnt = 0 try: if(query['start'][0]): start = int(query['start'][0]) if(query['end'][0]): end = int(query['end'][0]) except: print("can not get query parameters " + str(self.path)) print("query=", start, end) try: with FileReadBackwards(filename, encoding="utf-8") as fh: for line in fh: if(cnt >= start and cnt < end): data += line + "\n" cnt += 1 data += "Total lines===" + str(cnt) + "\n" # print(cnt, "data=", filename, "\n=", data) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(str.encode(data)) except: print("".join(traceback.format_stack())) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(bytes("Error open log", "utf8")) return
def watch(self): with FileReadBackwards(self.filename, encoding="utf-8") as frb: for line in frb: if not "DHCPACK" in line: continue client = self.client(line) if not client: break self.save(client) fp = open(self.filename, 'r') fp.seek(0, 2) while True: try: line = fp.readline() except: continue if not line: time.sleep(1) continue if "DHCPACK" in line: client = self.client(line) if not client: continue self.save(client)
def already_scraped(post_id, created): ''' Check to see if submission with scrape_key = "submission.name , submission.permalink" exists in file ''' # To only check the post going back so far # Use date post made onto board and set the date back SEARCH_BACK_DAYS days post_date_check = created - (DAY_IN_SECS * SEARCH_BACK_DAYS) # Read line by line backwards to get most up-to-date posts first with FileReadBackwards(SCRAPED_POSTS_FILE, encoding="utf-8") as spf: for line in spf: line = line.replace('\n', '') # Lines are of form creation,id # Get each argument to check with our current post line_created, line_copy = line.split(",") # If we've found a match then stop ## print line_copy + " == " + post_id + " -> " + str(line_copy == post_id) if line_copy == post_id: ## print "Match found for " + post_id return 1 # If we've gotten past SEARCH_BACK_DAYS days then stop ## print str(line_created) + " < " + str(post_date_check) + " -> " + str(float(line_created) < post_date_check) if float(line_created) < post_date_check: ## print "Looked past " + str(SEARCH_BACK_DAYS) + " days for " + post_id return 0 # If there is no match then the current post hasn't been scraped. ## print "No match found for " + post_id return 0
def build_gem5_stat_dict(file, stat_list=[], read_from_file=True): """ Build a dict of stats from the Gem5 stats.txt """ stats = defaultdict(lambda: [0] * 64) if read_from_file: with FileReadBackwards(file, encoding="utf-8") as sf: for line in sf: if line.strip() == "": continue elif "End Simulation Statistics" in line: stats = defaultdict(lambda: [0] * 64) elif "Begin Simulation Statistics" in line: return stats else: stat = [] sstr = re.sub('\s+', ' ', line).strip().split("#")[0].strip() if ('-----' in sstr): continue elif (sstr == ''): continue elif ('|' in sstr): # Ruby Stats l = [] for i in sstr.split('|')[1:]: l.append(i.strip().split(' ')) stat.append("ruby_multi") stat.append(l) else: stat.append("single") stat.append(sstr.split(' ')[1]) stats[sstr.split(' ')[0]] = stat else: for sl in stat_list[::-1]: for line in sl.split("\n"): if line.strip() == "": continue elif "End Simulation Statistics" in line: stats = defaultdict(lambda: [0] * 64) elif "Begin Simulation Statistics" in line: return stats else: stat = [] sstr = re.sub('\s+', ' ', line).strip().split("#")[0].strip() if ('-----' in sstr): continue elif (sstr == ''): continue elif ('|' in sstr): # Ruby Stats l = [] for i in sstr.split('|')[1:]: l.append(i.strip().split(' ')) stat.append("ruby_multi") stat.append(l) else: stat.append("single") stat.append(sstr.split(' ')[1]) stats[sstr.split(' ')[0]] = stat return stats
def _send_email(self, subject): """ Méthode permettant d'envoyer un mail """ try: # Lecture des logs du backup log_lines = [] with FileReadBackwards(self.loggerfile, encoding='utf-8') as log_file: for line in log_file: log_lines.insert(0, line) if "Starting backup" in line: break # Constitution du message body = "Backup Report : \n" + '\n'.join(log_lines) message = 'Subject: {}\n\n{}'.format(subject, body) # Envoi du mail with smtplib.SMTP_SSL(self.smtphost, self.smtpport) as smtp: smtp.login(self.emailsender, self.passsender) smtp.sendmail(self.emailsender, [self.emaildest], message) except Exception as e: self.logger.error(str(e))
def checkOutput(path, config='single'): ''' The config variable accepts either the value "single" or the value "binary" ''' hasFinished = False keyword1 = 'stop because' keyword2 = 'termination' line_counter = 1 if config == 'single': name = path[-20:] elif config == 'binary': idx = path.index('bin') name = path[idx:] else: raise ValueError('config must be set either to single or binary!') with FileReadBackwards(f'{path}/condor.out') as file: for line in file: # check only the last 20 lines if line_counter <= 20: if line.startswith(f'{keyword1}') or line.startswith( f'{keyword2}'): current_status = f'Model {name} has been terminated!' hasFinished = True break else: current_status = f'Model {name} still running!' line_counter += 1 else: break return hasFinished, current_status
def findXInLog( m="/Users/Benjamin/Documents/crpsp/software_source/test00/log.txt"): """ <--- fonctionne ---> This function will determine using the log, what is the last file that was created. This will allow the encryption protocol to determine which file to use. """ last_item_index = [] # liste vide with FileReadBackwards(m) as LogFile: # Opens the log file (read backwds) for line in ( LogFile): # Searched every line from the bottom for S or D if line[0] == "X": # If we find the last created file i = int(line[4]) * 10 # id does not include "C", "D" operator. i += int(line[5]) # id does not include "C", "D" operator. print("The i value is: ", i) last_item_index.append(i) j = int(line[7]) * 10 # id does not include "C", "D" operator. j += int(line[8]) # id does not include "C", "D" operator. last_item_index.append(j) print("The j value is: ", j) k = int( line[10]) * 10 # id does not include "C", "D" operator. k += int(line[11]) # id does not include "C", "D" operator. last_item_index.append(k) print("The k value is: ", k) return last_item_index # ex: "99-48-74.xml" break
def getAccessLogs(self): accessLogsList = [] store = open(self.meta['storePath'], 'rb') #reading the previous persistently stored variable. cumMetrics = pickle.load(store) store.close() isset = False try: with FileReadBackwards( self.meta['accessLogPath'], encoding='utf-8' ) as f: # reading the file backwards till the lines which are generated in the last one minute. for line in f: accessLogData = {} for value in line.split('?'): pair = value.split('*') accessLogData[pair[0].strip()] = pair[1] # we parse the access, #Converting time to unix timestamp format timestamp = datetime.strptime( accessLogData['time_local'], '%d/%b/%Y:%H:%M:%S %z').timestamp() if not isset: new_time_stamp = timestamp isset = True if timestamp <= cumMetrics[ 'timeStamp']: ## Reading till last read log break accessLogsList.append(accessLogData) self.cum['timeStamp'] = new_time_stamp self.extractMetrics(accessLogsList, new_time_stamp) except: logger.log("Error while parsing the access logs of nginx")
def read_sensor_file( sensor_name: str, minutes_to_read: int = 0) -> List[Tuple[datetime, float]]: log_file_path = SensorLogger.get_sensor_logfile_path(sensor_name) if not log_file_path.exists(): return [] current_time = datetime.now() time_value_pairs: List[Tuple[datetime, float]] = [] try: with FileReadBackwards(str(log_file_path), encoding="utf-8") as fp: # log has format 2021-03-12 18:51:16=55\n... for line in fp: time_value_pair = line.split("=") timestamp = datetime.fromisoformat(time_value_pair[0]) if not minutes_to_read: time_value_pair[1] = float( time_value_pair[1]) # always cast to float return [time_value_pair] if (current_time - timestamp) > timedelta(minutes=minutes_to_read): break time_value_pairs.append( (timestamp, float(time_value_pair[1].strip()))) except: # try to delete when file is corrupted delete_log_file(log_file_path) return time_value_pairs
def get_data(): """ Takes data from cache if possible, else reads file /var/log/apt/history.log in reverse order using the file_read_backwards library. Using the file structure reads one block at a time and, if this is a system update, saves in the cache the unix time of the beginning and the ending as json object of type {"start_time": ... , "end_time": ...} and returns it """ cache_data = check_cache() if cache_data is not None: return cache_data time.sleep(5) with FileReadBackwards(file_name, 'utf-8') as file: update_time = None while True: block = read_block(file) if not block: break update_time = process_block(block) if update_time is not None: break if update_time is None: result = json.dumps({'error': 'can not get information about last system update'}) else: result = json.dumps({'start_time': update_time['start'], 'end_time': update_time['end']}) update_cache(result) return result
def __init__(self): """ position: degrees of antenna's base rotated by motor counter_for_overlap: counter to check for overlap overlap_thress: maximun degrees that antenna is able to rotate = 360 + overlap """ if Antenna.__instance is not None: raise Exception("This class is a singleton!") else: try: self.paths = paths.Paths() file_name = "{dir}/{filename}".format(dir="Logs", filename='adcs.log') with FileReadBackwards(file_name, encoding="utf-8") as log_file: for line in log_file: position = line.decode().split(',')[0] counter = line.decode().split(',')[1] theta_antenna_pointing = line.split(',')[2] theta = line.split(',')[3] break except: #@TODO change the default init position and counter position = 0 counter = 0 theta_antenna_pointing = 0 theta = 0 self.position = position self.counter_for_overlap = counter self.theta = theta self.theta_antenna_pointing = theta_antenna_pointing self.overlap_thress = 380 self.sign_for_counter = +1 self.angle_plot = 0 Antenna.__instance = self
def read_log(): last_dt = None with FileReadBackwards('/var/log/apt/history.log', encoding="utf-8") as frb: for line in frb: if line.startswith("End-Date"): last_dt = datetime.strptime(line[10:], "%Y-%m-%d %H:%M:%S") if line.endswith("apt-get upgrade") or line.endswith( "apt-get dist-upgrade"): if not last_dt: # обработка возможности запроса во время скачивания обновлений response = { "data": { "timestamp": int(datetime.timestamp(datetime.now())) } } else: response = { "data": { "timestamp": int(last_dt.timestamp()) } } return response return None
def tail(filename, pattern, maxlines=60): global restartreason waitcount = 0 while True: with FileReadBackwards(filename, encoding="utf-8") as frb: lines = 0 for l in frb: lines += 1 #I dont want to scan old logs. So break this loop if before we go too deep. if lines > maxlines: continue if 'Totals' in l: hashrate = re.search(pattern, l) if hashrate: return hashrate.group(1) print(bcolors.WARNING + "No 60s hash found yet. Waiting for that to appear.. ({})". format(waitcount) + bcolors.ENDC) waitcount += 1 if waitcount > 31: print(bcolors.FAIL + "Waited too long for an average hash! Kill it all." + bcolors.ENDC) restartreason += "{} - Timeout waiting for 60s Hash".format(now) restarttime() waitcount = 0 time.sleep(10)
def lines(self): """ :return: Generator of metrics.log lines """ for filename in self.filenames(): with FileReadBackwards(filename, encoding="utf-8") as open_file: for line in open_file: yield line
def searchOutput(path): with FileReadBackwards(path+'/condor.out') as file: for line in file: if line.startswith('save ' + path + '/photos/') or line.startswith('save photos/'): words = re.split('/|, | ', line) break return words[-4], words[-1]
def get_log(file): with FileReadBackwards(file, encoding="utf-8") as frb: lines = [] while len(lines) < 1000: line = frb.readline() if not line: break lines.append(line) return "<br/>".join(lines)