class Parser: def __init__(self): self.config = Configuration() self.launch = Init() date_from = self.launch.dates[0] date_to = self.launch.dates[1] self.db = Db() for host in self.config.hosts: self.ssh = Ssh(host, 22, self.config.login, self.config.password) logs = self.find_logs(date_from, date_to) for log in logs: strings = self.find_strings(log) batch_counter = 0 batch = [] for string in strings: parsed = self.parse_row(string) if re.match(r'(\d{4})-(\d{2})-(\d{2})', parsed[0]): batch.append(parsed) batch_counter += 1 if batch_counter >= 100: self.db.insert(batch) batch = [] batch_counter = 0 def find_logs(self, date_from, date_to): try: cmd = "find {0}/ -maxdepth 1 -name 'server.log.*' -newermt '{1}' " \ "! -newermt '{2}' | sort".format(self.config.log_path, date_from, date_to) logs = self.ssh.remote_cmd(cmd) return logs except ValueError as e: print(e) def find_strings(self, log): try: cmd = "zgrep -a 'INFO' {0} | zgrep -a '\[statistics\]' ".format( log) cmd += "| awk '{ print $1\" \"$2\";\"$7}'" strings = self.ssh.remote_cmd(cmd) return strings except ValueError as e: print(e) def parse_row(self, row): date = row.split(';')[0].split(',')[0] method = row.split(';')[1] result = [date, method] return result
def __init__(self): self.config = self.read_config() self.args = self.parse_args() cmd = self.make_cmd() print(cmd) for host in self.config.get('ssh', 'hosts').split(','): ssh = Ssh(host, int(self.config.get('ssh', 'port')), self.config.get('ssh', 'login'), self.config.get('ssh', 'password')) try: grep_out = ssh.remote_cmd(cmd) for row in grep_out: print(row) except ValueError as e: print(e)