def tailf(logfile): print("Starting to monitor {0} with pattern for rclone {1}".format( logfile, cfg['backend'])) # Validate which backend we're using if cfg['backend'] == 'cache': # Use cache backend for line in tail("-Fn0", logfile, _iter=True): if re.match( r".*(mkv:|mp4:|mpeg4:|avi:) received cache expiry notification", line): f = re.sub(r"^(.*rclone\[[0-9]+\]: )([^:]*)(:.*)$", r'\2', line) print("Detected new file: {0}".format(f)) scan(os.path.dirname(f)) elif cfg['backend'] == 'vfs': # Use vfs backend timePrev = '' for line in tail("-Fn0", logfile, _iter=True): if re.match(r".*: forgetting directory cache", line): f = re.sub(r"^.*\s:\s(.*):\sforgetting directory cache", r'\1', line) timeCurr = re.sub( r"^.*\s([0-9]+:[0-9]+:[0-9]+)\s.*\s:\s.*:\sforgetting directory cache", r'\1', line) if timeCurr != timePrev: print("Detected directory cache expiration: {0}".format(f)) scan(os.path.dirname(f)) timePrev = timeCurr
def _do(self): with tempfile.TemporaryFile('w+t') as tf: # Write tail of logfile into temporary file sh.tail(self.path, lines=self.nlines, _in=self.path, _out=tf) # Rewind temporary file tf.seek(0) # Overwrite old file sh.cat(_in=tf, _out=self.path) return True
def _execute(self, command_line): LOG.debug( 'Begin executing commmand: %s', ' '.join("'" + arg + "'" for arg in command_line)) self.status = "EXECUTING" self.exit_code = None try: sh.bash('-x', *command_line, _out=self.write_stdout, _err=self.write_stderr) except sh.ErrorReturnCode as error: exit_code = error.exit_code status = "FAILED" severity = logging.ERROR except Exception: exit_code = 1 status = "ERROR" severity = logging.ERROR LOG.exception('Internal error.') except BaseException: exit_code = 1 severity = logging.WARNING status = "INTERRUPTED" else: exit_code = 0 status = 'SUCCESS' severity = logging.DEBUG self.exit_code = exit_code self.status = status if exit_code != 0 and self.log_level < logging.ERROR: stream = sys.stderr stream.write('=' * 79 + '\n') sh.tail('-n', '100', self.log_path + '.ansi', _out=stream) stream.write('=' * 79 + '\n') LOG.log( severity, 'Finished executing command:\n' ' Command line: %s\n' ' Status: %s\n' ' Exit code: %s\n' ' Log file: %s\n', command_line, status, exit_code, self.log_path) return exit_code
def xxd_patch(old, new, patch_file): tmp_old = '/var/tmp/old_hex' sh.xxd('-p', old, _out=tmp_old) sh.tail('-n', '+2', patch_file, _out=patch_file+'.nohead') sh.patch(tmp_old, patch_file+'.nohead') if new: sh.xxd('-r', '-p', tmp_old, _out=new) else: sh.xxd('-r', '-p', tmp_old, _out=old) sh.rm('-f', tmp_old) sh.rm('-f', patch_file+'.nohead')
def containers(): """ Return list of container (names) instanciated """ from io import StringIO buf = StringIO() try: tail(awk(docker('ps', '-a'), '{print $NF}'), '-n+2', _out=buf) except Exception as err: log.error(err) containers = buf.getvalue().split() return containers
def max_file_hash(n=10, short=False): pack_path = glob('.git/objects/pack/*.idx') if not pack_path: git.gc() pack_path = glob('.git/objects/pack/*.idx') if short: return awk( tail(sort(git('verify-pack', '-v', pack_path), '-k', '3'), '-n', '-{0:d}'.format(n)), '{print $1}') else: return tail(sort(git('verify-pack', '-v', pack_path), '-k', '3', '-n'), '-{0:d}'.format(n))
def _get_data(self): try: data = {'produced': 0} search = [] now = DT.datetime.now() last = now + relativedelta(seconds=-126) for line in tail("-n 500", self.log_path, _iter=True): if "Received block" in line or "Produced block" in line: match = re.match( r'.*info (.*) thread.* signed by (.*) \[.*', line) if match != None: info = { 'date': match.group(1), 'producer': match.group(2) } date = parse(info["date"]) offset = -self.configuration.get( 'timezone_offset') * 3600 if self.configuration.get( 'timezone_offset') else 0 date = date + relativedelta(seconds=offset) #pdb.set_trace() # filtering entries for the last 126 seconds if date > last and date < now or date == now: search.append(info) if info['producer'] == self.configuration.get( 'producer'): data['produced'] += 1 return data except (ValueError, AttributeError): return None
def _get_keynames(): global key_cache key_cache = [] result = fgrep(tail(nova('keypair-list'), '-n', '+4'), '-v', '+') for line in result: (front, name, signature, back) = line.split('|') key_cache.append(name.strip())
def tail_and_follow(path, ansible_verbosity): """Follow and provide output :param path: tuple containing thepath to file to follow :param ansible_verbosity: the verbosity level """ if len(path) > 0: # pylint: disable=len-as-condition truncate = 1 if ansible_verbosity: truncate = ansible_verbosity print_line = truncate plabook_started = False truncated = False # pylint: disable=no-member for line in sh.tail('-f', '-n', '+0', path, _iter=True): line = line.strip('\n') if line.startswith('TASK') or line.startswith('PLAY'): print(line) print_line = truncate plabook_started = True truncated = False elif print_line > 0: line_len = len(line) char_truncate = truncate * 100 if line_len > char_truncate: print(line[0:char_truncate] + '...') else: print(line) print_line = print_line - 1 elif print_line == 0 and not truncated and plabook_started: print(_('-- output truncated --')) truncated = True
def agent_logs(amount): static = get_static() if not os.path.isfile(static.main_log): return jsonify(messages=[]) lines = tail('-n {}'.format(amount), static.main_log).splitlines() return jsonify(messages=lines)
def post_update(): if not self._service._enabled: return lines = [ x for x in sh.tail("/var/log/syslog", "-n", "10", _iter=True) ] dns_query = None for line in lines[::-1]: if "CONNECT" in line and "dreampi" in line: # Don't seek back past connection break if "query[A]" in line: # We did a DNS lookup, what was it? remainder = line[line.find("query[A]") + len("query[A]"):].strip() domain = remainder.split(" ", 1)[0].strip() dns_query = sha256(domain).hexdigest() break user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT), Dreamcast Now' header = { 'User-Agent' : user_agent } mac_address = self._service._mac_address data = {} if dns_query: data["dns_query"] = dns_query data = urllib.urlencode(data) req = urllib2.Request(API_ROOT + UPDATE_END_POINT.format(mac_address=mac_address), data, header) urllib2.urlopen(req) # Send POST update
def tail_F(some_file): while True: try: for line in sh.tail("-f", some_file, _iter=True): yield line except sh.ErrorReturnCode_1: yield None
def get_log(self, request, format=None): def tail(file_path, num_lines=20000): if os.path.exists(file_path) is False: return "File does not exist." else: try: if os.path.getsize(file_path) == 0: return "File is empty." else: from sh import tail return tail("-n", int(num_lines), file_path) except OSError as err: return "Failed getting file size: {}".format(err) qp = request.query_params log_filename = str( qp["log_filename"]) if "log_filename" in qp else None download = True if "download" in qp else False file_path = os.path.join("/app/logs", os.path.basename(log_filename)) response = HttpResponse(tail(file_path), content_type="text/plain") if download is True: response[ "Content-Disposition"] = "attachment; filename=\"{}\"".format( log_filename) response["Cache-Control"] = "no-cache" return response
def post_update(): if not self._service._enabled: return lines = [ x for x in sh.tail("/var/log/syslog", "-n", "10", _iter=True) ] dns_query = None for line in lines[::-1]: if "CONNECT" in line and "dreampi" in line: # Don't seek back past connection break if "query[A]" in line: # We did a DNS lookup, what was it? remainder = line[line.find("query[A]") + len("query[A]"):].strip() domain = remainder.split(" ", 1)[0].strip() dns_query = sha256(domain).hexdigest() break user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT), Dreamcast Now' header = {'User-Agent': user_agent} mac_address = self._service._mac_address data = {} if dns_query: data["dns_query"] = dns_query data = urllib.urlencode(data) req = urllib2.Request( API_ROOT + UPDATE_END_POINT.format(mac_address=mac_address), data, header) urllib2.urlopen(req) # Send POST update
def get_appyters(appyter_path): for path in map(os.path.dirname, glob.glob(os.path.join(appyter_path, '*', 'appyter.json'))): appyter = json.load(open(os.path.join(path, 'appyter.json'), 'r')) yield dict( appyter, path=path, long_description=open(os.path.join(path, 'README.md'), 'r').read(), # find the oldest commit containing the appyter's appyter.json (follow for detecting renames) creation_timestamp=str( sh.tail( sh.git.log( '--follow', r'--pretty=format:%aI', '--', os.path.join(path, 'appyter.json'), _tty_out=False, ), '-n1')).strip(), # find the most recent commit containing the appyter's directory update_timestamp=str( sh.head( sh.git.log( r'--pretty=format:%aI', '--', path, _tty_out=False, ), '-n1')).strip(), )
def read_log(id: int): log = find_log_file_by_id(id) if not log: raise NotFound() filepath = log.filepath lines: str = sh.tail("-n200", filepath) lines = lines.split("\n") ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]') lines = [ansi_escape.sub('', line) for line in lines] # clrc.info("Splitted lines: ") # clrc.info(lines) data = [ "2019-04-28 13:53:36.694 | INFO | clrc.printers.loguru_printer:info:15 - SMS COOLDOWN FOR 79964716158 LEFT 0", "2019-04-28 12:07:47.668 | SUCCESS | clrc.printers.loguru_printer:success:9 - Success sending typed email!", "2019-04-28 13:53:23.992 | ERROR | clrc.printers.loguru_printer:error:21 - Traceback (most recent call last):" ] return jsonify(lines)
def get_alert_sms(now_stm): alert_sms = [] sms_data = [] try: sms_data = list(sh.tail("-n5", "/var/log/sms.log")) except Exception, e: pass
def test_nonblocking_iter(self): import tempfile from sh import tail from errno import EWOULDBLOCK tmp = tempfile.NamedTemporaryFile() for line in tail("-f", tmp.name, _iter_noblock=True): break self.assertEqual(line, EWOULDBLOCK)
def _doRollover(self): for filename in self.filenames: tmp_name = filename + '.tmp' _ = sh.tail('-n', int(self.maxlen), filename, _out=open(tmp_name, 'w')) _ = sh.cp(tmp_name, filename)
def append_file(path, end): last_row = tail('-n', '1', path) date = [ json.loads(d)['timestamp'] for d in last_row ] print "Appending" with open(path, 'a') as fh: write_data(fh, date[0], end)
def run(self): log_info("Run thread LogParser") while True: try: for line in tail("-n", 1, "-f", eosio_log_file, _iter=True): log_parse(line) except: log_err("eosio log file:" + eosio_log_file + " parse failed!") time.sleep(10)
def run(self): if is_docker == "true": client = docker.from_env() nodeos = client.containers.get("nodeos") for line in nodeos.logs(stream=True, tail=1): log_parse(line) else: for line in tail("-n", 1, "-f", log_file, _iter=True): log_parse(line)
def follow(self, process_line_func): def ignore(*args): pass def _process(line): return process_line_func({"name": self.cname, "msg": line, "pid": None, "t": datetime.now()}) p = sh.tail("--follow=name", "-q", "--lines=3", self.filename, _out=_process, _err=ignore) p.wait()
def monitor_system(now_stm): system_pool = [] system_log = [] system_name = {'active': 30887, 'wservice': 30780, 'order': 31499, 'passport': 30592, 'pay': 31545, 'product': 30342, 'shopping': 29138, 'slist': 33027, 'track': 30779, 'tuan': 30778, 'wap_v5': 33008} try: system_log = list(sh.tail("-n5", "/var/log/shuffling.log")) except Exception, e: pass
def _get_swarm_service_state(service_name): docker = sh.Command("docker") try: state = sh.awk( sh.tail(sh.head(docker(["service", "ps", service_name]), "-2"), "-1"), "{print $6}").strip(' \t\n\r\'') except: state = "Not present" return state
def check_deployments_status(deployments, exit_on_error=False): """Simple asynchronous checks if some deployment is finished""" # In single cluster deployment, logs are forwarded to stdout. if len(deployments) == 1: logging.info( "Only one cluster is being deployed. Redirecting logs to stdout.") sh.tail( "-n", "100000", "-f", # pylint: disable=no-member deployments[0].log_file.name, _out=print_log_line, _bg=True, _new_session=False) else: logging.info( "More than one deployment is running, " "please check the log files for detailed deployment logs.") while not has_deployments_ended(deployments): for _, deployment in enumerate(deployments): if deployment.process.poll() is None or deployment.has_ended: pass elif deployment.process.poll() == 0: logging.info('%s %s: succeed.', deployment.cluster_name, deployment.playbook_file_name) deployment.log_file.close() deployment.has_ended = True deployment.ended_successfully = True elif deployment.process.poll() != 0: logging.error('%s %s: failed. Please check the logs: %s', deployment.cluster_name, deployment.playbook_file_name, os.path.realpath(deployment.log_file.name)) deployment.log_file.close() deployment.has_ended = True deployment.ended_successfully = False if exit_on_error is True: logging.info( "--any-errors-fatal flag raised, terminating other deployments..." ) kill_deployments(deployments) time.sleep(1)
def load_kvs(): with open(FLORFILE, "r", encoding="utf-8") as f: d = json.load(f) p = Path.home() p = p / ".flor" p = p / d["NAME"] # type: ignore p = p / "replay_jsons" seq = [] for q in p.iterdir(): # q will contain the timestamp: 2022-02-07T20:42:25.json tstamp = q.stem # 2022-02-07T20:42:25 with open(str(q), "r", encoding="utf-8") as f: d = json.load(f) _kvs = d["KVS"] for k in _kvs: z = k.split(".") e = z.pop(0) r = z.pop(0) n = ".".join(z) for s, x in enumerate(_kvs[k]): # pvresnx seq.append((d["NAME"], d["MEMO"], tstamp, r, e, s, n, x)) df1 = pd.DataFrame( seq, columns=[ "projid", "vid", "tstamp", "alpha", "epoch", "step", "name", "value" ], # dtype=(str, str, np.datetime64, str, int, int, str, object), ).astype({ "projid": str, "vid": str, "tstamp": np.datetime64, "alpha": str, "epoch": int, "step": int, "name": str, "value": object, }) # I want to build a mapper from FLORFILE to GIT HASH vid_mapper = dict() for path in df1["vid"].drop_duplicates().to_list(): eof = json.loads(tail("-1", path, _iter=True).next()) vid_mapper[path] = eof["COMMIT_SHA"] df1["vid"] = df1["vid"].apply(lambda x: vid_mapper[x]) return df1.sort_values(by=["tstamp", "epoch", "step"])
def log(): uwsgi_logfile = config.common['UWSGI_LOGFILE'] def process_output(line): print(line.strip(), file=sys.stdout) p = sh.tail(uwsgi_logfile, follow=True, _out=process_output) try: p.wait() except KeyboardInterrupt: print(colored('\nleave log', 'red')) p.terminate()
def process_tmp_ta(self, filename="", is_standalone=True): if filename == "": log.info("INFO: No files to process now.") else: if not os.path.isfile(filename): log.error("ERROR: filename " + filename + " is not a file.") return False else: log.info("Processing: " + filename) self.iostatus[filename]['running'] = True self.iostatus[filename]['tail'] = sh.tail("-F", filename, _iter=True, _bg_exc=False) while self.iostatus[filename]['running'] and self.thread_ctrl[ 'continue']: try: line = self.iostatus[filename]['tail'].next() if is_standalone: j = json.loads(line) summary = get_summary_from_json(j) if summary is not None: self.influxdb.influxdb_queue.put({ 'summary': summary, 'json': line }) else: log.warning( "can't get summary from {0}".format(line)) else: self.rabbitmq.rabbitmq_queue.put( line) # < --- sending raw lines for now except sh.ErrorReturnCode_1: # as e: log.info( "process_tmp_ta: tail terminated {0}, (permission denied ?) " .format(filename)) break except sh.SignalException_SIGKILL as e: log.info("process_tmp_ta: tail terminated {0} {1}".format( filename, e)) break except Exception as e: exc_type, _, exc_tb = sys.exc_info() fname = os.path.split( exc_tb.tb_frame.f_code.co_filename)[1] log.error("process_tmp_ta: {0} {1} {2} {4}".format( exc_type, fname, exc_tb.tb_lineno, e)) log.info('process_tmp_ta: exiting ' + filename) #with open(iostatus, 'wb') as handle: # pickle.dump(iostatus, handle, protocol=pickle.HIGHEST_PROTOCOL) return True
def transport_assembly_run( seed_name, tfilename, workdir='./build' ): # Push package: to-be processed packages are placed in 'put' directory. # Pulled package: finished work is in the 'get' directory. retry = 1 # Credentials wide out in the open: ssh = createSSHClient(CL_SERVER_NAME, 22, CL_SERVER_USER, CL_SERVER_PW) while (retry < 10): try: scp = SCPClient(ssh.get_transport()) scp.put(tfilename, 'run') # If it got this far, it succeeded break; except Exception as err: print '==============' print err[0] print ' Retry attemp #'+retry print '==============' sleep(5) # # Server Processing Phase # waittime = 0 while (waittime < 600): try: scp = SCPClient(ssh.get_transport()) # Check the log file scp.get('get/'+seed_name+'_run.log', local_path=workdir) line = tail("-1", os.path.join(workdir,seed_name+'_run.log') ).strip() print line if 'Assembly Complete' in line: break if 'ERROR' in line: exit(1) except Exception as err: print '==============' print err[0] print '==============' if 'No such file or directory' in err[0]: print "---- No status yet ----" sleep(2) waittime = waittime + 1 scp.close() # Compile complete. Download results. try: scp.get('get/'+seed_name+'_assembly.dat', local_path=workdir) os.rename( os.path.join(workdir, seed_name+'_assembly.dat'), os.path.join(workdir,'assembly.dat')) except Exception as err: print '==============' print err[0] print '==============' exit(1) return True
def __init__(self): global LOGFILE self.NETWORKSERVER = NetworkServer() self.LoadCellDataRecorder = DataRecorder(LoadCellDataRecorderKeys, "LoadCellVoltageData", network=self.NETWORKSERVER) self.LOGGER = logging.getLogger("TESTBED") self.Phypervisor = PressureHyperVisor(700, 700) init_gpio() self.LOGTAIL = sh.tail("-f", LOGFILE, _iter_noblock=True) self.report()
def __init__(self, path): if os.path.exists(path): self.path = path else: raise Exception("Path '{0}' does not exist.".format(path)) self.dependencies = [] with pushd(self.path): self.package_name = tail(python('setup.py', '--name'), '-1').rstrip()
def get_chinamap_ping_data(now_stm): ping_dic = {} ping_data = list(sh.tail("-n5", "/var/log/map_ping.log")) for ptm in ping_data: if int(ptm.split()[0]) > now_stm: try: ping_dic[ptm.split()[2].split('_')[3]] = {'type': ptm.split()[1], 'loss': ptm.split()[4], 'rtt': ptm.split()[6]} except: pass return json.dumps(ping_dic)
def agent_get_outputs_reporter_parallel(lock, handler, defaultfilepath): import sh if not os.path.exists(defaultfilepath) or not os.path.isfile( defaultfilepath): print('Ops! Not a file in reporter parallel (%s)' % (defaultfilepath)) return print('Listening defaults (%s)' % (defaultfilepath)) for line in sh.tail('-f', defaultfilepath, _iter=True): div = line.find(" ") service = line[:div] output = line[div + 1:] _agent_send_output(service, output)
def tail(file_path, num_lines=20000): if os.path.exists(file_path) is False: return "File does not exist." else: try: if os.path.getsize(file_path) == 0: return "File is empty." else: from sh import tail return tail("-n", int(num_lines), file_path) except OSError as err: return "Failed getting file size: {}".format(err)
def agent_get_file_outputs_parallel(lock, handler, filepath, defaultfilepath): import sh if not os.path.exists(filepath) or not os.path.exists( defaultfilepath) or not os.path.isfile( filepath) or not os.path.isfile(defaultfilepath): print('Wow! Invalid arguments in get outputs parallel (%s, %s)' % (filepath, defaultfilepath)) return print('Listening (%s, %s)' % (filepath, defaultfilepath)) for line in sh.tail('-f', filepath, _iter=True): with lock: with open(defaultfilepath, 'a') as defaultf: defaultf.write('%s\n' % (line))
def _tag_commit(self): sh.git.fetch("origin", "--tags") deploy_commit = sh.git("rev-parse", self.env.code_branch).strip() self._check_deploy_commit(deploy_commit, self.env.code_branch) pattern = "*{}*".format(self.env.environment) self.last_tag = sh.tail(sh.git.tag("-l", pattern), "-1").strip() tag_name = "{}-{}-deploy".format(self.timestamp, self.env.environment) # turn whatever `code_branch` is into a commit msg = "{} deploy at {}".format(self.env.environment, self.timestamp) sh.git.tag(tag_name, "-m", msg, deploy_commit) sh.git.push("origin", tag_name) self._deploy_ref = tag_name
def output(job_id): lines = int(request.params.get('lines', 40)) jobs_path = app.config.get('jobs.path') job_path = os.path.abspath(os.path.join(jobs_path, job_id)) job_out = os.path.join(job_path, 'output') job_info = os.path.join(job_path, 'job.json') if not os.path.exists(job_out) or not os.path.exists(job_info): raise StopIteration with open(job_info) as f: info = json.load(f) args = ['--lines=%d' % lines, job_out] if "exit_code" in info else ['--lines=%d' % lines, '--pid=%d' % info['job_pid'], '-f', job_out] for line in sh.tail(*args, _iter=True): yield line
def main(test): """Console script for docker-svtplay-dl""" try: command: sh.RunningCommand = tail('-f', '/var/log/system.log', _bg=True) time.sleep(2) command.process.terminate() print(command.stdout) except BaseException as err: pass
def log_parser(self): logger.info("Start LogParser") if not os.path.exists(eosio_log_file): logger.error("%s not exists", eosio_log_file) return while True: try: pre_line = "" for line in tail("-n", 1, "-f", eosio_log_file, _iter=True): line = line.rstrip('\n') pre_line = self.handle_line(pre_line, line) except Exception as e: logger.error("parse failed %s", e)
def main(argv, settings): logging.basicConfig(filename = settings.LOG_FILE, level = logging.DEBUG, format='[%(asctime)s][%(levelname)s] %(message)s') first = True for line in sh.tail('-f', settings.WATCH_LOG, _iter=True): line = line.strip() if not first: match = re.search(settings.MATCH_LINES, line) logging.debug('Looking for "%s" in "%s" -> %r.', settings.MATCH_LINES, line, bool(match)) if match: message = '[{stamp}][{host}] {match}'.format(stamp = getDateTime(), host = sh.hostname('-s').strip(), match = match.group(0)) sendDm(settings.TWITTER_AUTH, settings.DM_RECIPIENTS, message) first = False
def get_container_info(): container_info = {} try: container_info['container_name'] = tail(sed(grep(cat('/proc/self/cgroup'), 'docker'), 's/^.*\///'), '-n1') except: container_info['container_name'] = 'unknown' try: container_info['app_version'] = open('version', 'r').read() except: container_info['app_version'] = 'unknown' return container_info
def follow(self, process_line_func): def ignore(*args): pass def _process(line): return process_line_func({ 'name': self.cname, 'msg': line, 'pid': None, 't': datetime.now() }) p = sh.tail('--follow=name', '-q', '--lines=0', self.filename, _out=_process, _err=ignore) p.wait()
def populateImageParts(img): import sh, re from sh import tail logger = logging.getLogger('virtimg') df = sh.Command('virt-df') out = str(tail(df('-a', img.path), '-n+2')) logger.debug('%s: df-h: %s', img.name, out) regex = re.compile('^[\w.\-]+:/dev/sda(?P<partid>\d+)\s+(?P<size>\d+)\s+\d+\s+(?P<free>\d+)', re.M) res = re.findall(regex, out) img.parts = dict() for part in res: img.parts[part[0]] = dict() img.parts[part[0]]["size"] = part[1] img.parts[part[0]]["free"] = part[2] img.lastpart = max(img.parts.keys())
def logs(bot, update): debug_arr = [] imap_arr = [] app_arr = [] for line in tail( "-n 1", "/home/feyruz/sandbox/RideAway-AutoResponder/logs/app.verbose.log", _iter=False): debug_arr.append(line) for line in tail( "-n 3", "/home/feyruz/sandbox/RideAway-AutoResponder/logs/app.log", _iter=False): app_arr.append(line) for line in tail( "-n 3", "/home/feyruz/sandbox/RideAway-AutoResponder/logs/imap.live.log", _iter=False): line = line.rstrip() imap_arr.append(line[0:30] + '\n') update.message.reply_text('******* APP ********\n' + ''.join(app_arr) + '\n******* DEBUG ********\n' + ''.join(debug_arr) + '\n******* IMAP ********\n' + ''.join(imap_arr))
def _tag_commit(): sh.git.fetch("origin", env.code_branch) deploy_time = datetime.datetime.utcnow() tag_name = "{:%Y-%m-%d_%H.%M}-{}-deploy".format(deploy_time, env.environment) pattern = "*{}*".format(env.environment) last_tag = sh.tail(sh.git.tag("-l", pattern), "-1").strip() branch = "origin/{}".format(env.code_branch) msg = getattr(env, "message", "") msg += "\n{} deploy at {}".format(env.environment, deploy_time.isoformat()) sh.git.tag(tag_name, "-m", msg, branch) sh.git.push("origin", tag_name) diff_url = "https://github.com/dimagi/commcare-hq/compare/{}...{}".format( last_tag, tag_name ) print "Here's a link to the changes you just deployed:\n{}".format(diff_url) return diff_url
def run(self): # checkThread = False ## Ping the webApp threadLife = 1800 try: timeSlot = int(time.time()) n = datetime.datetime.now() dt = datetime.datetime(n.year, n.month, n.day, n.hour, n.minute, 0, tzinfo=tzlocal()) ts = dt.isoformat(' ') for line in tail("-f", self.access_log, _iter=True): if int(time.time()) - timeSlot >= 60: self.investigate(ts) timeSlot = int(time.time()) n = datetime.datetime.now() dt = datetime.datetime(n.year, n.month, n.day, n.hour, n.minute, 0, tzinfo=tzlocal()) ts = dt.isoformat(' ') if self.utility.isJson(line): # checkThread = False ## Ping the webApp threadLife = 1800 res = json.loads(line) res["query"] = None if res["query"] == '' else res["query"] res["status"] = int(res["status"]) res["size"] = 0 if res["size"] == "-" else int(res["size"]) res["referer"] = None if res["referer"] == "-" else res["referer"] res["responseTime"] = int(res["responseTime"]) self.addToQueue(res) else: if threadLife == 0: if not self.is_ssl: syslog.syslog('%s gatekeeper thread restarting...' % self.name) else: syslog.syslog('%s ssl gatekeeper thread restarting...' % self.name) break else: threadLife = threadLife - 30 time.sleep(30) continue except IOError: syslog.syslog('Error(%s): can\'t find file (%s) or read data.' % (self.name, self.access_log)) if threadLife == 0: self.run()
def main(filename): """ Open `filename` and start processing it line by line. If `filename` is none, process lines from `stdin`. """ if filename: if not os.path.exists(filename): logger.error("'%s' doesn't exists!" % filename) sys.stderr.write("'%s' doesn't exists!\n" % filename) sys.exit(1) logger.info("Processing '%s'" % filename) for ir in process_log(sh.tail("-f", filename, _iter=True)): print ir else: logger.info("Processing stdin.") for ir in process_log(_read_stdin()): print ir
def run(self): import sh file_path = self.inputs['PATH'].receive() if file_path is EndOfStream: self.terminate() return self.log.debug(u'Tailing file: {}'.format(file_path)) for line in sh.tail('-f', file_path, _iter=True): stripped_line = line.rstrip() self.log.debug(u'Tailed line: {}'.format(stripped_line)) self.outputs['OUT'].send(stripped_line) if self.is_terminated(): break
def body(self): """ This method handles AMQP connection details and reacts to FTP events by sending messages to output queue. """ self.connection = pika.BlockingConnection(self.connection_param) self.channel = self.connection.channel() print "Monitoring file '%s'." % self.ftp_extended_log file_iter = process_log( sh.tail("-f", self.ftp_extended_log, _iter=True) ) for import_request in file_iter: self.sendMessage( exchange=self.output_exchange, routing_key=self.output_key, message=serializers.serialize(import_request), UUID=str(uuid.uuid1()) )
def start_reading(self, params): for line in tail("-n", 0, "-f", params['file'], _iter=True): if not self.running: return line = line.replace('\n', '') # notify subscribers self.notify_subscribers(line) if self.log_level > 0: # add line to buffer self.buffer.append(line) if len(self.buffer) >= self.buffer_size: logger_func = None for line in self.buffer: if re.match(self.filter_regex, line): logger_func = self.get_logger_for_level_in_log_line(line) break if logger_func: for buffered_line in self.buffer: logger_func(buffered_line) self.buffer = []
def extract(self, conf): merge = conf.get("should_line_merge", False) before = conf.get("break_only_before", None) if merge: before = re.compile(before) value = HashBuilder({ "type": conf.get("type", "log"), "source_path": conf["path"], "tags": conf.get("tags", []) }) if conf.get("host"): host = "%s@%s" % (conf["user"], conf["host"]) if conf.get("user") else conf["host"] remote_server = ssh.bake(host) mac = str(ssh(host, "ifconfig | grep 'eth0'")) value.shared["source_host"] = host value.shared["source_mac"] = re.findall("..:..:..:..:..:..", mac)[0] log = remote_server.tail("-f", conf["path"], _iter=True) else: touch(conf["path"]) if conf.get("from", None) == "now": filesize = "-c0" else: filesize = "-c%s" % (int(stat("-nf" ,'%z', conf["path"])) * 2 + 10) log = tail("-F", filesize, conf["path"], _iter=True) for n, line in enumerate(log): if merge: if before.match(line): if value.has_data(): yield value.dict() # time.sleep(0.1) value.build() value.log(line) else: value.build(line) yield value.dict()
def run(self): try: #Starting background thread which process and cleans draft statistics periodically bg_thread = Thread(target=self._monitor_bg_process) bg_thread.daemon = True bg_thread.start() #Reading last line from tailing the log and appending it to draft statistics. for line in tail('-f', '-n 0', self._access_log_path, _iter=True): #logger.debug('Processing log line {0} ... '.format(line)) sample = self._parser.parse(line) if (sample is not None): #Adding even if request was for non-existing resource. Easier to write test. self._draft_stats_lock.acquire() self._draft_stats.add_sample(sample) self._draft_stats_lock.release() bg_thread.join() except (KeyboardInterrupt, SystemExit): logger.error('Interrupted')
def resolve_from_dir(self, path): self._pip_install_opts.append('-e') if not os.path.exists(path): raise Exception("Path not found '{0}'".format(path)) rm('-r', '-f', "/tmp/pip_build_{0}".format(getuser())) with pushd(path): print("") print("'git status' in '{0}':".format(path)) print("------------") print(git('status')) print("------------") self.package_name = tail(python("setup.py", "--name"), "-1").rstrip() print("") print("Gathering package requirements ...") for line in pip('install', self._pip_install_opts, '.'): string = line.rstrip() match = re.search( 'Downloading/unpacking (.*?) \(from (.*?)\)', string ) if match: self._add_pip_package(match.group(1), from_package=match.group(2)) continue match = re.search( 'Requirement already satisfied.*?: (.*?) in .*?\(from (.*?)\)', string ) if match: self._add_pip_package(match.group(1), from_package=match.group(2)) continue print("Done. {0} records found.".format(len(self.entries)))
def process(): killer = GracefulKiller() dial_tone_enabled = not "--disable-dial-tone" in sys.argv # Make sure pppd isn't running with open(os.devnull, 'wb') as devnull: subprocess.call(["sudo", "killall", "pppd"], stderr=devnull) BAUD_SPEED = 57600 device_and_speed, internet_connected = None, False ## Startup checks, make sure that we don't do anything until ## we have a modem and internet connection while True: logger.info("Detecting connection and modem...") internet_connected = check_internet_connection() device_and_speed = detect_device_and_speed() if internet_connected and device_and_speed: logger.info("Internet connected and device found!") break elif not internet_connected: logger.warn("Unable to detect an internet connection. Waiting...") elif not device_and_speed: logger.warn("Unable to find a modem device. Waiting...") time.sleep(5) modem = Modem(device_and_speed[0], device_and_speed[1], BAUD_SPEED, dial_tone_enabled) dreamcast_ip = autoconfigure_ppp(modem.device_name, modem.device_speed) mode = "LISTENING" modem.connect() if dial_tone_enabled: modem.start_dial_tone() time_digit_heard = None dcnow = DreamcastNowService() while True: if killer.kill_now: break now = datetime.now() if mode == "LISTENING": modem.update() char = modem._serial.read(1).strip() if not char: continue if ord(char) == 16: #DLE character try: char = modem._serial.read(1) digit = int(char) logger.info("Heard: %s", digit) mode = "ANSWERING" modem.stop_dial_tone() time_digit_heard = now except (TypeError, ValueError): pass elif mode == "ANSWERING": if (now - time_digit_heard).total_seconds() > 8.0: time_digit_heard = None modem.answer() modem.disconnect() mode = "CONNECTED" elif mode == "CONNECTED": dcnow.go_online(dreamcast_ip) # We start watching /var/log/messages for the hang up message for line in sh.tail("-f", "/var/log/messages", "-n", "1", _iter=True): if "Modem hangup" in line: logger.info("Detected modem hang up, going back to listening") time.sleep(5) # Give the hangup some time break dcnow.go_offline() mode = "LISTENING" modem = Modem(device_and_speed[0], device_and_speed[1], BAUD_SPEED, dial_tone_enabled) modem.connect() if dial_tone_enabled: modem.start_dial_tone() return 0
def read(self): for l in sh.tail("-f", self.path, _iter=True): try: yield l.replace('\r\n','') except sh.ErrorReturnCode: yield None
events.append(int(line)) # /eos/atlas/atlastier0/rucio/data15_cos/express_express/00266661/data15_cos.00266661.express_express.merge.RAW files = [x[:-1] for x in list(eos.ls(args.eos, _iter=True))] if args.bylb: for i, file in enumerate(["root://eosatlas/%s/%s" % (args.eos, x) for x in files]): lb = re_lb.search(file).group(1) run = re_run.search(file).group(1) output = "run%s_lb%s.RAW" % (run, lb) if os.path.exists(output): print("%d/%d Skip %s" % (i, len(files), file)) continue else: print("%d/%d Process %s" % (i, len(files), file)) file_events = set(int(x[:-1]) for x in list(cut(cut(tail(atl_list("-l", file), '-n+12'), '-d', " ", '-f3'), '-d', '=', '-f2', _iter=True)) if len(str(x[:-1])) > 1) extract_events = file_events.intersection(events) if not extract_events: continue params = [] for e in extract_events: params += ["-e", e] params += ["-o", output] params += [file] atl_copy(*params) print("Created %s file with events %s" % (output, str(extract_events))) else: run = None params = []
from sh import git, ssh, head, tail, wc, sort, grep, du # Get list of files grep('*') # Get sizes of each du('-hM', grep('*')) # Sort, numerically sort(du('-hM', grep('*')), '-n') # And get the largest tail(sort(du('-hM', grep('*')), '-n'), '-n' 5)