def main(from_date, jobs):
# We don't need to send stats so configure logs only
collect_agent.register_collect('')
sock, send_log = build_socket_sender()
date_format = '%Y-%m-%dT%H%M%S.log'
with sock:
for filename in os.listdir(LOGS_DIR):
job_name, _ = filename.rsplit('_', 1)
with suppress(ValueError):
file_last_modified_date = os.path.getmtime(LOGS_DIR+'/'+filename)
from_date_string = datetime.timestamp(from_date)
if jobs and job_name not in jobs:
continue
if file_last_modified_date >= from_date_string:
send_logs(filename, send_log)
def send_stats(filename):
print(filename)
with open(filename) as statistics:
try:
# Parse the first line independently
# so we can update os.ENVIRON
statistic = json.loads(next(statistics))
except StopIteration:
return # File was empty
# Setup os.ENVIRON for register_collect to work properly
metadata = statistic.pop('_metadata')
timestamp = metadata['time']
suffix = metadata.get('suffix')
for name in ENVIRON_METADATA:
# This way rstats will be aware and will not locally store the
# stats again
if name == 'job_name':
metadata[name] = 'send_stats-' + str(metadata[name])
os.environ[name.upper()] = str(metadata[name])
# Recreate connection with rstats
success = collect_agent.register_collect(CONF_FILE, new=True)
if not success:
message = 'Cannot communicate with rstats'
collect_agent.send_log(syslog.LOG_ERR, message)
raise ConnectionError(message)
collect_agent.send_stat(timestamp, suffix=suffix, **statistic)
for line in statistics:
statistic = json.loads(line)
metadata = statistic.pop('_metadata')
timestamp = metadata['time']
suffix = metadata.get('suffix')
collect_agent.send_stat(timestamp, suffix=suffix, **statistic)
def use_configuration(filepath):
success = collect_agent.register_collect(filepath)
if not success:
message = 'ERROR connecting to collect-agent'
collect_agent.send_log(syslog.LOG_ERR, message)
sys.exit(message)
collect_agent.send_log(syslog.LOG_DEBUG, 'Starting job ' + os.environ.get('JOB_NAME', '!'))
try:
yield
except Exception:
message = traceback.format_exc()
collect_agent.send_log(syslog.LOG_CRIT, message)
raise
except SystemExit as e:
if e.code != 0:
collect_agent.send_log(syslog.LOG_CRIT, 'Abrupt program termination: ' + str(e.code))
raise