def main():
settings = get_settings()
conn = psycopg2.connect(**parse_pgurl(settings.db_url))
conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
rabbit_conn = pika.BlockingConnection(
pika.URLParameters(settings.rabbit_url))
rabbit = rabbit_conn.channel()
exchange = settings['state_exchange']
rabbit.exchange_declare(exchange=exchange, type='topic', durable=True)
pubsub = pgpubsub.PubSub(conn)
pubsub.listen(PG_CHANNEL)
logger.info('Listening on Postgres channel "%s"' % PG_CHANNEL)
for event in pubsub.events():
payload = json.loads(event.payload)
table = payload['tablename']
with conn.cursor() as curs:
# Handle payloads too big for a PG NOTIFY.
if payload.get('error') == 'too long':
payload = get_record_as_json(curs, table, payload['id'])
# add service_name and pipeline_name where applicable, doing
# another DB lookup if necessary.
extra = extra_data(curs, table, payload['id'])
payload.update(extra)
publish_event(rabbit, exchange, payload)
def send_email(to, subj, body):
"""
Send an email using the server specified by the 'smtp_url' setting.
'to' should be a list of addresses.
'from' should be a single address.
Any address may be provided as a plain string like '*****@*****.**', or
as a tuple like ('Joe Somebody', '*****@*****.**').
"""
settings = get_settings()
msg = MIMEText(body)
msg['From'] = format_email_address(settings.smtp_sender)
msg['Subject'] = subj
for r in to:
addr = format_email_address(r)
msg.add_header('To', addr)
logger.info('Sending email "{subject}" to {recipients}.'.format(
subject=subj,
recipients=', '.join([str(x) for x in to]),
))
if settings.smtp_url is not None:
with SMTPServer(settings.smtp_url) as server:
server.sendmail(just_email_address(settings.smtp_sender),
msg.get_all('To'), msg.as_string())
else:
logger.warning('settings.smtp_url is None. Email not sent!')
def main():
settings = get_settings()
rabbit_conn = pika.BlockingConnection(
pika.URLParameters(settings.rabbit_url))
rabbit = rabbit_conn.channel()
mp.declare_exchanges(rabbit)
queue_name = 'mettle_job_logs'
rabbit.queue_declare(queue=queue_name, exclusive=False, durable=True)
rabbit.queue_bind(exchange=mp.JOB_LOGS_EXCHANGE,
queue=queue_name,
routing_key='#')
logger.info('Bound exchange %s to queue %s' %
(mp.JOB_LOGS_EXCHANGE, queue_name))
Session = make_session_cls(settings.db_url)
for method, properties, body in rabbit.consume(queue=queue_name):
db = Session()
data = json.loads(body)
job_id = data['job_id']
line_num = data['line_num']
message = data['msg']
try:
db.add(
JobLogLine(
job_id=job_id,
line_num=line_num,
message=message,
))
db.commit()
logger.info(message)
except IntegrityError:
# We probably got a duplicate log line, which can happen given
# Rabbit retries. Query DB for log line matching job_id and
# line_num. If we have one, and it is the same message, then just
# carry on. If the message is different, then log an error.
db.rollback()
existing_line = db.query(JobLogLine).filter_by(
job_id=job_id, line_num=line_num).one()
if existing_line.message != message:
err = """Job {job_id}, log line {num} is stored as
this:\n{old}\n\n but the queue has just produced a new message
for the same line, with this value:\n{new}"""
logger.error(
textwrap.dedent(err).format(
job_id=job_id,
num=line_num,
old=existing_line.message,
new=message,
))
rabbit.basic_ack(method.delivery_tag)
def main():
settings = get_settings()
rabbit_conn = pika.BlockingConnection(
pika.URLParameters(settings.rabbit_url))
rabbit = rabbit_conn.channel()
mp.declare_exchanges(rabbit)
queue_name = 'mettle_dispatcher'
rabbit.queue_declare(queue=queue_name, exclusive=False, durable=True)
rabbit.queue_bind(exchange=mp.ANNOUNCE_SERVICE_EXCHANGE,
queue=queue_name,
routing_key='#')
rabbit.queue_bind(exchange=mp.ACK_PIPELINE_RUN_EXCHANGE,
queue=queue_name,
routing_key='#')
rabbit.queue_bind(exchange=mp.NACK_PIPELINE_RUN_EXCHANGE,
queue=queue_name,
routing_key='#')
rabbit.queue_bind(exchange=mp.CLAIM_JOB_EXCHANGE,
queue=queue_name,
routing_key='#')
rabbit.queue_bind(exchange=mp.END_JOB_EXCHANGE,
queue=queue_name,
routing_key='#')
rabbit.queue_bind(exchange=settings.dispatcher_ping_exchange,
queue=queue_name,
routing_key='timer')
Session = make_session_cls(settings.db_url)
for method, properties, body in rabbit.consume(queue=queue_name):
db = Session()
if method.exchange == mp.ANNOUNCE_SERVICE_EXCHANGE:
on_announce_service(settings, db, json.loads(body))
elif method.exchange == mp.ACK_PIPELINE_RUN_EXCHANGE:
on_pipeline_run_ack(settings, rabbit, db, json.loads(body))
elif method.exchange == mp.NACK_PIPELINE_RUN_EXCHANGE:
on_pipeline_run_nack(settings, rabbit, db, json.loads(body))
elif method.exchange == mp.CLAIM_JOB_EXCHANGE:
on_job_claim(settings, rabbit, db, json.loads(body),
properties.correlation_id)
elif method.exchange == mp.END_JOB_EXCHANGE:
on_job_end(settings, rabbit, db, json.loads(body))
# get messages from process timer restart queue
elif method.exchange == settings.dispatcher_ping_exchange:
db.merge(Checkin(proc_name='dispatcher', time=utc.now()))
db.commit()
rabbit.basic_ack(method.delivery_tag)
def test_long_routing_key():
settings = get_settings()
conn = pika.BlockingConnection(pika.URLParameters(settings.rabbit_url))
chan = conn.channel()
exchange = settings['state_exchange']
chan.exchange_declare(exchange=exchange, type='topic', durable=True)
with pytest.raises(ValueError):
publish_event(
chan, exchange,
dict(
description=None,
tablename='a' * 8000,
name="foo",
pipeline_names=None,
id=15,
updated_by='vagrant',
))
def run():
settings = get_settings()
conn = psycopg2.connect(**parse_pgurl(settings.db_url))
conn.autocommit = True
cur = conn.cursor()
migrations_folder = resource_filename('mettle', 'migrations')
migration_names = [
i for i in os.listdir(migrations_folder)
if os.path.isdir(os.path.join(migrations_folder, i))
]
logger.info('Found migrations: ' + ', '.join(migration_names))
try:
cur.execute('SELECT name FROM migration_history ORDER BY name;')
completed_migrations = [m[0] for m in cur]
except psycopg2.ProgrammingError:
# The first migration creates the migration_history table. So the query
# on that table will fail if we have never run migrations.
completed_migrations = []
logger.info('Already run: ' + ', '.join(completed_migrations))
to_run = sorted(list(
set(migration_names).difference(completed_migrations)))
if not len(to_run):
logger.info('No migrations need running.')
return
logger.info('Migrations to run: ' + ', '.join(to_run))
for m in to_run:
logger.info('Running %s.' % m)
script = os.path.join(migrations_folder, m, 'forward.sql')
sql = open(script).read()
cur.execute(sql)
def main():
settings = get_settings()
session = make_session_cls(settings.db_url)()
services = {}
for service_name, service_data in settings.stub_services.items():
service = session.query(Service).filter_by(name=service_name).first()
if not service:
logger.info('Making service %s' % service_name)
service = Service(
name=service_name,
updated_by='datastub',
pipeline_names=service_data['pipeline_names'],
)
session.add(service)
services[service.name] = service
nl_lists = {}
for nl_data in settings.stub_notification_lists:
nl = session.query(NotificationList).filter_by(
name=nl_data['name'],
).first()
if not nl:
logger.info('Making notification list %s' % nl_data['name'])
nl = NotificationList(
name=nl_data['name'],
updated_by='datastub',
)
session.add(nl)
nl.recipients = nl_data['recipients']
nl_lists[nl.name] = nl
pipelines = {}
for pl_data in settings.stub_pipelines:
pipeline = session.query(Pipeline).filter_by(
name=pl_data['name'],
service=services[pl_data['service']],
).first()
if not pipeline:
logger.info('Making pipeline %s' % pl_data['name'])
pipeline = Pipeline(
name=pl_data['name'],
service=services[pl_data['service']],
updated_by='datastub',
)
if 'crontab' in pl_data:
pipeline.crontab = pl_data['crontab']
elif 'chained_from' in pl_data:
pipeline.chained_from=pipelines[pl_data['chained_from']]
session.add(pipeline)
pipeline.notification_list = nl_lists[pl_data['notification_list']]
pipelines[pl_data['name']] = pipeline
# query for all pipelines. any not in the data stub should be made
# inactive.
for pipeline in session.query(Pipeline):
if pipeline.name not in pipelines:
logger.info('Deactivating unstubbed pipeline "%s".' % pipeline.name)
pipeline.active = False
session.commit()
logger.info('Sleeping')
sys.stdin.read()
def main():
settings = get_settings()
while True:
do_scheduled_tasks(settings)
logger.info("Sleeping for %s seconds" % settings.timer_sleep_secs)
time.sleep(settings.timer_sleep_secs)
def get_db():
settings = get_settings()
return make_session_cls(settings.db_url)