def put(self, **kwargs):
deploy = self._get_deploy(**kwargs)
if deploy is None:
return self.error("Invalid deploy",
name="invalid_resource",
status_code=404)
with lock(redis, f"deploy:{deploy.id}", timeout=5):
# we have to refetch in order to ensure lock state changes
deploy = Deploy.query.get(deploy.id)
task = Task.query.get(deploy.task_id)
args = self.put_parser.parse_args()
if args.status:
assert task.status in (TaskStatus.pending,
TaskStatus.in_progress)
assert args.status == "cancelled"
did_cancel = task.status == TaskStatus.pending
task.status = TaskStatus.cancelled
db.session.add(task)
db.session.commit()
if args.status and did_cancel:
send_task_notifications(task, NotifierEvent.TASK_FINISHED)
return self.respond(serialize(deploy))
def check_queue():
"""
Checks the pending task queue and, given there's not an in-progress task
for the given APP + ENV, marks the latest as in progress and fires the
execute_task job.
"""
pending_queues = list(db.session.query(
Task.app_id, Task.environment
).filter(
Task.status == TaskStatus.pending
).group_by(
Task.app_id, Task.environment
))
logging.info('Found pending tasks for %d queues', len(pending_queues))
for app_id, environment in pending_queues:
app = App.query.get(app_id)
with lock(redis, 'taskcheck:{}-{}'.format(app.id, environment), timeout=5):
if has_active_task(app.id, environment):
logging.info('Task already in progress for %s/%s', app.name, environment)
continue
task_id = get_pending_task_id(app.id, environment)
if not task_id:
logging.info('Unable to find a pending task for %s/%s', app.name, environment)
continue
Task.query.filter(
Task.id == task_id
).update({
'status': TaskStatus.in_progress,
}, synchronize_session=False)
celery.send_task("freight.execute_task", [task_id])
def send_pending_notifications():
while True:
with lock(redis, 'notificationcheck', timeout=5):
data = queue.get()
if data is None:
logging.info('No due notifications found')
return
task = Task.query.get(data['task'])
if task is None:
logging.error('Task not found for notification (id=%s)',
data['task'])
continue
notifier = notifiers.get(data['type'])
try:
notifier.send(
task=task,
config=data['config'],
event=data['event'],
)
except Exception:
logging.exception('%s notifier failed to send Task(id=%s)',
data['type'], task.id)
def check_queue():
"""
Checks the pending task queue and, given there's not an in-progress task
for the given APP + ENV, marks the latest as in progress and fires the
execute_task job.
"""
pending_queues = list(
db.session.query(Task.app_id, Task.environment).filter(
Task.status == TaskStatus.pending).group_by(
Task.app_id, Task.environment))
logging.info('Found pending tasks for %d queues', len(pending_queues))
for app_id, environment in pending_queues:
app = App.query.get(app_id)
with lock(redis,
'taskcheck:{}-{}'.format(app.id, environment),
timeout=5):
if has_active_task(app.id, environment):
logging.info('Task already in progress for %s/%s', app.name,
environment)
continue
task_id = get_pending_task_id(app.id, environment)
if not task_id:
logging.info('Unable to find a pending task for %s/%s',
app.name, environment)
continue
Task.query.filter(Task.id == task_id).update(
{
'status': TaskStatus.in_progress,
},
synchronize_session=False)
celery.send_task("freight.execute_task", [task_id])
def send_pending_notifications():
while True:
with lock(redis, 'notificationcheck', timeout=5):
data = queue.get()
if data is None:
logging.info('No due notifications found')
return
task = Task.query.get(data['task'])
if task is None:
logging.error('Task not found for notification (id=%s)',
data['task'])
continue
notifier = notifiers.get(data['type'])
try:
notifier.send(
task=task,
config=data['config'],
event=data['event'],
)
except Exception:
logging.exception('%s notifier failed to send Task(id=%s)',
data['type'], task.id)
def check_queue():
"""
Checks the pending task queue and, given there's not an in-progress task
for the given APP + ENV, marks the latest as in progress and fires the
execute_task job.
"""
tasks = list(db.session.query(
Task.id, Task.app_id
).filter(
Task.status == TaskStatus.pending
).group_by(
Task.id, Task.app_id
))
if not tasks:
return
logging.info('Found pending tasks for %d queues', len(tasks))
deploys = list(db.session.query(
Deploy.id, Deploy.app_id, Deploy.environment
).filter(
Deploy.task_id.in_(set(t.id for t in tasks))
).group_by(
Deploy.id, Deploy.app_id, Deploy.environment
))
apps = {
a.id: a
for a in App.query.filter(
App.id.in_(set(t.app_id for t in tasks)),
)
}
for deploy_id, app_id, environment in deploys:
app = apps[app_id]
with lock(redis, 'deploycheck:{}-{}'.format(app.id, environment), timeout=5):
if has_active_deploy(app.id, environment):
logging.info('Deploy already in progress for %s/%s', app.name, environment)
continue
task_id = get_pending_task_id(app.id, environment)
if not task_id:
logging.info('Unable to find a pending deploy for %s/%s', app.name, environment)
continue
Task.query.filter(
Task.id == task_id
).update({
'status': TaskStatus.in_progress,
}, synchronize_session=False)
queue.push("freight.jobs.execute_deploy", [deploy_id])
def execute_deploy(deploy_id):
logging.debug(
"ExecuteDeploy fired with %d active thread(s)", threading.active_count()
)
with lock(redis, f"deploy:{deploy_id}", timeout=5):
deploy = Deploy.query.get(deploy_id)
task = Task.query.get(deploy.task_id)
if not task:
logging.warning("ExecuteDeploy fired with missing Deploy(id=%s)", deploy_id)
return
if task.status not in (TaskStatus.pending, TaskStatus.in_progress):
logging.warning(
"ExecuteDeploy fired with finished Deploy(id=%s)", deploy_id
)
return
task.date_started = datetime.utcnow()
task.status = TaskStatus.in_progress
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_STARTED)
provider_config = task.provider_config
# wipe the log incase this is a retry
LogChunk.query.filter(LogChunk.task_id == task.id).delete()
taskrunner = TaskRunner(
task=task,
timeout=provider_config.get("timeout", current_app.config["DEFAULT_TIMEOUT"]),
read_timeout=provider_config.get(
"read_timeout", current_app.config["DEFAULT_READ_TIMEOUT"]
),
)
taskrunner.start()
taskrunner.wait()
# reload the task from the database due to subprocess changes
db.session.expire(task)
db.session.refresh(task)
if task.status in (TaskStatus.pending, TaskStatus.in_progress):
logging.error("Task(id=%s) did not finish cleanly", task.id)
task.status = TaskStatus.failed
task.date_finished = datetime.utcnow()
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_FINISHED)
def execute_deploy(deploy_id):
logging.debug('ExecuteDeploy fired with %d active thread(s)',
threading.active_count())
with lock(redis, 'deploy:{}'.format(deploy_id), timeout=5):
deploy = Deploy.query.get(deploy_id)
task = Task.query.get(deploy.task_id)
if not task:
logging.warning('ExecuteDeploy fired with missing Deploy(id=%s)', deploy_id)
return
if task.status not in (TaskStatus.pending, TaskStatus.in_progress):
logging.warning('ExecuteDeploy fired with finished Deploy(id=%s)', deploy_id)
return
task.date_started = datetime.utcnow()
task.status = TaskStatus.in_progress
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_STARTED)
provider_config = task.provider_config
# wipe the log incase this is a retry
LogChunk.query.filter(
LogChunk.task_id == task.id,
).delete()
taskrunner = TaskRunner(
task=task,
timeout=provider_config.get('timeout', current_app.config['DEFAULT_TIMEOUT']),
read_timeout=provider_config.get('read_timeout', current_app.config['DEFAULT_READ_TIMEOUT']),
)
taskrunner.start()
taskrunner.wait()
# reload the task from the database due to subprocess changes
db.session.expire(task)
db.session.refresh(task)
if task.status in (TaskStatus.pending, TaskStatus.in_progress):
logging.error('Task(id=%s) did not finish cleanly', task.id)
task.status = TaskStatus.failed
task.date_finished = datetime.utcnow()
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_FINISHED)
def check_queue():
"""
Checks the pending task queue and, given there's not an in-progress task
for the given APP + ENV, marks the latest as in progress and fires the
execute_task job.
"""
tasks = list(
db.session.query(Task.id, Task.app_id).filter(
Task.status == TaskStatus.pending).group_by(Task.id, Task.app_id))
if not tasks:
return
logging.info('Found pending tasks for %d queues', len(tasks))
deploys = list(
db.session.query(Deploy.id, Deploy.app_id, Deploy.environment).filter(
Deploy.task_id.in_(set(t.id for t in tasks))).group_by(
Deploy.id, Deploy.app_id, Deploy.environment))
apps = {
a.id: a
for a in App.query.filter(App.id.in_(set(t.app_id for t in tasks)), )
}
for deploy_id, app_id, environment in deploys:
app = apps[app_id]
with lock(redis,
'deploycheck:{}-{}'.format(app.id, environment),
timeout=5):
if has_active_deploy(app.id, environment):
logging.info('Deploy already in progress for %s/%s', app.name,
environment)
continue
task_id = get_pending_task_id(app.id, environment)
if not task_id:
logging.info('Unable to find a pending deploy for %s/%s',
app.name, environment)
continue
Task.query.filter(Task.id == task_id).update(
{
'status': TaskStatus.in_progress,
},
synchronize_session=False)
queue.push("freight.jobs.execute_deploy", [deploy_id])
def _cancel(self):
logging.error("Task(id=%s) was cancelled", self.task.id)
logging.debug("Sending terminate() to LogReporter")
self._logreporter.terminate()
forcefully_stop_process(self._process)
self._logreporter.save_chunk(">> Task was cancelled\n")
with lock(redis, f"task:{self.task.id}", timeout=5):
# TODO(dcramer): ideally we could just send the signal to the subprocess
# so it can still manage the failure state
self.task.date_finished = datetime.utcnow()
db.session.add(self.task)
db.session.commit()
def _cancel(self):
logging.error('Task(id=%s) was cancelled', self.task.id)
logging.debug('Sending terminate() to LogReporter')
self._logreporter.terminate()
forcefully_stop_process(self._process)
self._logreporter.save_chunk('>> Task was cancelled\n')
with lock(redis, 'task:{}'.format(self.task.id), timeout=5):
# TODO(dcramer): ideally we could just send the signal to the subprocess
# so it can still manage the failure state
self.task.date_finished = datetime.utcnow()
db.session.add(self.task)
db.session.commit()
def _read_timeout(self):
logging.error('Task(id=%s) did not receive any updates in %ds', self.task.id, self.read_timeout)
logging.debug('Sending terminate() to LogReporter')
self._logreporter.terminate()
forcefully_stop_process(self._process)
self._logreporter.save_chunk('>> Process did not receive updates in %ds\n' % self.read_timeout)
with lock(redis, 'task:{}'.format(self.task.id), timeout=5):
# TODO(dcramer): ideally we could just send the signal to the subprocess
# so it can still manage the failure state
self.task.status = TaskStatus.failed
self.task.date_finished = datetime.utcnow()
db.session.add(self.task)
db.session.commit()
def _timeout(self):
logging.error(
"Task(id=%s) exceeded time limit of %ds", self.task.id, self.timeout
)
logging.debug("Sending terminate() to LogReporter")
self._logreporter.terminate()
forcefully_stop_process(self._process)
self._logreporter.save_chunk(
">> Process exceeded time limit of %ds\n" % self.timeout
)
with lock(redis, f"task:{self.task.id}", timeout=5):
# TODO(dcramer): ideally we could just send the signal to the subprocess
# so it can still manage the failure state
self.task.status = TaskStatus.failed
self.task.date_finished = datetime.utcnow()
db.session.add(self.task)
db.session.commit()
def send_pending_notifications():
while True:
with lock(redis, "notificationcheck", timeout=5):
data = notifiers.queue.get()
if data is None:
logging.info("No due notifications found")
return
task = Task.query.get(int(data["task"]))
if task is None:
logging.error("Task not found for notification (id=%s)", data["task"])
continue
notifier = notifiers.get(data["type"])
try:
notifier.send(task=task, config=data["config"], event=data["event"])
except Exception:
logging.exception(
"%s notifier failed to send Task(id=%s)", data["type"], task.id
)
def put(self, **kwargs):
task = self._get_task(**kwargs)
if task is None:
return self.error('Invalid task', name='invalid_resource', status_code=404)
with lock(redis, 'task:{}'.format(task.id), timeout=5):
# we have to refetch in order to ensure lock state changes
task = Task.query.get(task.id)
args = self.put_parser.parse_args()
if args.status:
assert task.status in (TaskStatus.pending, TaskStatus.in_progress)
assert args.status == 'cancelled'
did_cancel = task.status == TaskStatus.pending
task.status = TaskStatus.cancelled
db.session.add(task)
db.session.commit()
if args.status and did_cancel:
send_task_notifications(task, NotifierEvent.TASK_FINISHED)
return self.respond(serialize(task))
def post(self):
"""
Given any constraints for a task are within acceptable bounds, create
a new task and enqueue it.
"""
args = self.post_parser.parse_args()
app = App.query.filter(App.name == args.app).first()
if not app:
return self.error('Invalid app', name='invalid_resource', status_code=404)
repo = Repository.query.get(app.repository_id)
workspace = Workspace(
path=repo.get_path(),
)
vcs_backend = vcs.get(
repo.vcs,
url=repo.url,
workspace=workspace,
)
with lock(redis, 'repo:update:{}'.format(repo.id)):
vcs_backend.clone_or_update()
ref = app.get_default_ref(args.env)
try:
sha = vcs_backend.describe(ref)
except vcs.UnknownRevision:
return self.error('Invalid ref', name='invalid_ref', status_code=400)
if not args.force:
for check_config in app.checks:
check = checks.get(check_config['type'])
try:
check.check(app, sha, check_config['config'])
except CheckPending:
pass
except CheckError as e:
return self.error(
message=unicode(e),
name='check_failed',
)
with lock(redis, 'task:create:{}'.format(app.id), timeout=5):
# TODO(dcramer): this needs to be a get_or_create pattern and
# ideally moved outside of the lock
user = User.query.filter(User.name == args.user).first()
if not user:
user = User(name=args.user)
db.session.add(user)
db.session.flush()
if not args.force and self._has_active_task(app, args.env):
return self.error(
message='Another task is already in progress for this app/environment',
name='locked',
)
task = Task(
app_id=app.id,
environment=args.env,
number=TaskSequence.get_clause(app.id, args.env),
name=TaskName.deploy,
# TODO(dcramer): ref should default based on app config
ref=ref,
sha=sha,
status=TaskStatus.pending,
user_id=user.id,
provider=app.provider,
data={
'force': args.force,
'provider_config': app.provider_config,
'notifiers': app.notifiers,
'checks': app.checks,
},
)
db.session.add(task)
db.session.commit()
celery.send_task("freight.execute_task", [task.id])
return self.respond(serialize(task), status_code=201)
def post(self):
"""
Given any constraints for a task are within acceptable bounds, create
a new task and enqueue it.
"""
args = self.post_parser.parse_args()
user = get_current_user()
if not user:
username = args.user
if not username:
return self.error('Missing required argument "user"',
status_code=400)
with lock(redis, 'user:create:{}'.format(username), timeout=5):
# TODO(dcramer): this needs to be a get_or_create pattern and
# ideally moved outside of the lock
user = User.query.filter(User.name == username).first()
if not user:
user = User(name=username)
db.session.add(user)
db.session.flush()
elif args.user:
return self.error(
'Cannot specify user when using session authentication.',
status_code=400)
app = App.query.filter(App.name == args.app).first()
if not app:
return self.error('Invalid app',
name='invalid_resource',
status_code=404)
params = None
repo = Repository.query.get(app.repository_id)
workspace = Workspace(path=repo.get_path(), )
vcs_backend = vcs.get(
repo.vcs,
url=repo.url,
workspace=workspace,
)
with lock(redis, 'repo:update:{}'.format(repo.id)):
vcs_backend.clone_or_update()
ref = args.ref or app.get_default_ref(args.env)
# look for our special refs (prefixed via a colon)
# TODO(dcramer): this should be supported outside of just this endpoint
if ref.startswith(':'):
sha = self._get_internal_ref(app, args.env, ref)
if not sha:
return self.error('Invalid ref',
name='invalid_ref',
status_code=400)
else:
try:
sha = vcs_backend.get_sha(ref)
except vcs.UnknownRevision:
return self.error('Invalid ref',
name='invalid_ref',
status_code=400)
if args.params is not None:
params = args.params
if not args.force:
for check_config in app.checks:
check = checks.get(check_config['type'])
try:
check.check(app, sha, check_config['config'])
except CheckPending:
pass
except CheckError as e:
return self.error(
message=unicode(e),
name='check_failed',
)
with lock(redis, 'task:create:{}'.format(app.id), timeout=5):
task = Task(
app_id=app.id,
environment=args.env,
number=TaskSequence.get_clause(app.id, args.env),
# TODO(dcramer): ref should default based on app config
ref=ref,
sha=sha,
params=params,
status=TaskStatus.pending,
user_id=user.id,
provider=app.provider,
data={
'force': args.force,
'provider_config': app.provider_config,
'notifiers': app.notifiers,
'checks': app.checks,
},
)
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_QUEUED)
return self.respond(serialize(task), status_code=201)
def post(self):
"""
Given any constraints for a task are within acceptable bounds, create
a new task and enqueue it.
"""
args = self.post_parser.parse_args()
user = get_current_user()
if not user:
username = args.user
if not username:
return self.error('Missing required argument "user"',
status_code=400)
with lock(redis, f"user:create:{username}", timeout=5):
# TODO(dcramer): this needs to be a get_or_create pattern and
# ideally moved outside of the lock
user = User.query.filter(User.name == username).first()
if not user:
user = User(name=username)
db.session.add(user)
db.session.flush()
elif args.user:
return self.error(
"Cannot specify user when using session authentication.",
status_code=400,
)
app = App.query.filter(App.name == args.app).first()
if not app:
return self.error("Invalid app",
name="invalid_resource",
status_code=404)
deploy_config = TaskConfig.query.filter(
TaskConfig.app_id == app.id,
TaskConfig.type == TaskConfigType.deploy).first()
if not deploy_config:
return self.error("Missing deploy config",
name="missing_conf",
status_code=404)
params = None
repo = Repository.query.get(app.repository_id)
workspace = Workspace(path=repo.get_path())
vcs_backend = vcs.get(repo.vcs, url=repo.url, workspace=workspace)
with lock(redis, f"repo:update:{repo.id}"):
vcs_backend.clone_or_update()
ref = args.ref or app.get_default_ref(args.env)
# look for our special refs (prefixed via a colon)
# TODO(dcramer): this should be supported outside of just this endpoint
if ref.startswith(":"):
sha = self._get_internal_ref(app, args.env, ref)
if not sha:
return self.error("Invalid ref",
name="invalid_ref",
status_code=400)
else:
try:
sha = vcs_backend.get_sha(ref)
except vcs.UnknownRevision:
return self.error("Invalid ref",
name="invalid_ref",
status_code=400)
if args.params is not None:
params = args.params
if not args.force:
for check_config in deploy_config.checks:
check = checks.get(check_config["type"])
try:
check.check(app, sha, check_config["config"])
except CheckPending:
pass
except CheckError as e:
return self.error(message=str(e), name="check_failed")
with lock(redis, f"deploy:create:{app.id}", timeout=5):
task = Task(
app_id=app.id,
# TODO(dcramer): ref should default based on app config
ref=ref,
sha=sha,
params=params,
status=TaskStatus.pending,
user_id=user.id,
provider=deploy_config.provider,
data={
"force": args.force,
"provider_config": deploy_config.provider_config,
"notifiers": deploy_config.notifiers,
"checks": deploy_config.checks,
},
)
db.session.add(task)
db.session.flush()
db.session.refresh(task)
deploy = Deploy(
task_id=task.id,
app_id=app.id,
environment=args.env,
number=DeploySequence.get_clause(app.id, args.env),
)
db.session.add(deploy)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_QUEUED)
return self.respond(serialize(deploy), status_code=201)
def post(self):
"""
Given any constraints for a task are within acceptable bounds, create
a new task and enqueue it.
"""
args = self.post_parser.parse_args()
app = App.query.filter(App.name == args.app).first()
if not app:
return self.error('Invalid app', name='invalid_resource', status_code=404)
repo = Repository.query.get(app.repository_id)
workspace = Workspace(
path=repo.get_path(),
)
vcs_backend = vcs.get(
repo.vcs,
url=repo.url,
workspace=workspace,
)
with lock(redis, 'repo:update:{}'.format(repo.id)):
vcs_backend.clone_or_update()
ref = args.ref or app.get_default_ref(args.env)
# look for our special refs (prefixed via a colon)
# TODO(dcramer): this should be supported outside of just this endpoint
if ref.startswith(':'):
sha = self._get_internal_ref(app, args.env, ref)
if not sha:
return self.error('Invalid ref', name='invalid_ref', status_code=400)
else:
try:
sha = vcs_backend.get_sha(ref)
except vcs.UnknownRevision:
return self.error('Invalid ref', name='invalid_ref', status_code=400)
if not args.force:
for check_config in app.checks:
check = checks.get(check_config['type'])
try:
check.check(app, sha, check_config['config'])
except CheckPending:
pass
except CheckError as e:
return self.error(
message=unicode(e),
name='check_failed',
)
with lock(redis, 'task:create:{}'.format(app.id), timeout=5):
# TODO(dcramer): this needs to be a get_or_create pattern and
# ideally moved outside of the lock
user = User.query.filter(User.name == args.user).first()
if not user:
user = User(name=args.user)
db.session.add(user)
db.session.flush()
task = Task(
app_id=app.id,
environment=args.env,
number=TaskSequence.get_clause(app.id, args.env),
name=TaskName.deploy,
# TODO(dcramer): ref should default based on app config
ref=ref,
sha=sha,
status=TaskStatus.pending,
user_id=user.id,
provider=app.provider,
data={
'force': args.force,
'provider_config': app.provider_config,
'notifiers': app.notifiers,
'checks': app.checks,
},
)
db.session.add(task)
db.session.commit()
send_task_notifications(task, NotifierEvent.TASK_QUEUED)
return self.respond(serialize(task), status_code=201)
def post(self):
"""
Given any constraints for a task are within acceptable bounds, create
a new task and enqueue it.
"""
args = self.post_parser.parse_args()
app = App.query.filter(App.name == args.app).first()
if not app:
return self.error('Invalid app',
name='invalid_resource',
status_code=404)
repo = Repository.query.get(app.repository_id)
workspace = Workspace(path=repo.get_path(), )
vcs_backend = vcs.get(
repo.vcs,
url=repo.url,
workspace=workspace,
)
with lock(redis, 'repo:update:{}'.format(repo.id)):
vcs_backend.clone_or_update()
ref = app.get_default_ref(args.env)
try:
sha = vcs_backend.describe(ref)
except vcs.UnknownRevision:
return self.error('Invalid ref',
name='invalid_ref',
status_code=400)
if not args.force:
for check_config in app.checks:
check = checks.get(check_config['type'])
try:
check.check(app, sha, check_config['config'])
except CheckPending:
pass
except CheckError as e:
return self.error(
message=unicode(e),
name='check_failed',
)
with lock(redis, 'task:create:{}'.format(app.id), timeout=5):
# TODO(dcramer): this needs to be a get_or_create pattern and
# ideally moved outside of the lock
user = User.query.filter(User.name == args.user).first()
if not user:
user = User(name=args.user)
db.session.add(user)
db.session.flush()
if not args.force and self._has_active_task(app, args.env):
return self.error(
message=
'Another task is already in progress for this app/environment',
name='locked',
)
task = Task(
app_id=app.id,
environment=args.env,
number=TaskSequence.get_clause(app.id, args.env),
name=TaskName.deploy,
# TODO(dcramer): ref should default based on app config
ref=ref,
sha=sha,
status=TaskStatus.pending,
user_id=user.id,
provider=app.provider,
data={
'force': args.force,
'provider_config': app.provider_config,
'notifiers': app.notifiers,
'checks': app.checks,
},
)
db.session.add(task)
db.session.commit()
celery.send_task("freight.execute_task", [task.id])
return self.respond(serialize(task), status_code=201)