async def html_app(request, app_name):
app = Repo.select().where(Repo.name == app_name)
if app.count == 0:
raise NotFound()
return {"app": app[0], 'relative_path_to_root': '../../', 'path': request.path}
async def launch_monthly_job():
today = date.today().day
for repo in Repo.select().where(Repo.random_job_day == today):
task_logger.info(
f"Launch monthly job for {repo.name} on day {today} of the month ")
await create_job(repo.name, repo.url)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user").where(
(User.name == username) & (Repo.name == reponame)).get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
if key and not timemap:
self.render("repo/memento.html",
repo=repo,
key=key,
datetime=datetime)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
else:
cs = (CSet.select(fn.distinct(
CSet.hkey)).where(CSet.repo == repo).limit(5).alias("cs"))
samples = (HMap.select(HMap.val).join(
cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html",
title=title,
repo=repo,
samples=list(samples))
except Repo.DoesNotExist:
raise HTTPError(404)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user")
.where((User.name == username) & (Repo.name == reponame))
.get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
if key and not timemap:
self.render("repo/memento.html", repo=repo, key=key,
datetime=datetime)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
else:
cs = (CSet.select(fn.distinct(CSet.hkey))
.where(CSet.repo == repo).limit(5).alias("cs"))
samples = (HMap.select(HMap.val)
.join(cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html", title=title, repo=repo,
samples=list(samples))
except Repo.DoesNotExist:
raise HTTPError(404)
def get(self, username, reponame):
try:
repo = (Repo.select().join(User).alias("user")
.where((User.name == username) & (Repo.name == reponame))
.get())
title = repo.user.name + "/" + repo.name
timemap = self.get_query_argument("timemap", "false") == "true"
datetime = self.get_query_argument("datetime", None)
key = self.get_query_argument("key", None)
index = self.get_query_argument("index", "false") == "true"
if self.get_query_argument("datetime", None):
datestr = self.get_query_argument("datetime")
try:
ts = date(datestr, QSDATEFMT)
except ValueError:
raise HTTPError(reason="Invalid format of datetime param", status_code=400)
elif "Accept-Datetime" in self.request.headers:
datestr = self.request.headers.get("Accept-Datetime")
ts = date(datestr, RFC1123DATEFMT)
else:
ts = now()
if key and not timemap:
chain = revision_logic.get_chain_at_ts(repo, key, ts)
# use ts of cset instead of now(), to make prev work
if len(chain) != 0:
ts = chain[-1].time
cs_prev = revision_logic.get_cset_prev_before_ts(repo, key, ts)
cs_next = revision_logic.get_cset_next_after_ts(repo, key, ts)
if cs_prev:
cs_prev_str = self.request.protocol + "://" + self.request.host + self.request.path + "?key=" + key + "&datetime=" + cs_prev.time.strftime(QSDATEFMT)
else:
cs_prev_str = ""
if cs_next:
cs_next_str = self.request.protocol + "://" + self.request.host + self.request.path + "?key=" + key + "&datetime=" + cs_next.time.strftime(QSDATEFMT)
else:
cs_next_str = ""
commit_message = revision_logic.get_commit_message(repo, key, ts)
self.render("repo/memento.html", repo=repo, key=key, datetime=datetime, cs_next_str=cs_next_str, cs_prev_str=cs_prev_str, commit_message=commit_message)
elif key and timemap:
self.render("repo/history.html", repo=repo, key=key)
elif index:
cs = (CSet.select(fn.distinct(CSet.hkey)).where((CSet.repo == repo) & (CSet.time <= ts)).alias("cs"))
key_count = (HMap.select(HMap.val).join(cs, on=(HMap.sha == cs.c.hkey_id))).count()
page = int(self.get_query_argument("page", "1"))
hm = revision_logic.get_repo_index(repo, ts, page)
self.render("repo/index.html", repo=repo, title=title, key_count=key_count, page_size=revision_logic.INDEX_PAGE_SIZE, hm=hm, current_page=page)
else:
hm = list(revision_logic.get_repo_index(repo, ts, 1, 5))
# cs = (CSet.select(fn.distinct(CSet.hkey)).where(CSet.repo == repo).limit(5).alias("cs"))
# samples = (HMap.select(HMap.val).join(cs, on=(HMap.sha == cs.c.hkey_id)))
self.render("repo/show.html", title=title, repo=repo, hm=hm)
except Repo.DoesNotExist:
raise HTTPError(reason="Repo not found.", status_code=404)
def set_random_day_for_monthy_job():
for repo in Repo.select().where((Repo.random_job_day == None)):
repo.random_job_day = random.randint(1, 28)
task_logger.info(
f"set random day for monthly job of repo '{repo.name}' at '{repo.random_job_day}'"
)
repo.save()
def get(self, username, reponame):
try:
repo = Repo.select().join(User).alias("user").where((User.name == username) & (Repo.name == reponame)).get()
if not repo.private:
self._get(repo)
else:
self._getAuth(repo)
except Repo.DoesNotExist:
raise HTTPError(reason="Repo not found.", status_code=404)
def get(self):
query = tornado.escape.url_unescape(self.get_argument("q", ""))
if query:
pattern = "%" + query + "%"
repos = Repo.select().join(User).alias("user").where(Repo.name ** pattern, Repo.private == False)
users = User.select().where(User.name ** pattern)
else:
repos = []
users = []
self.render("search/show.html", query=query, repos=repos, users=users)
async def launch_monthly_job(type):
# XXX DRY
job_command_last_part = ""
if type == "arm":
job_command_last_part = " (~ARM~)"
elif type == "testing-unstable":
job_command_last_part = [" (testing)", " (unstable)"]
today = date.today().day
for repo in Repo.select().where(Repo.random_job_day == today):
task_logger.info(f"Launch monthly job for {repo.name} on day {today} of the month ")
await create_job(repo.name, repo.app_list, repo, job_command_last_part)
def get(self):
query = tornado.escape.url_unescape(self.get_argument("q", ""))
if query:
pattern = "%" + query + "%"
repos = (Repo.select().join(User).alias("user").where(
Repo.name**pattern))
users = User.select().where(User.name**pattern)
else:
repos = []
users = []
self.render("search/show.html", query=query, repos=repos, users=users)
async def ws_app(request, websocket, app_name):
# XXX I don't check if the app exists because this websocket is supposed to
# be only loaded from the app page which does this job already
app = Repo.select().where(Repo.name == app_name)[0]
subscribe(websocket, f"app-jobs-{app.url}")
await websocket.send(ujson.dumps({
"action": "init_jobs",
"data": Job.select().where(Job.url_or_path ==
app.url).order_by(-Job.id),
}))
await websocket.wait_closed()
async def html_job(request, job_id):
job = Job.select().where(Job.id == job_id)
if job.count == 0:
raise NotFound()
job = job[0]
app = Repo.select().where(Repo.url == job.url_or_path)
app = app[0] if app else None
return {
"job": job,
'app': app,
'relative_path_to_root': '../',
'path': request.path
}
def delete(self, username, reponame):
# Check whether the key exists and if maybe the last change already is
# a delete, else insert a `CSet.DELETE` entry without any blob data.
key = self.get_query_argument("key")
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
try:
last = (CSet.select(
CSet.time,
CSet.type).where((CSet.repo == repo)
& (CSet.hkey == sha)).order_by(
CSet.time.desc()).limit(1).naive().get())
except CSet.DoesNotExist:
# No changeset was found for the given key -
# the resource does not exist.
raise HTTPError(400)
if not ts > last.time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if last.type == CSet.DELETE:
# The resource was deleted already, return instantly.
return self.finish()
# Insert the new "delete" change.
CSet.create(repo=repo, hkey=sha, time=ts, type=CSet.DELETE, len=0)
def test_commits_create(self, get_commits):
fixtures = self.data.buffer.read()
jsn = json.loads(fixtures.decode())
request = HTTPRequest(self.TEST_REPO['href'])
response = HTTPResponse(request, client.OK, buffer=io.BytesIO(fixtures))
future = Future()
future.set_result(response)
get_commits.return_value = future
body = {'href': self.TEST_REPO['href']}
response = self.fetch(self.get_app().reverse_url('create'), method='POST',
body=urlencode(body).encode())
self.assertIn('всего {}'.format(len(jsn)), response.body.decode())
self.assertEqual(len(jsn), Commit.select().count())
self.assertEqual(1, Repo.select().count())
def get(self, username):
try:
user = User.select().where(User.name == username).get()
except User.DoesNotExist:
raise HTTPError(reason="User not found.", status_code=404)
repos = Repo.select().where(Repo.user == user)
reposit = repos.iterator()
# TODO: Paginate?
first = None
try:
first = reposit.next()
except StopIteration:
# No repos for user
# No need to raise an error, just return empty list in json
pass
accept = self.request.headers.get("Accept", "")
user_url = (self.request.protocol + "://" + self.request.host)
if "application/json" in accept or "*/*" in accept:
self.set_header("Content-Type", "application/json")
self.write('{"username": '******', "repositories": {"list":[')
m = ('{{"name": "{0}", "uri": "' + user_url +
'/'+username+'/{0}"}}')
if first:
self.write(m.format(first.name))
for repo in reposit:
self.write(', ' + m.format(repo.name))
self.write(']}')
self.write('}')
def get_repo_count(): return Repo.select().count()
def get(self, username, reponame):
timemap = self.get_query_argument("timemap", "false") == "true"
index = self.get_query_argument("index", "false") == "true"
key = self.get_query_argument("key", None)
if (index and timemap) or (index and key) or (timemap and not key):
raise HTTPError(400)
if self.get_query_argument("datetime", None):
datestr = self.get_query_argument("datetime")
ts = date(datestr, QSDATEFMT)
elif "Accept-Datetime" in self.request.headers:
datestr = self.request.headers.get("Accept-Datetime")
ts = date(datestr, RFC1123DATEFMT)
else:
ts = now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
if key and not timemap:
# Recreate the resource for the given key in its latest state -
# if no `datetime` was provided - or in the state it was in at
# the time indicated by the passed `datetime` argument.
self.set_header("Content-Type", "application/n-quads")
self.set_header("Vary", "accept-datetime")
sha = shasum(key.encode("utf-8"))
# Fetch all relevant changes from the last "non-delta" onwards,
# ordered by time. The returned delta-chain consists of either:
# a snapshot followed by 0 or more deltas, or
# a single delete.
chain = list(
CSet.select(CSet.time, CSet.type).where((CSet.repo == repo) & (
CSet.hkey == sha) & (CSet.time <= ts) & (CSet.time >= SQL(
"COALESCE((SELECT time FROM cset "
"WHERE repo_id = %s "
"AND hkey_id = %s "
"AND time <= %s "
"AND type != %s "
"ORDER BY time DESC "
"LIMIT 1), 0)", repo.id, sha, ts, CSet.DELTA))).
order_by(CSet.time).naive())
if len(chain) == 0:
# A resource does not exist for the given key.
raise HTTPError(404)
timegate_url = (self.request.protocol + "://" + self.request.host +
self.request.path)
timemap_url = (self.request.protocol + "://" + self.request.host +
self.request.uri + "&timemap=true")
self.set_header(
"Link", '<%s>; rel="original"'
', <%s>; rel="timegate"'
', <%s>; rel="timemap"' % (key, timegate_url, timemap_url))
self.set_header("Memento-Datetime",
chain[-1].time.strftime(RFC1123DATEFMT))
if chain[0].type == CSet.DELETE:
# The last change was a delete. Return a 404 response with
# appropriate "Link" and "Memento-Datetime" headers.
raise HTTPError(404)
# Load the data required in order to restore the resource state.
blobs = (Blob.select(Blob.data).where(
(Blob.repo == repo) & (Blob.hkey == sha)
& (Blob.time << map(lambda e: e.time, chain))).order_by(
Blob.time).naive())
if len(chain) == 1:
# Special case, where we can simply return
# the blob data of the snapshot.
snap = blobs.first().data
return self.finish(decompress(snap))
stmts = set()
for i, blob in enumerate(blobs.iterator()):
data = decompress(blob.data)
if i == 0:
# Base snapshot for the delta chain
stmts.update(data.splitlines())
else:
for line in data.splitlines():
mode, stmt = line[0], line[2:]
if mode == "A":
stmts.add(stmt)
else:
stmts.discard(stmt)
self.write(join(stmts, "\n"))
elif key and timemap:
# Generate a timemap containing historic change information
# for the requested key. The timemap is in the default link-format
# or as JSON (http://mementoweb.org/guide/timemap-json/).
sha = shasum(key.encode("utf-8"))
csets = (CSet.select(
CSet.time).where((CSet.repo == repo)
& (CSet.hkey == sha)).order_by(
CSet.time.desc()).naive())
# TODO: Paginate?
csit = csets.iterator()
try:
first = csit.next()
except StopIteration:
# Resource for given key does not exist.
raise HTTPError(404)
req = self.request
base = req.protocol + "://" + req.host + req.path
accept = self.request.headers.get("Accept", "")
if "application/json" in accept or "*/*" in accept:
self.set_header("Content-Type", "application/json")
self.write('{"original_uri": ' + json_encode(key))
self.write(', "mementos": {"list":[')
m = ('{{"datetime": "{0}", "uri": "' + base + '?key=' +
url_escape(key) + '&datetime={1}"}}')
self.write(
m.format(first.time.isoformat(),
first.time.strftime(QSDATEFMT)))
for cs in csit:
self.write(', ' + m.format(cs.time.isoformat(),
cs.time.strftime(QSDATEFMT)))
self.write(']}')
self.write('}')
else:
m = (',\n'
'<' + base + '?key=' + url_escape(key) + '&datetime={0}>'
'; rel="memento"'
'; datetime="{1}"'
'; type="application/n-quads"')
self.set_header("Content-Type", "application/link-format")
self.write('<' + key + '>; rel="original"')
self.write(
m.format(first.time.strftime(QSDATEFMT),
first.time.strftime(RFC1123DATEFMT)))
for cs in csit:
self.write(
m.format(cs.time.strftime(QSDATEFMT),
cs.time.strftime(RFC1123DATEFMT)))
elif index:
# Generate an index of all URIs contained in the dataset at the
# provided point in time or in its current state.
self.set_header("Vary", "accept-datetime")
self.set_header("Content-Type", "text/plain")
page = int(self.get_query_argument("page", "1"))
# Subquery for selecting max. time per hkey group
mx = (CSet.select(
CSet.hkey,
fn.Max(CSet.time).alias("maxtime")).where(
(CSet.repo == repo) & (CSet.time <= ts)).group_by(
CSet.hkey).order_by(CSet.hkey).paginate(
page, INDEX_PAGE_SIZE).alias("mx"))
# Query for all the relevant csets (those with max. time values)
cs = (CSet.select(CSet.hkey, CSet.time).join(
mx,
on=((CSet.hkey == mx.c.hkey_id) & (CSet.time == mx.c.maxtime)
)).where((CSet.repo == repo)
& (CSet.type != CSet.DELETE)).alias("cs"))
# Join with the hmap table to retrieve the plain key values
hm = (HMap.select(HMap.val).join(
cs, on=(HMap.sha == cs.c.hkey_id)).naive())
for h in hm.iterator():
self.write(h.val + "\n")
else:
raise HTTPError(400)
async def api_list_app(request):
query = Repo.select()
return response.json([model_to_dict(x) for x in query.order_by(Repo.name)])
async def ws_apps(request, websocket):
subscribe(websocket, "jobs")
subscribe(websocket, "apps")
# I need to do this because peewee strangely f**k up on join and remove the
# subquery fields which breaks everything
repos = Repo.raw('''
SELECT
"id",
"name",
"url",
"revision",
"state",
"random_job_day",
"job_id",
"job_name",
"job_state",
"created_time",
"started_time",
"end_time"
FROM
"repo" AS "t1"
INNER JOIN (
SELECT
"t1"."id" as "job_id",
"t1"."name" as "job_name",
"t1"."url_or_path",
"t1"."state" as "job_state",
"t1"."created_time",
"t1"."started_time",
"t1"."end_time"
FROM
"job" AS "t1"
INNER JOIN (
SELECT
Max("t2"."id") AS "max_id"
FROM
"job" AS "t2"
GROUP BY
"t2"."url_or_path"
)
AS
"t3"
ON
("t1"."id" = "t3"."max_id")
) AS
"t5"
ON
("t5"."url_or_path" = "t1"."url")
ORDER BY
"name"
''')
repos = [{
"id":
x.id,
"name":
x.name,
"url":
x.url,
"revision":
x.revision,
"state":
x.state,
"random_job_day":
x.random_job_day,
"job_id":
x.job_id,
"job_name":
x.job_name,
"job_state":
x.job_state,
"created_time":
datetime.strptime(x.created_time.split(".")[0], '%Y-%m-%d %H:%M:%S')
if x.created_time else None,
"started_time":
datetime.strptime(x.started_time.split(".")[0], '%Y-%m-%d %H:%M:%S')
if x.started_time else None,
"end_time":
datetime.strptime(x.end_time.split(".")[0], '%Y-%m-%d %H:%M:%S')
if x.end_time else None,
} for x in repos]
# add apps without jobs
selected_repos = {x["id"] for x in repos}
for repo in Repo.select().where(Repo.id.not_in(selected_repos)):
repos.append({
"id": repo.id,
"name": repo.name,
"url": repo.url,
"revision": repo.revision,
"state": repo.state,
"random_job_day": repo.random_job_day,
"job_id": None,
"job_name": None,
"job_state": None,
"created_time": None,
"started_time": None,
"end_time": None,
})
repos = sorted(repos, key=lambda x: x["name"])
await websocket.send(ujson.dumps({
"action": "init_apps",
"data": repos,
}))
await websocket.wait_closed()
def put(self, username, reponame):
# Create a new revision of the resource specified by `key`.
fmt = self.request.headers.get("Content-Type", "application/n-triples")
key = self.get_query_argument("key", None)
if username != self.current_user.name:
raise HTTPError(403)
if not key:
raise HTTPError(400)
datestr = self.get_query_argument("datetime", None)
ts = datestr and date(datestr, QSDATEFMT) or now()
try:
repo = (Repo.select(Repo.id).join(
User).where((User.name == username)
& (Repo.name == reponame)).naive().get())
except Repo.DoesNotExist:
raise HTTPError(404)
sha = shasum(key.encode("utf-8"))
chain = list(
CSet.select(CSet.time, CSet.type, CSet.len).where(
(CSet.repo == repo) & (CSet.hkey == sha) & (CSet.time >= SQL(
"COALESCE((SELECT time FROM cset "
"WHERE repo_id = %s "
"AND hkey_id = %s "
"AND type != %s "
"ORDER BY time DESC "
"LIMIT 1), 0)", repo.id, sha, CSet.DELTA))).order_by(
CSet.time).naive())
if len(chain) > 0 and not ts > chain[-1].time:
# Appended timestamps must be monotonically increasing!
raise HTTPError(400)
if len(chain) == 0:
# Mapping for `key` likely does not exist:
# Store the SHA-to-KEY mapping in HMap,
# looking out for possible collisions.
try:
HMap.create(sha=sha, val=key)
except IntegrityError:
val = HMap.select(HMap.val).where(HMap.sha == sha).scalar()
if val != key:
raise HTTPError(500)
# Parse and normalize into a set of N-Quad lines
stmts = parse(self.request.body, fmt)
snapc = compress(join(stmts, "\n"))
if len(chain) == 0 or chain[0].type == CSet.DELETE:
# Provide dummy value for `patch` which is never stored.
# If we get here, we always store a snapshot later on!
patch = ""
else:
# Reconstruct the previous state of the resource
prev = set()
blobs = (Blob.select(Blob.data).where(
(Blob.repo == repo) & (Blob.hkey == sha)
& (Blob.time << map(lambda e: e.time, chain))).order_by(
Blob.time).naive())
for i, blob in enumerate(blobs.iterator()):
data = decompress(blob.data)
if i == 0:
# Base snapshot for the delta chain
prev.update(data.splitlines())
else:
for line in data.splitlines():
mode, stmt = line[0], line[2:]
if mode == "A":
prev.add(stmt)
else:
prev.discard(stmt)
if stmts == prev:
# No changes, nothing to be done. Bail out.
return self.finish()
patch = compress(
join(
map(lambda s: "D " + s, prev - stmts) +
map(lambda s: "A " + s, stmts - prev), "\n"))
# Calculate the accumulated size of the delta chain including
# the (potential) patch from the previous to the pushed state.
acclen = reduce(lambda s, e: s + e.len, chain[1:], 0) + len(patch)
blen = len(chain) > 0 and chain[0].len or 0 # base length
if (len(chain) == 0 or chain[0].type == CSet.DELETE
or len(snapc) <= len(patch) or SNAPF * blen <= acclen):
# Store the current state as a new snapshot
Blob.create(repo=repo, hkey=sha, time=ts, data=snapc)
CSet.create(repo=repo,
hkey=sha,
time=ts,
type=CSet.SNAPSHOT,
len=len(snapc))
else:
# Store a directed delta between the previous and current state
Blob.create(repo=repo, hkey=sha, time=ts, data=patch)
CSet.create(repo=repo,
hkey=sha,
time=ts,
type=CSet.DELTA,
len=len(patch))
async def monitor_apps_lists(monitor_git=False,
monitor_only_good_quality_apps=False):
"parse apps lists every hour or so to detect new apps"
# only support github for now :(
async def get_master_commit_sha(url):
command = await asyncio.create_subprocess_shell(
f"git ls-remote {url} master",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
data = await command.stdout.read()
commit_sha = data.decode().strip().replace("\t", " ").split(" ")[0]
return commit_sha
async with aiohttp.ClientSession() as session:
task_logger.info(f"Downloading applist...")
async with session.get(APPS_LIST) as resp:
data = await resp.json()
data = data["apps"]
repos = {x.name: x for x in Repo.select()}
for app_id, app_data in data.items():
commit_sha = await get_master_commit_sha(app_data["git"]["url"])
if app_data["state"] != "working":
task_logger.debug(
f"skip {app_id} because state is {app_data['state']}")
continue
if monitor_only_good_quality_apps:
if app_data.get("level") in [None, "?"] or app_data["level"] <= 4:
task_logger.debug(
f"skip {app_id} because app is not good quality")
continue
# already know, look to see if there is new commits
if app_id in repos:
repo = repos[app_id]
# but first check if the URL has changed
if repo.url != app_data["git"]["url"]:
task_logger.info(
f"Application {app_id} has changed of url from {repo.url} to {app_data['git']['url']}"
)
repo.url = app_data["git"]["url"]
repo.save()
await broadcast(
{
"action": "update_app",
"data": model_to_dict(repo),
}, "apps")
# change the url of all jobs that used to have this URL I
# guess :/
# this isn't perfect because that could overwrite added by
# hand jobs but well...
for job in Job.select().where(Job.url_or_path == repo.url,
Job.state == "scheduled"):
job.url_or_path = repo.url
job.save()
task_logger.info(
f"Updating job {job.name} #{job.id} for {app_id} to {repo.url} since the app has changed of url"
)
await broadcast(
{
"action": "update_job",
"data": model_to_dict(job),
}, [
"jobs", f"job-{job.id}",
f"app-jobs-{job.url_or_path}"
])
# we don't want to do anything else
if not monitor_git:
continue
repo_is_updated = False
if repo.revision != commit_sha:
task_logger.info(
f"Application {app_id} has new commits on github "
f"({repo.revision} → {commit_sha}), schedule new job")
repo.revision = commit_sha
repo.save()
repo_is_updated = True
await create_job(app_id, repo.url)
repo_state = "working" if app_data[
"state"] == "working" else "other_than_working"
if repo.state != repo_state:
repo.state = repo_state
repo.save()
repo_is_updated = True
if repo.random_job_day is None:
repo.random_job_day = random.randint(1, 28)
repo.save()
repo_is_updated = True
if repo_is_updated:
await broadcast(
{
"action": "update_app",
"data": model_to_dict(repo),
}, "apps")
# new app
elif app_id not in repos:
task_logger.info(f"New application detected: {app_id} " +
(", scheduling a new job" if monitor_git else ""))
repo = Repo.create(
name=app_id,
url=app_data["git"]["url"],
revision=commit_sha,
state="working"
if app_data["state"] == "working" else "other_than_working",
random_job_day=random.randint(1, 28),
)
await broadcast(
{
"action": "new_app",
"data": model_to_dict(repo),
}, "apps")
if monitor_git:
await create_job(app_id, repo.url)
await asyncio.sleep(1)
# delete apps removed from the list
unseen_repos = set(repos.keys()) - set(data.keys())
for repo_name in unseen_repos:
repo = repos[repo_name]
# delete scheduled jobs first
task_logger.info(
f"Application {repo_name} has been removed from the app list, start by removing its scheduled job if there are any..."
)
for job in Job.select().where(Job.url_or_path == repo.url,
Job.state == "scheduled"):
await api_stop_job(None, job.id) # not sure this is going to work
job_id = job.id
task_logger.info(
f"Delete scheduled job {job.name} #{job.id} for application {repo_name} because the application is being deleted."
)
data = model_to_dict(job)
job.delete_instance()
await broadcast({
"action": "delete_job",
"data": data,
}, ["jobs", f"job-{job_id}", f"app-jobs-{job.url_or_path}"])
task_logger.info(
f"Delete application {repo_name} because it has been removed from the apps list."
)
data = model_to_dict(repo)
repo.delete_instance()
await broadcast({
"action": "delete_app",
"data": data,
}, "apps")