def test_update_state_existing_task(self):
self.session.koji_mock.getTaskInfo = Mock(return_value=rnv_task)
self.session.koji_mock.getTaskChildren = Mock(
return_value=rnv_subtasks)
package = self.prepare_packages('rnv')[0]
self.prepare_build('rnv', False)
running_build = self.prepare_build('rnv')
running_build.task_id = rnv_task['id']
koji_task = KojiTask(task_id=rnv_subtasks[0]['id'],
arch='noarch',
state=koji.TASK_STATES['OPEN'],
started=datetime.fromtimestamp(123),
build_id=running_build.id)
self.db.add(koji_task)
self.db.commit()
self.assertEqual('failing', package.state_string)
with patch('koschei.backend.dispatch_event') as event:
backend.update_build_state(self.session, running_build, 'CLOSED')
self.session.koji_mock.getTaskInfo.assert_called_once_with(
rnv_task['id'])
self.session.koji_mock.getTaskChildren.assert_called_once_with(
rnv_task['id'],
request=True,
)
self.assertEqual('ok', package.state_string)
event.assert_called_once_with('package_state_change',
session=self.session,
package=package,
prev_state='failing',
new_state='ok')
self.assertCountEqual([(x['id'], ) for x in rnv_subtasks],
self.db.query(KojiTask.task_id))
def add_task(self, build, arch, started, finished):
koji_task = KojiTask(
task_id=7541,
arch=arch,
state=1,
started=datetime.fromtimestamp(started),
finished=(datetime.fromtimestamp(finished) if finished else None),
build_id=build.id)
self.db.add(koji_task)
self.db.commit()
def prepare_build(self,
package,
state=None,
repo_id=None,
resolved=True,
arches=(),
task_id=None,
started=None,
untagged=False,
epoch=None,
version='1',
release='1.fc25',
real=False):
states = {
True: Build.COMPLETE,
False: Build.FAILED,
None: Build.RUNNING,
'complete': Build.COMPLETE,
'failed': Build.FAILED,
'running': Build.RUNNING,
}
if isinstance(state, (bool, str)):
state = states[state]
if isinstance(package, str):
found = self.db.query(Package).\
filter_by(name=package, collection_id=self.collection.id).first()
package = found or self.prepare_package(package)
build = Build(package=package,
state=state,
repo_id=repo_id
or (1 if state != Build.RUNNING else None),
version=version,
release=release,
untagged=untagged,
real=real,
task_id=task_id or self.task_id_counter,
started=started
or datetime.fromtimestamp(self.task_id_counter),
deps_resolved=resolved)
if not task_id:
self.task_id_counter += 1
self.db.add(build)
self.db.commit()
for arch in arches:
koji_task = KojiTask(task_id=7541,
arch=arch,
state=1,
started=datetime.fromtimestamp(123),
build_id=build.id)
self.db.add(koji_task)
self.db.commit()
return build
def prepare_task(self,
build,
state=1,
arch='x86_64',
started=None,
task_id=None):
if not task_id:
task_id = self.task_id_counter
self.task_id_counter += 1
task = KojiTask(
build=build,
state=state,
arch=arch,
task_id=task_id,
started=started or datetime.fromtimestamp(task_id),
)
self.db.add(task)
self.db.commit()
return task
def sync_tasks(session, collection, builds, real=False):
"""
Synchronizes task and subtask data from Koji.
Sets properties on build objects passed in and return KojiTask objects.
Uses koji_session passed as argument.
Returns map of build to list of tasks
"""
if not builds:
return
koji_session = (session.secondary_koji_for(collection)
if real else session.koji('primary'))
call = itercall(koji_session, builds,
lambda k, b: k.getTaskInfo(b.task_id))
valid_builds = []
for build, task_info in zip(builds, call):
if not task_info:
continue
build.started = datetime.fromtimestamp(task_info['create_ts'])
if task_info.get('completion_ts'):
build.finished = datetime.fromtimestamp(task_info['completion_ts'])
elif build.state != Build.RUNNING:
# When fedmsg delivery is fast, the time is not set yet
build.finished = datetime.now()
valid_builds.append(build)
call = itercall(koji_session, valid_builds,
lambda k, b: k.getTaskChildren(b.task_id, request=True))
build_tasks = {}
for build, subtasks in zip(valid_builds, call):
tasks = []
build_arch_tasks = [
task for task in subtasks if task['method'] == 'buildArch'
]
for task in build_arch_tasks:
set_build_repo_id(session, build, task, collection.secondary_mode)
db_task = KojiTask(task_id=task['id'])
db_task.build_id = build.id
db_task.state = task['state']
db_task.arch = task['arch']
db_task.started = datetime.fromtimestamp(task['create_ts'])
if task.get('completion_ts'):
db_task.finished = datetime.fromtimestamp(
task['completion_ts'])
tasks.append(db_task)
build_tasks[build] = tasks
return build_tasks
def test_update_state_existing_task(self):
collection = self.prepare_collection('f29')
package = self.prepare_package('rnv', collection=collection)
self.prepare_build(package, 'failed')
build = self.prepare_build(package, 'running', task_id=9107738)
koji_task = KojiTask(
task_id=9107739,
arch='armhfp',
state=koji.TASK_STATES['OPEN'],
started=datetime.fromtimestamp(123),
build=build,
)
self.db.add(koji_task)
self.db.commit()
with patch('koschei.backend.dispatch_event') as event:
backend.update_build_state(self.session, build, 'CLOSED')
self.assertEqual('complete', build.state_string)
self.assertEqual('ok', package.state_string)
self.assertEqual(0, package.build_priority)
event.assert_called_once_with(
'package_state_change',
session=self.session,
package=package,
prev_state='failing',
new_state='ok',
)
# ordered by arch
tasks = build.build_arch_tasks
self.assertEqual(3, len(tasks))
self.assertEqual(9107739, tasks[0].task_id)
self.assertEqual(koji.TASK_STATES['CLOSED'], tasks[0].state)
self.assertEqual('armhfp', tasks[0].arch)
self.assertEqual(9107741, tasks[1].task_id)
self.assertEqual(koji.TASK_STATES['CLOSED'], tasks[1].state)
self.assertEqual('i386', tasks[1].arch)
self.assertEqual(9107740, tasks[2].task_id)
self.assertEqual(koji.TASK_STATES['CLOSED'], tasks[2].state)
self.assertEqual('x86_64', tasks[2].arch)
def upgrade():
raise NotImplementedError()
s = Session(bind=op.get_bind())
k = create_koji_session(anonymous=True)
step = 100
off = 0
proc = 0
while True:
k.multicall = True
builds = s.query(Build).filter(Build.state.in_([Build.COMPLETE, Build.FAILED]))\
.order_by(Build.id)[off:off + step]
if not builds:
break
for build in builds:
k.getTaskInfo(build.task_id) # for finished
k.getTaskChildren(build.task_id, request=True) #for repo_id and koji_tasks
result = k.multiCall()
assert len(result) == len(builds) * 2
for build, [build_task], [subtasks] in zip(builds, result[::2], result[1::2]):
if build_task['completion_time']:
build.finished = parse_koji_time(build_task['completion_time'])
build_arch_tasks = [task for task in subtasks if task['method'] == 'buildArch']
for task in build_arch_tasks:
try:
build.repo_id = task['request'][4]['repo_id']
except KeyError:
pass
if not s.query(KojiTask).filter_by(task_id=task['id']).count():
db_task = KojiTask(build_id=build.id, task_id=task['id'],
state=task['state'], started=task['create_time'],
finished=task['completion_time'], arch=task['arch'])
s.add(db_task)
proc += len(builds)
s.flush()
off += step
print "processed {} builds".format(proc)
def sync_tasks(session, collection, builds, real=False):
"""
Synchronizes task and subtask data from Koji.
Sets properties on build objects passed in and return KojiTask objects.
Uses koji_session passed as argument.
Returns map of build to list of tasks
"""
if not builds:
return
koji_session = (session.secondary_koji_for(collection) if real
else session.koji('primary'))
call = itercall(koji_session, builds, lambda k, b: k.getTaskInfo(b.task_id))
valid_builds = []
for build, task_info in zip(builds, call):
if not task_info:
continue
build.started = datetime.fromtimestamp(task_info['create_ts'])
if task_info.get('completion_ts'):
build.finished = datetime.fromtimestamp(task_info['completion_ts'])
elif build.state != Build.RUNNING:
# When fedmsg delivery is fast, the time is not set yet
build.finished = datetime.now()
valid_builds.append(build)
call = itercall(koji_session, valid_builds,
lambda k, b: k.getTaskChildren(b.task_id, request=True))
build_tasks = {}
for build, subtasks in zip(valid_builds, call):
tasks = []
build_arch_tasks = [task for task in subtasks
if task['method'] == 'buildArch']
for task in build_arch_tasks:
set_build_repo_id(session, build, task, collection.secondary_mode)
db_task = KojiTask(task_id=task['id'])
db_task.build_id = build.id
db_task.state = task['state']
db_task.arch = task['arch']
db_task.started = datetime.fromtimestamp(task['create_ts'])
if task.get('completion_ts'):
db_task.finished = datetime.fromtimestamp(task['completion_ts'])
tasks.append(db_task)
build_tasks[build] = tasks
return build_tasks
def test_copy_collection(self):
now = datetime.now()
source = self.collection
_, _, maven1 = self.prepare_packages('rnv', 'eclipse', 'maven')
self.prepare_build('rnv')
self.prepare_build('eclipse')
# the next build is old and shouldn't be copied
self.prepare_build('maven', state=True, started='2016-01-01')
old_build1 = self.prepare_build(
'maven',
state=True,
started=datetime.fromtimestamp(time.time() - 10))
new_build1 = self.prepare_build('maven', started=now)
copy = Collection(
name='copy',
display_name='copy',
target='a',
build_tag='b',
dest_tag='c',
)
self.db.add(copy)
prev_dep = Dependency(name='foo',
version='1',
release='1',
arch='x86_64')
curr_dep = Dependency(name='foo',
version='1',
release='2',
arch='x86_64')
change1 = AppliedChange(
build_id=old_build1.id,
prev_dep=prev_dep,
curr_dep=curr_dep,
)
self.db.add(change1)
task1 = KojiTask(
build_id=new_build1.id,
task_id=new_build1.task_id,
state=1,
arch='x86_64',
started=new_build1.started,
)
self.db.add(task1)
rchange1 = ResolutionChange(
package_id=maven1.id,
resolved=False,
timestamp=now,
)
self.db.add(rchange1)
self.db.flush()
problem1 = ResolutionProblem(
resolution_id=rchange1.id,
problem="It's broken",
)
self.db.add(problem1)
self.db.commit()
data.copy_collection(self.session, source, copy)
self.db.commit()
maven2 = self.db.query(Package).filter_by(collection=copy,
name='maven').first()
self.assertIsNotNone(maven2)
self.assertNotEqual(maven1.id, maven2.id)
self.assertEqual(source.id, maven1.collection_id)
self.assertEqual(copy.id, maven2.collection_id)
self.assertEqual('maven', maven2.name)
self.assertEqual(2, len(maven2.all_builds))
new_build2, old_build2 = maven2.all_builds
self.assertNotEqual(new_build1.id, new_build2.id)
self.assertNotEqual(old_build1.id, old_build2.id)
self.assertEqual(new_build1.id, maven1.last_build_id)
self.assertEqual(old_build1.id, maven1.last_complete_build_id)
self.assertEqual(new_build2.id, maven2.last_build_id)
self.assertEqual(old_build2.id, maven2.last_complete_build_id)
self.assertEqual(1, new_build2.build_arch_tasks[0].state)
self.assertEqual(1, len(old_build2.dependency_changes))
change2 = old_build2.dependency_changes[0]
self.assertEqual('2', change2.curr_dep.release)
rchange2 = self.db.query(ResolutionChange).filter_by(
package_id=maven2.id).one()
self.assertEqual("It's broken", rchange2.problems[0].problem)