class BuildResource(models.Model):
key = models.CharField(
max_length=250, editable=False,
primary_key=True) # until django supports multi filed primary keys
build = models.ForeignKey(Build, on_delete=models.CASCADE)
name = models.CharField(max_length=50)
resource = models.ForeignKey(Resource, on_delete=models.PROTECT)
blueprint = models.CharField(max_length=BLUEPRINT_NAME_LENGTH)
config_values = MapField(blank=True)
quantity = models.IntegerField(default=1)
auto_run = models.BooleanField(default=False)
auto_provision = models.BooleanField(default=True)
interface_map = MapField(blank=True)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
self.key = '{0}_{1}_{2}'.format(self.build.key, self.name,
self.resource.name)
errors = {}
if errors:
raise ValidationError(errors)
def __str__(self):
return 'BuildResource from "{0}" for "{1}" named "{2}"'.format(
self.build.name, self.resource.name, self.name)
class Meta:
unique_together = ('build', 'name')
class Build(models.Model):
"""
This is a type of Build that can be done
"""
key = models.CharField(
max_length=160, editable=False,
primary_key=True) # until djanog supports multi filed primary keys
name = models.CharField(max_length=100)
project = models.ForeignKey(Project, on_delete=models.CASCADE)
dependancies = models.ManyToManyField(Package,
through='BuildDependancy',
help_text='')
resources = models.ManyToManyField(Resource,
through='BuildResource',
help_text='')
network_map = MapField(blank=True)
manual = models.BooleanField()
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return Build.objects.filter(project=project)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
self.key = '{0}_{1}'.format(self.project.name, self.name)
errors = {}
if not name_regex.match(self.name):
errors['name'] = 'Invalid'
for name in self.network_map.keys():
if not name_regex.match(name):
errors['network_map'] = 'Invalid network name "{0}"'.format(
name)
if errors:
raise ValidationError(errors)
def __str__(self):
return 'Build "{0}" of "{1}"'.format(self.name, self.project.name)
class Meta:
unique_together = ('name', 'project')
class StaticResource( Resource ):
"""
StaticResource
"""
sites = models.ManyToManyField( Site )
group_name = models.CharField( max_length=50 )
interface_map = MapField( blank=True )
def available( self, site, quantity, interface_map ):
if site not in self.sites:
return False
if self.statieresourceinstance_set.filter( buildjobresourceinstance__buildjob__isnull=True, site=site ).order_by( 'pk' ).iterator().count() >= quantity:
return False
for name in interface_map.keys():
try: # we only care if the interfaces named in the config match the resource, extra interfaces on the resource are fine
if interface_map[ name ][ 'network' ] != self.interface_map[ name ][ 'network' ].name:
return False
except KeyError:
return False
return True
def allocate( self, site, buildjob, buildresource, interface_map ): # TODO: do this!
if site not in self.sites:
raise ValueError( 'site "{0}" not in list of sites for this resource'.format( site ) )
# initial stab at it
try:
static_resource_instance = random.choice( self.statieresourceinstance_set.filter( buildjobresourceinstance__buildjob__isnull=True, site=site ) )
except IndexError:
raise ValueError( 'no instances aviable' )
BuildJobResourceInstance = apps.get_model( 'Processor', 'BuildJobResourceInstance' )
buildjob_resource = BuildJobResourceInstance()
buildjob_resource.buildjob = buildjob
buildjob_resource.resource_instance = static_resource_instance
buildjob_resource.name = buildresource.name
buildjob_resource.blueprint = buildresource.blueprint
buildjob_resource.config_values = buildresource.config_values
buildjob_resource.auto_run = buildresource.auto_run
buildjob_resource.auto_provision = buildresource.auto_provision
buildjob_resource.full_clean()
buildjob_resource.save()
buildjob_resource.allocate()
# TODO: cleanup too?
@cinp.check_auth()
@staticmethod
def checkAuth( user, verb, id_list, action=None ):
return True
def __str__( self ):
return 'StaticResource "{0}"'.format( self.name )
class DynamicResourceInstance( ResourceInstance ):
"""
DynamicResourceInstance
"""
dynamic_resource = models.ForeignKey( DynamicResource, on_delete=models.PROTECT ) # this is protected so we don't leave VMs laying arround
contractor_foundation_id = models.CharField( unique=True, max_length=100, blank=True, null=True ) # should match foundation locator
interface_map = MapField()
@property
def resource( self ):
return self.dynamic_resource
def allocate( self, blueprint, config_values, hostname ):
interface_map = self.interface_map
for interface in interface_map.keys():
if 'network_id' not in interface_map[ interface ]:
interface_map[ interface ][ 'network_id' ] = interface_map[ interface ][ 'network' ].contractor_network_id
interface_map[ interface ][ 'address_block_id' ] = interface_map[ interface ][ 'network' ].contractor_addressblock_id
contractor = getContractor()
self.contractor_foundation_id, self.contractor_structure_id = contractor.allocateDynamicResource( self.site.name, self.dynamic_resource.dynamicresourcesite_set.get( site=self.site ).complex_id, blueprint, config_values, interface_map, hostname )
self.full_clean()
self.save()
def build( self ):
contractor = getContractor()
if self.buildjobresourceinstance.auto_provision:
contractor.buildDynamicResource( self.contractor_foundation_id, self.contractor_structure_id )
contractor.registerWebHook( self.buildjobresourceinstance, True, structure_id=self.contractor_structure_id )
else:
contractor.buildDynamicResource( self.contractor_foundation_id )
contractor.registerWebHook( self.buildjobresourceinstance, True, foundation_id=self.contractor_foundation_id )
def release( self ):
contractor = getContractor()
if contractor.releaseDynamicResource( self.contractor_foundation_id, self.contractor_structure_id ):
contractor.registerWebHook( self.buildjobresourceinstance, False, foundation_id=self.contractor_foundation_id )
def cleanup( self ):
contractor = getContractor()
contractor.deleteDynamicResource( self.contractor_foundation_id, self.contractor_structure_id )
self.delete()
@cinp.check_auth()
@staticmethod
def checkAuth( user, verb, id_list, action=None ):
return True
def __str__( self ):
return 'DynamicResourceInstance for "{0}" contractor id: "{1}"'.format( self.dynamic_resource, self.contractor_structure_id )
class Promotion(models.Model):
status = models.ManyToManyField(Build,
through='PromotionBuild',
help_text='')
result_map = MapField(blank=True)
commit = models.ForeignKey(Commit, on_delete=models.PROTECT)
tag = models.CharField(max_length=TAG_NAME_LENGTH)
done_at = models.DateTimeField(blank=True, null=True)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
def signalComplete(self, build, success):
promotion_build = self.promotionbuild_set.get(build=build)
promotion_build.status = 'done'
promotion_build.success = success
promotion_build.full_clean()
promotion_build.save()
def setResults(self, name, results):
self.result_map[name] = results
self.full_clean()
self.save()
def getResults(self):
return self.result_map
@cinp.list_filter(name='in_process', paramater_type_list=[])
@staticmethod
def filter_in_process():
return Promotion.objects.filter(done_at__isnull=True)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return cinp.basic_auth_check(user, verb, Promotion)
class Meta:
default_permissions = ()
def __str__(self):
return 'Promotion for package commit "{0}" tag "{1}"'.format(
self.commit, self.tag)
class Commit(models.Model):
"""
A Single Commit of a Project
"""
project = models.ForeignKey(Project, on_delete=models.CASCADE)
branch = models.CharField(max_length=BRANCH_NAME_LENGTH)
commit = models.CharField(max_length=45)
version = models.CharField(max_length=50, blank=True, null=True)
build_name = models.CharField(max_length=BUILD_NAME_LENGTH,
blank=True,
null=True)
lint_results = MapField(blank=True)
test_results = MapField(blank=True)
build_results = MapField(blank=True)
doc_results = MapField(blank=True)
test_at = models.DateTimeField(editable=False, blank=True, null=True)
build_at = models.DateTimeField(editable=False, blank=True, null=True)
doc_at = models.DateTimeField(editable=False, blank=True, null=True)
done_at = models.DateTimeField(editable=False, blank=True, null=True)
package_file_map = MapField(blank=True)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@property
def state(self):
if self.done_at: # sometimes the rest of the steps are skipped
return 'done'
if self.doc_at and self.build_at and self.test_at:
return 'doced'
if self.build_at and self.test_at:
return 'built'
if self.test_at:
return 'tested'
return 'new'
@property
def summary(self):
result = {'test': {}, 'lint': {}}
overall_complete = True
overall_success = True
score_list = []
complete = True
success = True
for (name, value) in self.lint_results.items():
if value.get('score', None) is not None:
score_list.append(value['score'])
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
if score_list:
result['lint']['score'] = sum(score_list) / len(score_list)
else:
result['lint']['score'] = None
if not complete:
result['lint']['status'] = 'Incomplete'
elif success:
result['lint']['status'] = 'Success'
else:
result['lint']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
score_list = []
complete = True
success = True
for (name, value) in self.test_results.items():
if value.get('score', None) is not None:
try:
score_list.append(float(value['score']))
except ValueError:
score_list.append(0.0)
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
if score_list:
result['test']['score'] = sum(score_list) / len(score_list)
else:
result['test']['score'] = None
if not complete:
result['test']['status'] = 'Incomplete'
elif success:
result['test']['status'] = 'Success'
else:
result['test']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
complete = True
success = True
for target in self.build_results:
for (name, value) in self.build_results[target].items():
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
result['build'] = {}
if not complete:
result['build']['status'] = 'Incomplete'
elif success:
result['build']['status'] = 'Success'
else:
result['build']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
if self.branch == self.project.release_branch:
complete = True
success = True
for (name, value) in self.doc_results.items():
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
result['doc'] = {}
if not complete:
result['doc']['status'] = 'Incomplete'
elif success:
result['doc']['status'] = 'Success'
else:
result['doc']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
if not overall_success:
result['status'] = 'Failed'
elif not overall_complete:
result['status'] = 'Incomplete'
else:
result['status'] = 'Success'
return result
@property
def results(self): # for now in Markdown format
result = {}
wrk = {}
for name in self.lint_results:
tmp = self.lint_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'],
tmp.get('score', None))
if wrk:
result['lint'] = wrk
wrk = {}
for name in self.test_results:
tmp = self.test_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'],
tmp.get('score', None))
if wrk:
result['test'] = wrk
wrk = {}
for target in self.build_results:
wrk[target] = {}
tmp = self.build_results[target]
for name in tmp:
if tmp[name].get('results', None) is not None:
wrk[target][name] = (tmp[name].get('success', False),
tmp[name]['results'], None)
if wrk:
result['build'] = wrk
wrk = {}
for name in self.doc_results:
tmp = self.doc_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'], None)
if wrk:
result['doc'] = wrk
if not result:
return None
else:
return result
@property
def uiURL(self):
# self.commit not encded in curent UI
return '{0}/#project/{1}'.format(
settings.MCP_UI_HOST,
base64.b64encode('/api/v1/Project/Project:{0}:'.format(
self.project.name).encode()).decode())
def setResults(self, target, name, results):
if target not in ('lint', 'test', 'rpm', 'dpkg', 'respkg', 'resource',
'doc'):
return
if target == 'lint':
self.lint_results[name]['results'] = results
elif target == 'test':
self.test_results[name]['results'] = results
elif target == 'doc':
self.doc_results[name]['results'] = results
else:
self.build_results[target][name]['results'] = results
self.full_clean()
self.save()
def getResults(self, target):
if target == 'lint':
return dict([(i, self.lint_results[i].get('results', None))
for i in self.lint_results])
elif target == 'test':
return dict([(i, self.test_results[i].get('results', None))
for i in self.test_results])
elif target == 'doc':
return dict([(i, self.doc_results[i].get('results', None))
for i in self.doc_results])
elif target in ('rpm', 'dpkg', 'respkg', 'resource'):
return dict([(i,
self.build_results[target][i].get('results', None))
for i in self.build_results[target]])
return {}
def setScore(self, target, name, score):
if target not in ('lint', 'test'):
return
if target == 'lint':
self.lint_results[name]['score'] = score
elif target == 'test':
self.test_results[name]['score'] = score
self.full_clean()
self.save()
def getScore(self, target):
if target == 'lint':
return dict([(i, self.lint_results[i].get('score', None))
for i in self.lint_results])
elif target == 'test':
return dict([(i, self.test_results[i].get('score', None))
for i in self.test_results])
return {}
def signalComplete(self, target, name, success):
if target not in ('test', 'rpm', 'dpkg', 'respkg', 'resource', 'doc'):
return
if target == 'test':
self.lint_results[name]['status'] = 'done'
self.lint_results[name]['success'] = success
self.test_results[name]['status'] = 'done'
self.test_results[name]['success'] = success
elif target == 'doc':
self.doc_results[name]['status'] = 'done'
self.doc_results[name]['success'] = success
else:
self.build_results[target][name]['status'] = 'done'
self.build_results[target][name]['success'] = success
self.full_clean()
self.save()
def postInProcess(self):
if self.project.type == 'GitProject':
return
if not self.branch.startswith('_PR') and not self.branch.startswith(
'_MR'
): # TODO: there should be a better way to do this, apears one more time
return
self.project.scm.postCommitStatus(self.commit,
self.branch,
'pending',
target_url=self.uiURL)
def postResults(self):
if self.project.type == 'GitProject':
return
comment = self.results
scm = self.project.scm
try:
prev_comment = Commit.objects.filter(
project=self.project,
branch=self.branch,
done_at__lt=self.done_at).order_by('-done_at')[0].results
except (Commit.DoesNotExist, IndexError):
prev_comment = None
if prev_comment is None and comment is None:
comment = '**Nothing To Do**'
elif prev_comment is not None and comment is None:
comment = '**Last Commit Had Results, This One Did Not**'
else:
comment = _markdownResults(comment, prev_comment)
scm.postCommitComment(self.commit, comment)
if self.branch.startswith('_PR') or self.branch.startswith('_MR'):
summary = self.summary
if summary['status'] == 'Success':
scm.postCommitStatus(self.commit,
self.branch,
'success',
'Passed',
target_url=self.uiURL,
coverage=summary['test']['score'])
elif summary['status'] == 'Failed':
scm.postCommitStatus(self.commit,
self.branch,
'failure',
'Failure',
target_url=self.uiURL)
else:
scm.postCommitStatus(self.commit,
self.branch,
'error',
'Bad State "{0}"'.format(
summary['status']),
target_url=self.uiURL)
# gh.setOwner()
number = int(self.branch[3:])
scm.postMergeComment(number, _commitSumary2Str(self.summary))
def tagVersion(self):
if self.version is None:
return
self.project.internal_git.tag(self.version,
_commitSumary2Str(self.summary))
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return Commit.objects.filter(project=project).order_by('-created')
@cinp.list_filter(name='in_process', paramater_type_list=[])
@staticmethod
def filter_in_process():
return Commit.objects.filter(done_at__isnull=True)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
errors = {}
for key, value in self.package_file_map.items():
if not package_filename_regex.match(key):
errors['package_file_map'] = 'file name "{0}" invalid'.format(
key)
break
if not isinstance(value,
str) and not packagefile_regex.match(value):
errors[
'package_file_map'] = 'file uri invalid for "{0}"'.format(
key)
break
if errors:
raise ValidationError(errors)
def __str__(self):
return 'Commit "{0}" on branch "{1}" of project "{2}"'.format(
self.commit, self.branch, self.project.name)
class QueueItem(models.Model):
"""
QueueItem
"""
build = models.ForeignKey(Build, on_delete=models.CASCADE, editable=False)
project = models.ForeignKey(Project,
on_delete=models.CASCADE,
editable=False)
branch = models.CharField(max_length=BRANCH_NAME_LENGTH)
target = models.CharField(max_length=50)
priority = models.IntegerField(
default=50) # higher the value, higer the priority
manual = models.BooleanField(
) # if False, will not auto clean up, and will not block the project from updating/re-scaning for new jobs
user = models.CharField(max_length=150)
resource_status_map = MapField(blank=True)
commit = models.ForeignKey(Commit,
null=True,
blank=True,
on_delete=models.SET_NULL)
promotion = models.ForeignKey(Promotion,
null=True,
blank=True,
on_delete=models.SET_NULL)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
def allocateResources(self):
missing_list = []
buildresource_list = []
network_map = {}
other_ip_count = 0
site = Site.objects.all().order_by(
'?'
)[0] # yeah we might guess and pick the wrong site (without resources), but it will get retried, we should do some site scoring and do that instead
# first allocate the network(s)
for name, item in self.build.network_map.items():
if item['dedicated']:
try:
network_map[name] = Network.objects.filter(
monolithic=True,
size__gte=item['min_addresses'],
buildjob=None,
site=site)[0]
except IndexError:
missing_list.append(
'Network for "{0}" Not Available'.format(name))
else:
for network in Network.objects.filter(
monolithic=False,
size__gte=item['min_addresses'],
site=site):
if network.available(item['min_addresses']):
network_map[name] = network
break
else:
missing_list.append(
'Network for "{0}" Not Available in site "{1}"'.format(
name, site.name))
if missing_list:
return (list(set(missing_list)), None, None, None)
# second allocate the resource(s)
for buildresource in self.build.buildresource_set.all():
quantity = buildresource.quantity
resource = buildresource.resource.subclass
if not resource.available(site, quantity,
buildresource.interface_map):
missing_list.append(
'Resource "{0}" Not Available in site "{1}"'.format(
resource.name, site.name))
buildresource_list.append(buildresource)
for item in buildresource.interface_map.values():
if 'network' in item:
try:
network_map[item['network']]
except KeyError:
missing_list.append(
'Network "{0}" Not Defined in site "{1}"'.format(
item['network'], site.name))
else:
other_ip_count += quantity
# lastly make sure we have the IPs
if other_ip_count:
for network in Network.objects.filter(
monolithic=False, size__gte=other_ip_count,
site=site).exclude(pk__in=network_map.items()):
if network.available(other_ip_count):
network_map['_OTHER_'] = network
break
else:
missing_list.append(
'Other Network Not Available in site "{0}"'.format(
site.name))
if missing_list:
return (list(set(missing_list)), None, None, None)
return (None, buildresource_list, network_map, site)
@staticmethod
def inQueueBuild(build, branch, manual, priority, user, promotion=None):
item = QueueItem()
item.user = user
item.build = build
item.manual = manual
item.project = build.project
item.branch = branch
item.target = build.name
item.priority = priority
item.promotion = promotion
item.full_clean()
item.save()
return item
@staticmethod
def inQueueTarget(project,
branch,
manual,
blueprint,
target,
priority,
user,
commit=None):
try:
builtin_project = Project(pk='_builtin_')
except Project.DoesNotExist:
raise Exception('project "_builtin_" missing')
try:
build = Build.objects.get(project=builtin_project,
buildresource__blueprint=blueprint)
except Build.DoesNotExist:
raise Exception(
'build for _builtin_ project and blueprint "{0}" not found'.
format(blueprint))
item = QueueItem()
item.user = user
item.build = build
item.manual = manual
item.project = project
item.branch = branch
item.target = target
item.priority = priority
item.commit = commit
item.full_clean()
item.save()
return item
@cinp.action(return_type='Integer',
paramater_type_list=[{
'type': '_USER_'
}, {
'type': 'Model',
'model': Build
}, 'String', 'Integer'])
@staticmethod
def queue(user, build, branch=None, priority=100):
if branch is None:
branch = build.project.release_branch
item = QueueItem.inQueueBuild(build, branch, True, priority,
user.username)
return item.pk
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return QueueItem.objects.filter(project=project)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
if not cinp.basic_auth_check(user, verb, QueueItem):
return False
if verb == 'CALL':
if action == 'queue' and user.has_perm('Processor.can_build'):
return True
return False
else:
return True
class Meta:
default_permissions = ()
def __str__(self):
return 'QueueItem for "{0}" of priority "{1}"'.format(
self.build.name, self.priority)
class BuildJobResourceInstance(models.Model):
buildjob = models.ForeignKey(
BuildJob, blank=True, null=True, on_delete=models.PROTECT
) # protected so we don't leave stranded resources
resource_instance = models.OneToOneField(ResourceInstance,
blank=True,
null=True,
on_delete=models.SET_NULL)
blueprint = models.CharField(max_length=BLUEPRINT_NAME_LENGTH)
_config_values = MapField(blank=True)
auto_run = models.BooleanField(default=False)
auto_provision = models.BooleanField(default=True)
cookie = models.CharField(
max_length=36,
default=getCookie) # blank=True, null=True, editable=False ?
# build info
name = models.CharField(max_length=50, blank=True, null=True)
index = models.IntegerField(blank=True, null=True)
state = models.CharField(max_length=9,
default='new',
choices=[(i, i) for i in INSTANCE_STATE_LIST])
message = models.CharField(max_length=200, default='', blank=True)
# results info
success = models.BooleanField(default=False)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@property
def hostname(
self
): # TODO: make sure does not exceed contractor's locator length, and is locator/hostname safe
if self.buildjob is not None:
if self.buildjob.manual:
return 'mcp-manual--{0}-{1}-{2}'.format(
self.buildjob_id, self.name, self.index)
else:
return 'mcp-auto--{0}-{1}-{2}'.format(self.buildjob_id,
self.name, self.index)
else:
return 'mcp-prealloc--{0}'.format(self.pk)
@property
def config_values(self):
result = self._config_values
result.update(base_config_values())
if self.buildjob is not None:
result.update({
'mcp_job_id':
self.buildjob.pk,
'mcp_instance_id':
self.pk,
'mcp_project_name':
self.buildjob.project.name,
'mcp_instance_cookie':
self.cookie,
'mcp_resource_name':
self.name,
'mcp_resource_index':
self.index,
'mcp_store_packages':
self.buildjob.branch == self.buildjob.project.release_branch,
'mcp_git_url':
self.buildjob.project.internal_git_url,
'mcp_git_branch':
self.buildjob.branch,
'mcp_make_target':
self.buildjob.target
})
if self.buildjob.commit is not None:
result.update({
'mcp_commit_version': self.buildjob.commit.version,
'mcp_build_name': self.buildjob.commit.build_name
})
if self.buildjob.promotion is not None:
result.update(
{'mcp_promotion_tag': self.buildjob.promotion.tag})
return result
@config_values.setter
def config_values(self, value):
self._config_values = value
@cinp.action(paramater_type_list=['String'])
def signalBuilt(self, cookie): # called from webhook
if self.cookie != cookie:
return
if self.state not in ('new', 'allocated',
'building'): # allready moved on, don't touch
return
if self.auto_run:
self.state = 'ran'
self.success = True
else:
self.state = 'built'
self.full_clean()
self.save()
@cinp.action(paramater_type_list=['String'])
def signalDestroyed(self, cookie): # called from webhook
if self.cookie != cookie:
return
self.resource_instance.cleanup()
self.resource_instance = None # cleanup will delete the resource_instance
self.state = 'released'
self.full_clean()
self.save()
@cinp.action(paramater_type_list=['String', 'String'])
def setMessage(self, cookie, message):
if self.cookie != cookie:
return
self.message = message[-200:]
self.full_clean()
self.save()
@cinp.action(paramater_type_list=['String'])
def jobRan(self, cookie):
if self.cookie != cookie:
return
if self.state not in ('new', 'allocated', 'building',
'built'): # don't go back to a previous state
return
self.state = 'ran'
self.full_clean()
self.save()
@cinp.action(paramater_type_list=['String', 'Boolean'])
def setSuccess(self, cookie, success):
if self.cookie != cookie:
return
self.success = success
self.full_clean()
self.save()
@cinp.action(paramater_type_list=['String', 'String', 'String'])
def setResults(self, cookie, target, results):
if self.cookie != cookie:
return
if target != self.buildjob.target and not (
self.buildjob.target == 'test' and target in ('test', 'lint')):
return
if self.buildjob.commit:
self.buildjob.commit.setResults(target, self.name, results)
elif self.buildjob.promotion:
self.buildjob.promotion.setResults(self.name, results)
@cinp.action(paramater_type_list=['String', 'String', 'Float'])
def setScore(self, cookie, target, score):
if self.cookie != cookie:
return
if self.buildjob.target != 'test' or target not in ('test', 'lint'):
return
if self.buildjob.commit:
self.buildjob.commit.setScore(target, self.name, score)
@cinp.action(return_type='String',
paramater_type_list=['String', {
'type': 'Map'
}])
def addPackageFiles(self, cookie, package_file_map):
if self.cookie != cookie:
return
self.buildjob.package_file_map.update(package_file_map)
self.buildjob.full_clean()
self.buildjob.save()
@cinp.action(return_type='Map', paramater_type_list=['String'])
def getValueMap(self, cookie):
if self.cookie != cookie:
return
return self.buildjob.value_map
@cinp.action(paramater_type_list=['String', 'Map'])
def updateValueMap(self, cookie, value_map):
if self.cookie != cookie:
return
self.buildjob.value_map.update(value_map)
self.buildjob.full_clean()
self.buildjob.save()
@cinp.action(return_type='Map')
def getHostDetail(self):
"""
Get the Host detail, including info from Contractor, this is a bit expensive, use conservitivally.
"""
result = {
'structure_id': self.resource_instance.contractor_structure_id,
'hostname': self.hostname
}
contractor = getContractor()
try:
config = contractor.getFullConfig(
self.resource_instance.contractor_structure_id)
except NotFound:
config = None
if config is not None:
result['fqdn'] = config['_fqdn']
result['site'] = config['_site']
result['ip_address'] = config['_primary_address']['address']
result['config_uuid'] = config['_structure_config_uuid']
result['blueprint'] = config['_blueprint']
try:
result[
'foundation_id'] = self.resource_instance.contractor_foundation_id
except AttributeError:
pass
return result
def allocate(self):
if self.state != 'new':
raise Exception('Allready allocated')
self.resource_instance.allocate(self.blueprint, self.config_values,
self.hostname)
self.state = 'allocated'
self.full_clean()
self.save()
def updateConfig(self):
if self.state in ('releasing', 'released'):
return
self.resource_instance.updateConfig(self.config_values, self.hostname)
def build(self):
if self.state in ('building', 'built'):
return
if self.state != 'allocated':
raise Exception('Can only build when allocated')
self.resource_instance.build()
self.state = 'building'
self.full_clean()
self.save()
def release(self):
if self.state in ('releasing', 'released'):
return
if self.state == 'new':
self.state = 'released'
self.full_clean()
self.save()
return
elif self.state == 'allocated':
if self.resource_instance is not None:
self.resource_instance.cleanup()
self.resource_instance = None # cleanup will delete the resource_instance
self.state = 'released'
self.full_clean()
self.save()
return
if self.state not in (
'built', 'ran'
): # pretty much the only state that will cause problems is "building", have to wait for the build to finish before cleaning up
raise Exception('Can not release when not built')
if self.resource_instance is None:
self.state = 'released'
self.full_clean()
self.save()
return
self.resource_instance.release()
self.state = 'releasing'
self.full_clean()
self.save()
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return cinp.basic_auth_check(user, verb, BuildJobResourceInstance)
class Meta:
default_permissions = ()
def __str__(self):
return 'BuildJobResourceInstance "{0}" for BuildJob "{1}" Named "{2}"'.format(
self.pk, self.buildjob, self.hostname)
class BuildJob(models.Model):
"""
BuildJob
"""
build = models.ForeignKey(
Build, on_delete=models.PROTECT, editable=False
) # don't delete Builds/projects when things are in flight
project = models.ForeignKey(Project,
on_delete=models.PROTECT,
editable=False)
branch = models.CharField(max_length=BRANCH_NAME_LENGTH)
target = models.CharField(max_length=50)
value_map = MapField(blank=True) # for the job to store work values
network_list = models.ManyToManyField(Network)
resources = models.ManyToManyField(ResourceInstance,
through='BuildJobResourceInstance')
built_at = models.DateTimeField(editable=False, blank=True, null=True)
ran_at = models.DateTimeField(editable=False, blank=True, null=True)
reported_at = models.DateTimeField(editable=False, blank=True, null=True)
acknowledged_at = models.DateTimeField(editable=False,
blank=True,
null=True)
released_at = models.DateTimeField(editable=False, blank=True, null=True)
manual = models.BooleanField()
user = models.CharField(max_length=150)
commit = models.ForeignKey(Commit,
null=True,
blank=True,
on_delete=models.SET_NULL)
promotion = models.ForeignKey(Promotion,
null=True,
blank=True,
on_delete=models.SET_NULL)
package_file_map = MapField(blank=True)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@property
def state(self):
if self.released_at and self.acknowledged_at and self.reported_at and self.ran_at and self.built_at:
return 'released'
if self.acknowledged_at and self.reported_at and self.ran_at and self.built_at:
return 'acknowledged'
if self.reported_at and self.ran_at and self.built_at:
return 'reported'
if self.ran_at and self.built_at:
return 'ran'
if self.built_at:
return 'built'
return 'new'
# some jobs have more than one instances, in this case, if a instance hasn't
# report a status we will assume it has success, due to the fact that many
# of the sub instances will never report
@property
def succeeded(self):
if self.ran_at is None:
return None
result = True
for instance in self.buildjobresourceinstance_set.all():
result &= instance.success
return result
@property
def instance_summary(self):
if self.commit is not None:
if self.target == 'test':
lint_map = self.commit.getResults('lint')
test_map = self.commit.getResults('test')
results_map = {}
for name in lint_map:
results_map[name] = 'lint:\n{0}\n\ntest:\n{1}'.format(
lint_map[name] if lint_map[name] is not None else '',
test_map[name] if test_map[name] is not None else '')
else:
results_map = self.commit.getResults(self.target)
score_map = self.commit.getScore(self.target)
if self.promotion is not None:
results_map = self.promotion.getResults()
score_map = {}
else:
results_map = {}
score_map = {}
result = {}
for instance in self.buildjobresourceinstance_set.all():
item = {
'id': instance.pk,
'success': instance.success,
'state': instance.state,
'message': instance.message
}
try:
item['results'] = results_map[instance.name]
except KeyError:
pass
try:
item['score'] = score_map[instance.name]
except KeyError:
pass
try:
result[instance.name][instance.index] = item
except KeyError:
result[instance.name] = {instance.index: item}
return result
@cinp.action(paramater_type_list=[{'type': '_USER_'}])
def jobRan(self, user):
if self.ran_at is not None: # been done, don't touch
return
if not self.built_at:
self.built_at = datetime.now(timezone.utc)
self.ran_at = datetime.now(timezone.utc)
self.full_clean()
self.save()
@cinp.action(paramater_type_list=[{'type': '_USER_'}])
def acknowledge(self, user):
if self.acknowledged_at is not None: # been done, don't touch
return
if self.reported_at is None:
raise ValidationError('Can not Acknoledge un-reported jobs')
self.acknowledged_at = datetime.now(timezone.utc)
self.full_clean()
self.save()
@cinp.action(return_type='Map', paramater_type_list=['String'])
def getInstanceState(self, name=None):
result = {}
if name is not None:
try:
instance = self.buildjobresourceinstance_set.get(name=name)
result[instance.index] = instance.state
except BuildJobResourceInstance.DoesNotExist:
pass
else:
for instance in self.buildjobresourceinstance_set.all():
try:
result[instance.name][instance.index] = instance.state
except KeyError:
result[instance.name] = {instance.index: instance.state}
return result
@cinp.action(return_type='Map', paramater_type_list=['String'])
def getInstanceStructureId(self, name=None):
result = {}
if name is not None:
try:
instance = self.buildjobresourceinstance_set.get(name=name)
result[
instance.
index] = instance.resource_instance.contractor_structure_id
except BuildJobResourceInstance.DoesNotExist:
pass
else:
for instance in self.buildjobresourceinstance_set.all():
try:
result[instance.name][
instance.
index] = instance.resource_instance.contractor_structure_id
except KeyError:
result[instance.name] = {
instance.index:
instance.resource_instance.contractor_structure_id
}
return result
def buildResources(self):
for instance in self.buildjobresourceinstance_set.all():
instance.build()
def releaseResources(self):
for instance in self.buildjobresourceinstance_set.all():
instance.release()
@property
def instances_built(self):
for instance in self.buildjobresourceinstance_set.all():
if instance.state not in ('built', 'ran', 'releasing', 'released'):
return False
return True
@property
def instances_ran(self):
for instance in self.buildjobresourceinstance_set.all():
if instance.state not in ('ran', 'releasing', 'released'):
return False
return True
@property
def instances_released(self):
for instance in self.buildjobresourceinstance_set.all():
if instance.state != 'released':
return False
return True
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return BuildJob.objects.filter(project=project)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
if not cinp.basic_auth_check(user, verb, BuildJob):
return False
if verb == 'CALL':
if action in ('getInstanceState', 'getInstanceStructureId'):
return True
if action == 'jobRan' and user.has_perm('Processor.can_ran'):
return True
if action == 'acknowledge' and user.has_perm('Processor.can_ack'):
return True
return False
else:
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
errors = {}
for key, value in self.package_file_map.items():
if not package_filename_regex.match(key):
errors['package_file_map'] = 'file name "{0}" invalid'.format(
key)
break
if not isinstance(value,
str) and not packagefile_regex.match(value):
errors[
'package_file_map'] = 'file uri invalid for "{0}"'.format(
key)
break
if errors:
raise ValidationError(errors)
def delete(self, *args, **kwargs):
for instance in self.buildjobresourceinstance_set.all():
instance.delete()
super().delete(*args, **kwargs)
class Meta:
default_permissions = ()
permissions = (('can_build', 'Can queue builds'),
('can_ran', 'Can Flag a Build Resource as ran'),
('can_ack', 'Can Acknoledge a failed Build Resource'))
def __str__(self):
return 'BuildJob "{0}" for build "{1}"'.format(self.pk,
self.build.name)
class QueueItem(models.Model):
"""
QueueItem
"""
build = models.ForeignKey(Build, on_delete=models.CASCADE, editable=False)
project = models.ForeignKey(Project,
on_delete=models.CASCADE,
editable=False)
branch = models.CharField(max_length=50)
target = models.CharField(max_length=50)
priority = models.IntegerField(
default=50) # higher the value, higer the priority
manual = models.BooleanField(
) # if False, will not auto clean up, and will not block the project from updating/re-scaning for new jobs
user = models.ForeignKey(User,
null=True,
blank=True,
on_delete=models.SET_NULL)
resource_status_map = MapField(blank=True)
commit = models.ForeignKey(Commit,
null=True,
blank=True,
on_delete=models.SET_NULL)
promotion = models.ForeignKey(Promotion,
null=True,
blank=True,
on_delete=models.SET_NULL)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
def checkResources(self):
compute = {}
network = {}
total_ips = 0
for buildresource in self.build.buildresource_set.all():
quanity = buildresource.quanity
resource = buildresource.resource.subclass
tmp = resource.available(quanity)
if not tmp:
compute[resource.name] = 'Not Available'
total_ips += quanity
target_network = None
for resource in NetworkResource.objects.all().order_by('-preference'):
if resource.available(total_ips):
target_network = resource
break
if target_network is None:
network['network'] = 'Need: {0}'.format(total_ips)
return (compute, network, target_network)
def allocateResources(
self, job, target_network
): # warning, this dosen't check first, make sure you are sure there are resources available before calling
for buildresource in self.build.buildresource_set.all():
name = buildresource.name
quanity = buildresource.quanity
resource = buildresource.resource.subclass
resource.allocate(job, name, quanity, target_network)
@staticmethod
def inQueueBuild(build, branch, manual, priority, promotion=None):
item = QueueItem()
item.build = build
item.manual = manual
item.project = build.project
item.branch = branch
item.target = build.name
item.priority = priority
item.promotion = promotion
item.full_clean()
item.save()
return item
@staticmethod
def inQueueTarget(project,
branch,
manual,
distro,
target,
priority,
commit=None):
try:
build = Build.objects.get(project_id='_builtin_', name=distro)
except Build.DoesNotExist:
raise Exception('distro "{0}" not set up'.format(distro))
item = QueueItem()
item.build = build
item.manual = manual
item.project = project
item.branch = branch
item.target = target
item.priority = priority
item.commit = commit
item.full_clean()
item.save()
return item
@cinp.action(return_type='Integer',
paramater_type_list=[{
'type': '_USER_'
}, {
'type': 'Model',
'model': Build
}])
@staticmethod
def queue(user, build):
# if not user.has_perm( 'Processor.can_build' ):
# raise PermissionDenied()
item = QueueItem.inQueueBuild(build, 'master', True, 100)
return item.pk
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return QueueItem.objects.filter(project=project)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
errors = {}
if errors:
raise ValidationError(errors)
def __str__(self):
return 'QueueItem for "{0}" of priority "{1}"'.format(
self.build.name, self.priority)
class BuildJob(models.Model):
"""
BuildJob
"""
build = models.ForeignKey(
Build, on_delete=models.PROTECT, editable=False
) # don't delete Builds/projects when things are in flight
project = models.ForeignKey(Project,
on_delete=models.PROTECT,
editable=False)
branch = models.CharField(max_length=50)
target = models.CharField(max_length=50)
value_map = MapField(default={},
blank=True) # for the job to store work values
built_at = models.DateTimeField(editable=False, blank=True, null=True)
ran_at = models.DateTimeField(editable=False, blank=True, null=True)
reported_at = models.DateTimeField(editable=False, blank=True, null=True)
acknowledged_at = models.DateTimeField(editable=False,
blank=True,
null=True)
released_at = models.DateTimeField(editable=False, blank=True, null=True)
manual = models.BooleanField()
user = models.ForeignKey(User,
null=True,
blank=True,
on_delete=models.SET_NULL)
commit = models.ForeignKey(Commit,
null=True,
blank=True,
on_delete=models.SET_NULL)
promotion = models.ForeignKey(Promotion,
null=True,
blank=True,
on_delete=models.SET_NULL)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@property
def state(self):
if self.released_at and self.acknowledged_at and self.reported_at and self.ran_at and self.built_at:
return 'released'
if self.acknowledged_at and self.reported_at and self.ran_at and self.built_at:
return 'acknowledged'
if self.reported_at and self.ran_at and self.built_at:
return 'reported'
if self.ran_at and self.built_at:
return 'ran'
if self.built_at:
return 'built'
return 'new'
# some jobs have more than one instances, in this case, if a instance hasn't
# report a status we will assume it has success, due to the fact that many
# of the sub instances will never report
@property
def suceeded(self):
if self.ran_at is None:
return None
result = True
for instance in self.instance_set.all():
result &= instance.success
return result
@property
def instance_summary(self):
if self.commit is not None:
if self.target == 'test':
lint_map = self.commit.getResults('lint')
test_map = self.commit.getResults('test')
results_map = {}
for name in lint_map:
results_map[name] = 'lint:\n{0}\n\ntest:\n{1}'.format(
lint_map[name] if lint_map[name] is not None else '',
test_map[name] if test_map[name] is not None else '')
else:
results_map = self.commit.getResults(self.target)
score_map = self.commit.getScore(self.target)
else:
results_map = {}
score_map = {}
result = {}
for instance in self.instance_set.all():
item = {
'id': instance.pk,
'success': instance.success,
'state': instance.state,
'message': instance.message
}
try:
item['results'] = results_map[instance.name]
except KeyError:
pass
try:
item['score'] = score_map[instance.name]
except KeyError:
pass
try:
result[instance.name][instance.index] = item
except KeyError:
result[instance.name] = {instance.index: item}
return result
def _jobRan(self):
if self.ran_at is not None: # been done, don't touch
return
if not self.built_at:
self.built_at = datetime.now(timezone.utc)
self.ran_at = datetime.now(timezone.utc)
self.full_clean()
self.save()
@cinp.action(paramater_type_list=[{'type': '_USER_'}])
def jobRan(self, user):
# if not user.has_perm( 'Processor.can_ran' ):
# raise PermissionDenied()
self._jobRan()
@cinp.action(paramater_type_list=[{'type': '_USER_'}])
def acknowledge(self, user):
# if not user.has_perm( 'Processor.can_ack' ):
# raise PermissionDenied()
if self.acknowledged_at is not None: # been done, don't touch
return
if self.reported_at is None:
raise ValidationError('Can not Acknoledge un-reported jobs')
self.acknowledged_at = datetime.now(timezone.utc)
self.full_clean()
self.save()
@cinp.action(return_type='Map', paramater_type_list=['String'])
def getInstanceState(self, name=None):
result = {}
if name is not None:
for instance in self.instance_set.all():
if instance.name != name:
continue
result[instance.index] = instance.state
else:
for instance in self.instance_set.all():
try:
result[instance.name][instance.index] = instance.state
except KeyError:
result[instance.name] = {instance.index: instance.state}
return result
@cinp.action(return_type='Map', paramater_type_list=['String'])
def getInstanceDetail(self, name=None):
result = {}
if name is not None:
for instance in self.instance_set.all():
if instance.name != name:
continue
result[instance.index] = instance.getDetail()
else:
for instance in self.instance_set.all():
try:
result[instance.name][
instance.index] = instance.getDetail()
except KeyError:
result[instance.name] = {
instance.index: instance.getDetail()
}
return result
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return BuildJob.objects.filter(project=project)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
errors = {}
if errors:
raise ValidationError(errors)
def __str__(self):
return 'BuildJob "{0}" for build "{1}"'.format(self.pk,
self.build.name)
class Commit(models.Model):
"""
A Single Commit of a Project
"""
project = models.ForeignKey(Project, on_delete=models.CASCADE)
owner_override = models.CharField(max_length=50, blank=True, null=True)
branch = models.CharField(max_length=50)
commit = models.CharField(max_length=45)
lint_results = MapField(blank=True)
test_results = MapField(blank=True)
build_results = MapField(blank=True)
doc_results = MapField(blank=True)
test_at = models.DateTimeField(editable=False, blank=True, null=True)
build_at = models.DateTimeField(editable=False, blank=True, null=True)
doc_at = models.DateTimeField(editable=False, blank=True, null=True)
done_at = models.DateTimeField(editable=False, blank=True, null=True)
created = models.DateTimeField(editable=False, auto_now_add=True)
updated = models.DateTimeField(editable=False, auto_now=True)
@property
def state(self):
if self.done_at and self.doc_at and self.build_at and self.test_at:
return 'done'
if self.doc_at and self.build_at and self.test_at:
return 'doced'
if self.build_at and self.test_at:
return 'built'
if self.test_at:
return 'tested'
return 'new'
@property
def summary(self):
result = {'test': {}, 'lint': {}}
overall_complete = True
overall_success = True
score_list = []
complete = True
success = True
for (name, value) in self.lint_results.items():
if value.get('score', None) is not None:
score_list.append(value['score'])
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
if score_list:
result['lint']['score'] = sum(score_list) / len(score_list)
else:
result['lint']['score'] = None
if not complete:
result['lint']['status'] = 'Incomplete'
elif success:
result['lint']['status'] = 'Success'
else:
result['lint']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
score_list = []
complete = True
success = True
for (name, value) in self.test_results.items():
if value.get('score', None) is not None:
score_list.append(value['score'])
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
if score_list:
result['test']['score'] = sum(score_list) / len(score_list)
else:
result['test']['score'] = None
if not complete:
result['test']['status'] = 'Incomplete'
elif success:
result['test']['status'] = 'Success'
else:
result['test']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
complete = True
success = True
for target in self.build_results:
for (name, value) in self.build_results[target].items():
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
result['build'] = {}
if not complete:
result['build']['status'] = 'Incomplete'
elif success:
result['build']['status'] = 'Success'
else:
result['build']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
if self.branch == 'master':
complete = True
success = True
for (name, value) in self.doc_results.items():
complete &= value.get('status', '') == 'done'
success &= value.get('success', False)
result['doc'] = {}
if not complete:
result['doc']['status'] = 'Incomplete'
elif success:
result['doc']['status'] = 'Success'
else:
result['doc']['status'] = 'Failed'
overall_complete &= complete
overall_success &= success
if not overall_complete:
result['status'] = 'Incomplete'
elif overall_success:
result['status'] = 'Success'
else:
result['status'] = 'Failed'
return result
@property
def results(self): # for now in Markdown format
result = {}
wrk = {}
for name in self.lint_results:
tmp = self.lint_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'],
tmp.get('score', None))
if wrk:
result['lint'] = wrk
wrk = {}
for name in self.test_results:
tmp = self.test_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'],
tmp.get('score', None))
if wrk:
result['test'] = wrk
wrk = {}
for target in self.build_results:
wrk[target] = {}
tmp = self.build_results[target]
for name in tmp:
if tmp[name].get('results', None) is not None:
wrk[target][name] = (tmp[name].get('success', False),
tmp[name]['results'], None)
if wrk:
result['build'] = wrk
wrk = {}
for name in self.doc_results:
tmp = self.doc_results[name]
if tmp.get('results', None) is not None:
wrk[name] = (tmp.get('success', False), tmp['results'], None)
if wrk:
result['doc'] = wrk
if not result:
return None
else:
return result
def setResults(self, target, name, results):
if target not in ('lint', 'test', 'rpm', 'dpkg', 'respkg', 'resource',
'doc'):
return
if target == 'lint':
self.lint_results[name]['results'] = results
elif target == 'test':
self.test_results[name]['results'] = results
elif target == 'doc':
self.doc_results[name]['results'] = results
else:
self.build_results[target][name]['results'] = results
self.full_clean()
self.save()
def getResults(self, target):
if target == 'lint':
return dict([(i, self.lint_results[i].get('results', None))
for i in self.lint_results])
elif target == 'test':
return dict([(i, self.test_results[i].get('results', None))
for i in self.test_results])
elif target == 'doc':
return dict([(i, self.doc_results[i].get('results', None))
for i in self.doc_results])
elif target in ('rpm', 'dpkg', 'respkg', 'resource'):
return dict([(i,
self.build_results[target][i].get('results', None))
for i in self.build_results[target]])
return {}
def setScore(self, target, name, score):
if target not in ('lint', 'test'):
return
if target == 'lint':
self.lint_results[name]['score'] = score
elif target == 'test':
self.test_results[name]['score'] = score
self.full_clean()
self.save()
def getScore(self, target):
if target == 'lint':
return dict([(i, self.lint_results[i].get('score', None))
for i in self.lint_results])
elif target == 'test':
return dict([(i, self.test_results[i].get('score', None))
for i in self.test_results])
return {}
def signalComplete(self, target, name, success):
if target not in ('test', 'rpm', 'dpkg', 'respkg', 'resource', 'doc'):
return
if target == 'test':
self.lint_results[name]['status'] = 'done'
self.lint_results[name]['success'] = success
self.test_results[name]['status'] = 'done'
self.test_results[name]['success'] = success
elif target == 'doc':
self.doc_results[name]['status'] = 'done'
self.doc_results[name]['success'] = success
else:
self.build_results[target][name]['status'] = 'done'
self.build_results[target][name]['success'] = success
self.full_clean()
self.save()
def postInProcess(self):
if self.project.type != 'GitHubProject':
return
if not self.branch.startswith('_PR'):
return
gh = self.project.githubproject.github
if self.owner_override:
gh.setOwner(self.owner_override)
gh.postCommitStatus(self.commit, 'pending')
gh.setOwner()
def postResults(self):
if self.project.type != 'GitHubProject':
return
gh = self.project.githubproject.github
if self.owner_override:
gh.setOwner(self.owner_override)
comment = self.results
try:
prev_comment = Commit.objects.filter(
project=self.project,
branch=self.branch,
done_at__lt=self.done_at).order_by('-done_at')[0].results
except (Commit.DoesNotExist, IndexError):
prev_comment = None
if prev_comment is None and comment is None:
comment = '**Nothing To Do**'
elif prev_comment is not None and comment is None:
comment = '**Last Commit Had Results, This One Did Not**'
else:
comment = _markdownResults(comment, prev_comment)
gh.postCommitComment(self.commit, comment)
if self.branch.startswith('_PR'):
summary = self.summary
if not summary:
summary = '**Nothing To Do**'
if summary['status'] == 'Success':
gh.postCommitStatus(self.commit,
'success',
description='Passed')
elif summary['status'] == 'Failed':
gh.postCommitStatus(self.commit,
'failure',
description='Failure')
else:
gh.postCommitStatus(self.commit,
'error',
description='Bad State "{0}"'.format(
summary['status']))
gh.setOwner()
number = int(self.branch[3:])
if 'doc' in summary:
gh.postPRComment(
number,
'Lint: {0} {1}\nTest: {2} {3}\nBuild: {4}\nDoc: {5}\nOverall: {6}\n'
.format(
summary['lint']['status'],
'({0})'.format(summary['lint']['score'])
if summary['lint']['score'] else '',
summary['test']['status'],
'({0})'.format(summary['test']['score'])
if summary['test']['score'] else '',
summary['build']['status'], summary['doc']['status'],
summary['status']))
else:
gh.postPRComment(
number,
'Lint: {0} {1}\nTest: {2} {3}\nBuild: {4}\nOverall: {5}\n'.
format(
summary['lint']['status'],
'({0})'.format(summary['lint']['score'])
if summary['lint']['score'] else '',
summary['test']['status'],
'({0})'.format(summary['test']['score'])
if summary['test']['score'] else '',
summary['build']['status'], summary['status']))
@cinp.list_filter(name='project',
paramater_type_list=[{
'type': 'Model',
'model': Project
}])
@staticmethod
def filter_project(project):
return Commit.objects.filter(project=project).order_by('-created')
@cinp.list_filter(name='in_process', paramater_type_list=[])
@staticmethod
def filter_in_process():
return Commit.objects.filter(done_at__isnull=True)
@cinp.check_auth()
@staticmethod
def checkAuth(user, verb, id_list, action=None):
return True
def clean(self, *args, **kwargs):
super().clean(*args, **kwargs)
errors = {}
if errors:
raise ValidationError(errors)
def __str__(self):
return 'Commit "{0}" on branch "{1}" of project "{2}"'.format(
self.commit, self.branch, self.project.name)
class DynamicResource( Resource ):
"""
DynamicResource
"""
build_ahead_count_map = MapField( blank=True, null=True ) # mabey we should have a blueprint model for this and for the allowed blueprints
sites = models.ManyToManyField( Site, through='DynamicResourceSite' )
def _takeOver( self, dynamic_resource_instance, buildjob, buildresource, index ):
buildjob_resource = dynamic_resource_instance.buildjobresourceinstance
buildjob_resource.buildjob = buildjob
buildjob_resource.name = buildresource.name
buildjob_resource.index = index
buildjob_resource.config_values = buildresource.config_values
buildjob_resource.auto_run = buildresource.auto_run
buildjob_resource.auto_provision = buildresource.auto_provision
buildjob_resource.full_clean()
buildjob_resource.save()
buildjob_resource.updateConfig()
if buildjob_resource.state == 'built': # there may be some triggers(auto_run) that would of happened if this was built normally, do that now
buildjob_resource.signalBuilt( buildjob_resource.cookie )
def _createNew( self, site, interface_map, buildjob, buildresource, index ):
BuildJobResourceInstance = apps.get_model( 'Processor', 'BuildJobResourceInstance' )
resource_instance = DynamicResourceInstance()
resource_instance.dynamic_resource = self
resource_instance.site = site
resource_instance.interface_map = interface_map
resource_instance.full_clean()
resource_instance.save()
buildjob_resource = BuildJobResourceInstance()
buildjob_resource.buildjob = buildjob
buildjob_resource.resource_instance = resource_instance
buildjob_resource.name = buildresource.name
buildjob_resource.index = index
buildjob_resource.blueprint = buildresource.blueprint
buildjob_resource.config_values = buildresource.config_values
buildjob_resource.auto_run = buildresource.auto_run
buildjob_resource.auto_provision = buildresource.auto_provision
buildjob_resource.full_clean()
buildjob_resource.save()
buildjob_resource.allocate()
def _replenish( self, site, interface_map, blueprint ):
BuildJobResourceInstance = apps.get_model( 'Processor', 'BuildJobResourceInstance' )
quantity = self.build_ahead_count_map.get( blueprint, 0 ) - self.dynamicresourceinstance_set.filter( buildjobresourceinstance__buildjob__isnull=True, buildjobresourceinstance__blueprint=blueprint ).count()
if quantity < 1:
return
for _ in range( 0, quantity ):
resource_instance = DynamicResourceInstance()
resource_instance.dynamic_resource = self
resource_instance.site = site
resource_instance.interface_map = interface_map
resource_instance.full_clean()
resource_instance.save()
buildjob_resource = BuildJobResourceInstance()
buildjob_resource.resource_instance = resource_instance
buildjob_resource.name = 'prallocate'
buildjob_resource.index = 0
buildjob_resource.blueprint = blueprint
buildjob_resource.full_clean()
buildjob_resource.save()
buildjob_resource.allocate()
buildjob_resource.build()
def available( self, site, quantity, interface_map ):
if site.name not in list( self.dynamicresourcesite_set.all().values_list( 'site', flat=True ) ):
return False
if not interface_map: # for now this is {} when empty would be nice if it was also None, this will cover both
return _getAvailibleNetwork( site, quantity ) is not None
return True
def allocate( self, site, buildjob, buildresource, interface_map ):
if site.name not in list( self.dynamicresourcesite_set.all().values_list( 'site', flat=True ) ):
raise ValueError( 'site "{0}" not in list of sites for this resource'.format( site ) )
is_custom = interface_map or buildresource.config_values
if not interface_map:
network = _getAvailibleNetwork( site, buildresource.quantity ) # yes, if we are getting only pre-allocated stuff, we are double counting the network ips, however we need ips for the new resources that are going to backfill
interface_map = { 'eth0': { 'network_id': network.contractor_network_id, 'address_block_id': network.contractor_addressblock_id, 'is_primary': True } }
if not is_custom: # no preallocation for non-default networks, and custom config might have values to tweek the build ie: cpu count
# for pre-allocated stuff, we don't care about which site, we use what is there, the backfill will go to the targetd site, if a build wants a gurentee of all the resources being in the same stie, they will need to specify a network anyway
dynamic_resource_instance_list = self.dynamicresourceinstance_set.filter( buildjobresourceinstance__buildjob__isnull=True, buildjobresourceinstance__blueprint=buildresource.blueprint ).order_by( 'pk' ).iterator()
for index in range( 0, buildresource.quantity ):
dynamic_resource_instance = next( dynamic_resource_instance_list, None )
if dynamic_resource_instance is not None:
self._takeOver( dynamic_resource_instance, buildjob, buildresource, index )
else:
self._createNew( site, interface_map, buildjob, buildresource, index )
self._replenish( site, interface_map, buildresource.blueprint )
else:
for index in range( 0, buildresource.quantity ):
self._createNew( site, interface_map, buildjob, buildresource, index )
@cinp.check_auth()
@staticmethod
def checkAuth( user, verb, id_list, action=None ):
return True
def clean( self, *args, **kwargs ):
super().clean( *args, **kwargs )
errors = {}
if not name_regex.match( self.name ):
errors[ 'name' ] = 'Invalid'
if errors:
raise ValidationError( errors )
def __str__( self ):
return 'Resource "{0}"({1})'.format( self.description, self.name )