def __upload_decisions(self, job): uploaded_map = {} decisions_data = self.__read_json_file('{}.json'.format(Decision.__name__)) if not decisions_data: # The job is without reports return # Upload decisions for decision in decisions_data: if 'id' not in decision or not isinstance(decision['id'], int): raise exceptions.ValidationError({'decision': _('Decision data is corrupted')}) serializer = DownloadDecisionSerializer(data=decision) serializer.is_valid(raise_exception=True) uploaded_map[decision['id']] = serializer.save( job=job, operator=self._user, status=DECISION_STATUS[0][0] ).id self._final_statuses[uploaded_map[decision['id']]] = serializer.validated_data['status'] # Upload decision cache cache_data_list = self.__read_json_file('{}.json'.format(DecisionCache.__name__)) new_cache_objects = [] for dec_cache in cache_data_list: if 'decision' not in dec_cache or not uploaded_map.get(dec_cache['decision']): raise exceptions.ValidationError({'decision': _('Decision data is corrupted')}) serializer = DecisionCacheSerializer(data=dec_cache) serializer.is_valid(raise_exception=True) new_cache_objects.append(DecisionCache( decision_id=uploaded_map[dec_cache['decision']], **serializer.validated_data )) DecisionCache.objects.bulk_create(new_cache_objects) return uploaded_map
def __recalc(self): cache_data = {} for report in ReportComponent.objects.filter( decision__in=self._decisions): if (report.decision_id, report.component) not in cache_data: cache_data[(report.decision_id, report.component)] = { 'total': 0, 'finished': 0, 'cpu_time': 0, 'wall_time': 0, 'memory': 0 } cache_data[(report.decision_id, report.component)]['total'] += 1 if report.finish_date: cache_data[(report.decision_id, report.component)]['finished'] += 1 if report.cpu_time: cache_data[(report.decision_id, report.component)]['cpu_time'] += report.cpu_time if report.wall_time: cache_data[(report.decision_id, report.component)]['wall_time'] += report.wall_time if report.memory: cache_data[(report.decision_id, report.component)]['memory'] = max( cache_data[(report.decision_id, report.component)]['memory'], report.memory) DecisionCache.objects.filter(decision_id__in=self._decisions).delete() DecisionCache.objects.bulk_create( list( DecisionCache( decision_id=d_id, component=component, **obj_kwargs) for (d_id, component), obj_kwargs in cache_data.items()))
def __update_decision_cache(self, component, **kwargs): try: cache_obj = DecisionCache.objects.select_for_update().get(decision=self.decision, component=component) except DecisionCache.DoesNotExist: cache_obj = DecisionCache(decision=self.decision, component=component) if kwargs.get('cpu_time'): cache_obj.cpu_time += kwargs['cpu_time'] if kwargs.get('wall_time'): cache_obj.wall_time += kwargs['wall_time'] if kwargs.get('memory'): cache_obj.memory = max(cache_obj.memory, kwargs['memory']) if kwargs.get('started'): cache_obj.total += 1 if kwargs.get('finished'): cache_obj.finished += 1 cache_obj.save()
def __upload_decisions(self): decisions_data = self.__read_json_file('{}.json'.format( Decision.__name__)) if not decisions_data: # The job is without reports return # Upload decisions for decision in decisions_data: if 'id' not in decision or not isinstance(decision['id'], int): raise exceptions.ValidationError( {'decision': _('Decision data is corrupted')}) serializer = DownloadDecisionSerializer(data=decision) serializer.is_valid(raise_exception=True) decision_obj = serializer.save(job=self.job, operator=self._upload_obj.author, status=DECISION_STATUS[0][0]) self._decisions[decision['id']] = decision_obj.id self._identifiers_in_use[decision_obj.id] = set() self._final_statuses[ decision_obj.id] = serializer.validated_data['status'] if not self._decisions: # The job does not have decisions return # Upload decision cache cache_data_list = self.__read_json_file('{}.json'.format( DecisionCache.__name__)) if not cache_data_list: # All decisions should have cache raise exceptions.ValidationError( {'decision': _('Decision data is corrupted')}) new_cache_objects = [] for dec_cache in cache_data_list: if 'decision' not in dec_cache or not self._decisions.get( dec_cache['decision']): raise exceptions.ValidationError( {'decision': _('Decision data is corrupted')}) serializer = DecisionCacheSerializer(data=dec_cache) serializer.is_valid(raise_exception=True) new_cache_objects.append( DecisionCache( decision_id=self._decisions[dec_cache['decision']], **serializer.validated_data)) DecisionCache.objects.bulk_create(new_cache_objects)