def save(self, *args, **kwargs): from django_transaction_signals import defer if settings.USE_GRAPH_DB == True and 'admin' not in get_WG(): if self.pk is None: print "saving on graph ",self.gen_id defer(add_aliquot_node, self) super(Aliquots, self).save(*args, **kwargs)
def answer(request, id, form_class=forms.AnswerForm):#process a new answer """view that posts new answer anonymous users post into anonymous storage and redirected to login page authenticated users post directly """ question = get_object_or_404(models.Post, post_type='question', id=id) if request.method == "POST": #this check prevents backward compatilibility if form_class == forms.AnswerForm: custom_class_path = getattr(settings, 'ASKBOT_NEW_ANSWER_FORM', None) if custom_class_path: form_class = load_module(custom_class_path) else: form_class = forms.AnswerForm form = form_class(request.POST, user=request.user) if form.is_valid(): if request.user.is_authenticated(): # run slow task in celery, post-commit from askbot import tasks django_transaction_signals.defer( tasks.remove_draft_answer.delay, author=request.user, thread=question.thread ) try: user = form.get_post_user(request.user) answer = form.save(question, user) signals.new_answer_posted.send(None, answer=answer, user=user, form_data=form.cleaned_data ) return HttpResponseRedirect(answer.get_absolute_url()) except askbot_exceptions.AnswerAlreadyGiven, e: request.user.message_set.create(message = unicode(e)) answer = question.thread.get_answers_by_user(request.user)[0] return HttpResponseRedirect(answer.get_absolute_url()) except exceptions.PermissionDenied, e: request.user.message_set.create(message = unicode(e)) else: request.session.flush() models.AnonymousAnswer.objects.create( question=question, wiki=form.cleaned_data['wiki'], text=form.cleaned_data['text'], session_key=request.session.session_key, ip_addr=request.META['REMOTE_ADDR'], ) return HttpResponseRedirect(url_utils.get_login_url())
def save(self, *args, **kwargs): from django_transaction_signals import defer if settings.USE_GRAPH_DB == True and 'admin' not in get_WG(): if self.pk is None: print "differisco", self.id_genealogy defer(add_mice_node, self) else: print "non differisco" super(BioMice, self).save(*args, **kwargs)
def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None, close_timeout=None, sse=False, email=False, url=None): """ Adds or queues a message to the storage :param level: Level of the message :param message: Message text to be saved :param extra_tags: String with separated tags to add to message Ex: "secret classified" :param subject: Subject of the message :param user: `auth.User` that receives the message :param from_user: `auth.User` that sends the message :param expires: Timestamp that indicates when the message expires :param close_timeout: Integer :param url: Optional string with URL leading to details of a given message .. note:: The message is only saved if it contains something and its level is over the recorded level (`MESSAGE_LEVEL`) `self.level` """ to_user = user or get_user(self.request) if not to_user.is_authenticated(): if Monit(level=level).is_persistent(): raise NotImplementedError( 'Persistent message levels cannot be used for anonymous users.') else: return super(PersistentMessageStorage, self).add(level, message, extra_tags) if not message: return # Save the message only if its level is over the recorded level, see `MESSAGE_LEVEL` in Django docs level = int(level) if level < self.level: return # Add the message message = Monit(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject, from_user=from_user, expires=expires, close_timeout=close_timeout, url=url) # Messages need a primary key when being displayed so that they can be # closed/marked as read by the user. Hence, save it now instead of # adding it to queue: if STORE_WHEN_ADDING: message.save() if sse: # Sent delayed SSE notification defer(notify.via_sse, message.pk) if email: defer(notify.via_email, message.pk) return message else: self.added_new = True self._queued_messages.append(message)
def register_heartbeat(cls, obj): state = obj.state state.last_seen_on = timezone.now() state.last_contact_on = state.last_seen_on state.save() if state.value in (cls.CRASHED, cls.OFFLINE): opts = type(obj)._meta module = '%s.%s' % (opts.app_label, opts.object_name) defer(get_state.delay, module, ids=[obj.pk], lock=False) node_heartbeat.send(sender=cls, node=obj.state.get_node())
def save(self, *args, **kwargs): from django_transaction_signals import defer if settings.USE_GRAPH_DB == True and 'admin' not in get_WG(): if self.pk is not None: print "non inserisco nodo, gia' presente" else: print self if self.id_genealogy is not None and self.id_explant is not None and self.pk is None: print "salvo aliquota da espianto nel grafo" defer(add_aliquot_node, self) super(Aliquots, self).save(*args, **kwargs) else: super(Aliquots, self).save(*args, **kwargs)
def put(self, tube_name, message, delay=0, connection=None): if not connection: connection = self.connection connection.use(tube_name) if settings.USE_TRANSACTION_SIGNALS: import django_transaction_signals django_transaction_signals.defer( self.__put, tube_name, message, delay=delay, connection=connection ) else: self.__put(tube_name, message, delay=delay, connection=connection)
def save(self, *args, **kwargs): if not self.pk: # generate and initialize name if not self.name: self.name = self._name # Try to restore object to allow update in nested serialization try: obj = TincHost.objects.get(content_type_id=self.content_type_id, object_id=self.object_id) except TincHost.DoesNotExist: pass else: self.pk = obj.pk super(TincHost, self).save(*args, **kwargs) defer(update_tincd.delay)
class Instance(models.Model): """ defines an operation execution in a particular node, keeping its state """ RECEIVED = 'RECEIVED' TIMEOUT = 'TIMEOUT' STARTED = 'STARTED' SUCCESS = 'SUCCESS' FAILURE = 'FAILURE' ERROR = 'ERROR' REVOKED = 'REVOKED' OUTDATED = 'OUTDATED' STATES = ( (RECEIVED, RECEIVED), (TIMEOUT, TIMEOUT), (STARTED, STARTED), (SUCCESS, SUCCESS), (FAILURE, FAILURE), (ERROR, ERROR), (REVOKED, REVOKED), (OUTDATED, OUTDATED)) execution = models.ForeignKey(Execution, related_name='instances') node = models.ForeignKey('nodes.Node', related_name='operations') state = models.CharField(max_length=16, choices=STATES, default=RECEIVED) last_try = models.DateTimeField(null=True) stdout = models.TextField() stderr = models.TextField() traceback = models.TextField() exit_code = models.IntegerField(null=True) task_id = models.CharField(max_length=36, unique=True, null=True, help_text="Celery task ID") class Meta: ordering = ['-last_try'] def __unicode__(self): return u"%s@%s" % (self.execution.operation.identifier, self.node) @classmethod def create(cls, execution, node): # outdate pending instances cls.objects.filter(execution__operation=execution.operation, node=node, state=cls.TIMEOUT).update(state=Instance.OUTDATED) instance = cls.objects.create(execution=execution, node=node) return instance def run(self, async=True): if self.state == self.STARTED: raise ConcurrencyError("One run at a time.") if async: defer(run_instance.delay, self.pk) else: run_instance(self.pk)
def delete(self, *args, **kwargs): super(TincAddress, self).delete(*args, **kwargs) defer(update_tincd.delay)
def ask(request):#view used to ask a new question """a view to ask a new question gives space for q title, body, tags and checkbox for to post as wiki user can start posting a question anonymously but then must login/register in order for the question go be shown """ form = forms.AskForm(request.REQUEST, user=request.user) if request.method == 'POST': if form.is_valid(): timestamp = datetime.datetime.now() title = form.cleaned_data['title'] wiki = form.cleaned_data['wiki'] tagnames = form.cleaned_data['tags'] text = form.cleaned_data['text'] ask_anonymously = form.cleaned_data['ask_anonymously'] post_privately = form.cleaned_data['post_privately'] group_id = form.cleaned_data.get('group_id', None) language = form.cleaned_data.get('language', None) if request.user.is_authenticated(): # run slow task in celery, post-commit from askbot import tasks django_transaction_signals.defer( tasks.remove_draft_question.delay, author=request.user ) # drafts = models.DraftQuestion.objects.filter( # author=request.user # ) # drafts.delete() user = form.get_post_user(request.user) try: question = user.post_question( title=title, body_text=text, tags=tagnames, wiki=wiki, is_anonymous=ask_anonymously, is_private=post_privately, timestamp=timestamp, group_id=group_id, language=language ) signals.new_question_posted.send(None, question=question, user=user, form_data=form.cleaned_data ) return HttpResponseRedirect(question.get_absolute_url()) except exceptions.PermissionDenied, e: request.user.message_set.create(message = unicode(e)) return HttpResponseRedirect(reverse('index')) else: request.session.flush() session_key = request.session.session_key models.AnonymousQuestion.objects.create( session_key = session_key, title = title, tagnames = tagnames, wiki = wiki, is_anonymous = ask_anonymously, text = text, added_at = timestamp, ip_addr = request.META['REMOTE_ADDR'], ) return HttpResponseRedirect(url_utils.get_login_url())
class Build(models.Model): """ Represents a built image for a research device. All the build information is copied, not referenced by related objects that can change over time. Only the most recent build of each node is stored in order to keep the model simple """ REQUESTED = 'REQUESTED' QUEUED = 'QUEUED' BUILDING = 'BUILDING' AVAILABLE = 'AVAILABLE' OUTDATED = 'OUTDATED' DELETED = 'DELETED' FAILED = 'FAILED' node = models.OneToOneField('nodes.Node', primary_key=True, related_name='firmware_build') date = models.DateTimeField(auto_now_add=True) version = models.CharField(max_length=64) image = PrivateFileField( storage=settings.FIRMWARE_BUILD_IMAGE_STORAGE, upload_to=settings.FIRMWARE_BUILD_IMAGE_PATH, max_length=256, condition=any_auth_method(lambda request, self: request.user.has_perm( 'nodes.getfirmware_node', obj=self.node))) base_image = models.FileField( storage=settings.FIRMWARE_BASE_IMAGE_STORAGE, upload_to=settings.FIRMWARE_BASE_IMAGE_PATH, help_text= 'Image file compressed in gzip. The file name must end in .img.gz') task_id = models.CharField(max_length=36, unique=True, null=True, help_text="Celery task ID") kwargs = models.TextField() objects = generate_chainer_manager(BuildQuerySet) class Meta: ordering = ['-date'] def __unicode__(self): return unicode(self.node) def delete(self, *args, **kwargs): """ Deletes the build and also the image file stored on the file system """ super(Build, self).delete(*args, **kwargs) try: os.remove(self.image.path) except: pass @property def image_name(self): return self.image.name.split('/')[-1] @property def kwargs_dict(self): """ Return stored kwargs as a python dictionary """ try: return ast.literal_eval(self.kwargs) except (SyntaxError, ValueError): # malformed kwargs string return {} @property @cached def db_task(self): """ Returns the celery task responsible for 'self' image build """ if not self.task_id: return None try: return TaskState.objects.get(task_id=self.task_id) except TaskState.DoesNotExist: return None @property def task(self): return build_task.AsyncResult(self.task_id) @property @cached def state(self): """ Gives the current state of the build """ if self.image: try: self.image.file except IOError: return self.DELETED except SuspiciousFileOperation: msg = ( "There is some issue accessing build image file. Check " "that image.name path is inside image.storage.location.") logging.exception(msg) return 'ACCESS DENIED' else: if self.match_config: return self.AVAILABLE else: return self.OUTDATED if not self.task_id: return self.REQUESTED if self.task and self.task.state == celery_states.RECEIVED: return self.QUEUED if self.task and self.task.state == celery_states.STARTED: return self.BUILDING return self.FAILED @property def state_description(self): description = { Build.REQUESTED: "Building task received.", Build.QUEUED: "Building task queued for processing.", Build.BUILDING: "Your task is now being processed, this can take a while.", Build.AVAILABLE: "Firmware available for download.", Build.DELETED: "The firmware is no longer available. Do you want to build a new one?", Build.OUTDATED: "The existing firmware is out-dated. Although is " "available to download, maybe you want to delete it " "and build an updated one.", Build.FAILED: "The last build has failed. The error logs are monitored " "and this issue will be fixed. But you can try again anyway.", } return description.get(self.state, '') @property @cached def image_sha256(self): try: return sha256(self.image.file.read()).hexdigest() except: return None @classmethod def build(cls, node, base_image, async=False, exclude=[], **kwargs): """ This method handles the building image, if async is True the building task will be executed with Celery """ try: old_build = cls.objects.get(node=node) except cls.DoesNotExist: pass else: if old_build.state == cls.BUILDING: raise ConcurrencyError("One build at a time.") old_build.delete() config = Config.objects.get() build_obj = Build.objects.create(node=node, version=config.version, base_image=base_image.image, kwargs=kwargs) # handle registry api #245: save cert content into DB cert = kwargs.pop('registry_cert') config = ConfigFile.objects.get(path='/etc/config/confine') build_obj.add_file('/etc/confine/registry-server.crt', cert, config) if async: defer(build_task.delay, build_obj.pk, exclude=exclude, **kwargs) else: build_obj = build_task(build_obj.pk, exclude=exclude, **kwargs) return build_obj
class Sliver(models.Model): """ Describes a sliver in the testbed, an sliver is a partition of a node's resources assigned to a specific slice. """ slice = models.ForeignKey(Slice, related_name='slivers') node = models.ForeignKey(Node, related_name='slivers') description = models.TextField( blank=True, help_text='An optional free-form textual description of this sliver.') instance_sn = models.PositiveIntegerField( default=0, blank=True, help_text='The number of times this sliver has been instructed to be ' 'updated (instance sequence number).', verbose_name='instance sequence number') data = models.FileField(blank=True, verbose_name='sliver data', upload_to=make_upload_to( 'data', settings.SLICES_SLIVER_DATA_DIR, settings.SLICES_SLIVER_DATA_NAME), help_text='File containing data for this sliver.') data_uri = NullableCharField( 'sliver data URI', max_length=256, blank=True, null=True, help_text='If present, the URI of a file containing data for ' 'this sliver, instead of the one specified by the slice. Its ' 'format and contents depend on the type of the template to be used.') data_sha256 = NullableCharField( 'sliver data SHA256', max_length=64, blank=True, null=True, help_text= 'The SHA256 hash of the sliver data file, used to check its integrity. ' 'Compulsory when a file has been specified.', validators=[validate_sha256]) set_state = NullableCharField( max_length=16, choices=Slice.STATES, blank=True, help_text='If present, the state set on this sliver (set state), ' 'which overrides a higher one specified by the slice ' '(e.g. register overrides start, but start does not override register). ' 'Possible values: register (initial) < deploy < start. ' 'See <a href="https://wiki.confine-project.eu/arch:' 'slice-sliver-states">slice and sliver states</a> for the full ' 'description of set states and possible transitions.', null=True) template = models.ForeignKey( Template, null=True, blank=True, limit_choices_to={'is_active': True}, help_text='If present, the template to be used by this sliver, instead ' 'of the one specified by the slice.') _iface_registry = {} class Meta: unique_together = ('slice', 'node') def __unicode__(self): return u'%s@%s' % (self.slice.name, self.node.name) def clean(self): super(Sliver, self).clean() clean_sha256(self, ('data', )) clean_uri(self, ('data', )) def save(self, *args, **kwargs): if not self.pk: self.instance_sn = self.slice.sliver_defaults.instance_sn save_files_with_pk_value(self, ('data', ), *args, **kwargs) set_sha256(self, ('data', )) set_uri(self, ('data', )) super(Sliver, self).save(*args, **kwargs) @property def max_num_ifaces(self): return 256 # limited by design -> #nr: unsigned 8 bits @property def effective_set_state(self): slice = self.slice # sliver set_state overrides sliver_defaults set_state = self.set_state or slice.sliver_defaults.set_state # effective set_state <= slice.set_state if slice.set_state == slice.DEPLOY and set_state == slice.REGISTER: return set_state elif slice.set_state == slice.START and set_state in [ slice.REGISTER, slice.DEPLOY ]: return set_state return slice.set_state @property def mgmt_iface(self): iface = self.interfaces.filter(type='management') return iface.first() if iface else None @property def mgmt_net(self): """ Only available if the sliver has interfaces of type management, in which case the management address is that of the first management interface, with "native" as a backend. Otherwise the whole member is null. """ if self.mgmt_iface is None: return None return { "backend": "native", "addr": self.mgmt_iface.ipv6_addr, } @property def api_id(self): """ The unique ID of this sliver (REST-API) """ return "%i@%i" % (self.slice_id, self.node_id) def update(self): self.instance_sn += 1 self.save() self.slice.sliver_defaults.instance_sn += 1 self.slice.sliver_defaults.save() def force_update(self, async=False): if async: defer(force_sliver_update.delay, self.pk) else: force_sliver_update(self.pk)
class Slice(models.Model): """ Describes a slice in the testbed. An slice is a set of resources spread over several nodes in a testbed which allows researchers to run experiments over it. """ MIN_VLAN_TAG = 0x100 MAX_VLAN_TAG = 0xfff REGISTER = 'register' DEPLOY = 'deploy' START = 'start' STATES = ( (REGISTER, 'REGISTER'), (DEPLOY, 'DEPLOY'), (START, 'START'), ) name = models.CharField( max_length=128, unique=True, help_text= 'A unique name of this slice. A single non-empty line of free-form ' 'text with no whitespace surrounding it.', validators=[validate_name]) description = models.TextField( blank=True, help_text='An optional free-form textual description of this slice.' 'e.g. what is being tested, what were the results, ' 'possible impact of the experiment, URL to related ' 'resources...') expires_on = models.DateField( null=True, blank=True, default=get_expires_on, help_text='Expiration date of this slice. Automatically deleted once ' 'expires.') instance_sn = models.PositiveIntegerField( default=0, blank=True, help_text= 'The number of times this slice has been instructed to be reset ' '(instance sequence number). Automatically incremented by the ' 'reset function.', verbose_name='instance sequence number') allow_isolated = models.BooleanField( default=False, help_text='Whether to request a VLAN tag for isolated sliver interfaces ' '(see node architecture) at slice deployment time. If the ' 'allocation is successful, the tag is stored in the ' '/isolated_vlan_tag member. Otherwise, the deployment of the ' 'slice fails', verbose_name='Request isolated VLAN tag') isolated_vlan_tag = models.IntegerField( 'Isolated VLAN tag', null=True, blank=True, help_text='VLAN tag allocated to this slice. The only values that can ' 'be set are null which means that no VLAN is wanted for the ' 'slice, and -1 which asks the server to allocate for the slice ' 'a new VLAN tag (100 <= vlan_tag < 0xFFF) while the slice is ' 'instantiated (or active). It cannot be changed on an ' 'instantiated slice with slivers having isolated interfaces.') set_state = models.CharField( max_length=16, choices=STATES, default=REGISTER, help_text='The state set on this slice (set state) and its slivers ' '(if they do not explicitly indicate a lower one). ' 'Possible values: register (initial) < deploy < start. ' 'See <a href="https://wiki.confine-project.eu/arch:' 'slice-sliver-states">slice and sliver states</a> for the full ' 'description of set states and possible transitions.') group = models.ForeignKey('users.Group', related_name='slices') def __unicode__(self): return self.name def update_set_state(self, commit=True): if self.set_state in [self.DEPLOY, self.START]: if self.isolated_vlan_tag is None and self.allow_isolated: try: self.isolated_vlan_tag = Slice._get_vlan_tag() except VlanAllocationError: self.set_state = self.REGISTER elif self.isolated_vlan_tag > 0: # REGISTER state # transition to a register state, deallocating... self.isolated_vlan_tag = None if commit: self.save() def save(self, *args, **kwargs): # TODO send message to user when error happens self.update_set_state(commit=False) if not self.pk: self.expires_on = now() + settings.SLICES_SLICE_EXP_INTERVAL super(Slice, self).save(*args, **kwargs) def clean(self): super(Slice, self).clean() # clean set_state if not self.pk: if self.set_state != Slice.REGISTER: raise ValidationError("Initial state must be Register") # clean allow_isolated else: old = Slice.objects.get(pk=self.pk) has_changed = self.allow_isolated != old.allow_isolated if has_changed and old.set_state != self.REGISTER: raise ValidationError( "Vlan can not be requested in state != register") def renew(self): """Renew expires_on date, except has alreday reached the maximum""" new_expires_on = get_expires_on() if self.expires_on == new_expires_on.date(): return False self.expires_on = new_expires_on self.save() return True def reset(self): self.instance_sn += 1 self.save() @property def max_vlan_nr(self): return Slice.MAX_VLAN_TAG @property def min_vlan_nr(self): return Slice.MIN_VLAN_TAG @classmethod def _get_vlan_tag(cls): qset = cls.objects.exclude( isolated_vlan_tag=None).order_by('-isolated_vlan_tag') last_nr = qset.first().isolated_vlan_tag if qset else 0 if last_nr < cls.MIN_VLAN_TAG: return cls.MIN_VLAN_TAG if last_nr >= cls.MAX_VLAN_TAG: # Try to recycle old values ( very, very ineficient ) for new_nr in range(cls.MIN_VLAN_TAG, cls.MAX_VLAN_TAG): if not cls.objects.filter(isolated_vlan_tag=new_nr).exists(): return new_nr raise VlanAllocationError("No VLAN address space left.") return last_nr + 1 def force_update(self, async=False): if async: defer(force_slice_update.delay, self.pk) else: force_slice_update(self.pk)
def add(self, level, message, extra_tags='', subject='', user=None, from_user=None, expires=None, close_timeout=None, sse=True, email=False): """ Adds or queues a message to the storage :param level: Level of the message :param message: Message text to be saved :param extra_tags: String with separated tags to add to message Ex: "secret classified" :param subject: Subject of the message :param user: `auth.User` that receives the message :param from_user: `auth.User` that sends the message :param expires: Timestamp that indicates when the message expires :param close_timeout: Integer .. note:: The message is only saved if it contains something and its level is over the recorded level (`MESSAGE_LEVEL`) `self.level` """ to_user = user or get_user(self.request) if not to_user.is_authenticated(): if Monit(level=level).is_persistent(): raise NotImplementedError( 'Persistent message levels cannot be used for anonymous users.' ) else: return super(PersistentMessageStorage, self).add(level, message, extra_tags) if not message: return # Save the message only if its level is over the recorded level, see `MESSAGE_LEVEL` in Django docs level = int(level) if level < self.level: return # Add the message message = Monit(user=to_user, level=level, message=message, extra_tags=extra_tags, subject=subject, from_user=from_user, expires=expires, close_timeout=close_timeout) # Messages need a primary key when being displayed so that they can be closed/marked as read by the user. # Hence, save it now instead of adding it to queue: if STORE_WHEN_ADDING: message.save() if sse: # Sent delayed SSE notification defer(notify.via_sse, message.pk) if email: defer(notify.via_email, message.pk) return message else: print "TAM" self.added_new = True self._queued_messages.append(message)
def defer_celery_task(task, **kwargs): if django_settings.CELERY_ALWAYS_EAGER: return task.apply_async(**kwargs) else: return defer(task.apply_async, **kwargs)