def bulk_create(self, request, **kwargs): validator = validators.MembersBulkValidator(data=request.DATA) if not validator.is_valid(): return response.BadRequest(validator.errors) data = validator.data project = models.Project.objects.get(id=data["project_id"]) invitation_extra_text = data.get("invitation_extra_text", None) self.check_permissions(request, 'bulk_create', project) if project.blocked_code is not None: raise exc.Blocked(_("Blocked element")) if "bulk_memberships" in data and isinstance(data["bulk_memberships"], list): total_new_memberships = len(data["bulk_memberships"]) self._check_if_project_can_have_more_memberships(project, total_new_memberships) try: with advisory_lock("membership-creation-{}".format(project.id)): members = services.create_members_in_bulk(data["bulk_memberships"], project=project, invitation_extra_text=invitation_extra_text, callback=self.post_save, precall=self.pre_save) except exc.ValidationError as err: return response.BadRequest(err.message_dict) members_serialized = self.admin_serializer_class(members, many=True) return response.Ok(members_serialized.data)
def store_league(result, region): """ Callback that stores the result of the RiotWatcher get_league calls. `summoner_id` is expected to be the single key of the `result` dict. Stores a previously unknown league or replaces the entirety of the summoner's league's entries if it was known. """ lock_id = 'store_league' with advisory_lock(lock_id) as acquired: # Empty dict means that the queried summoner is not in a league. if result != {}: for summoner_id in result: logger.debug('Reading leagues for summoner ID %s', summoner_id) for league in result[summoner_id]: League.objects.create_or_update_league(league, region) # TODO: This is misleading since we can block updates in update_league # based on last_update. logger.info('Stored %s leagues for [%s] %s', len(result[summoner_id]), region, summoner_id) return True else: return False
def create_template(self, request, **kwargs): template_name = request.DATA.get('template_name', None) template_description = request.DATA.get('template_description', None) if not template_name: raise response.BadRequest(_("Not valid template name")) if not template_description: raise response.BadRequest(_("Not valid template description")) with advisory_lock("create-project-template") as acquired_key_lock: template_slug = slugify_uniquely(template_name, models.ProjectTemplate) project = self.get_object() self.check_permissions(request, 'create_template', project) template = models.ProjectTemplate( name=template_name, slug=template_slug, description=template_description, ) template.load_data_from_project(project) template.save() return response.Created(serializers.ProjectTemplateSerializer(template).data)
def take_snapshot(obj: object, *, comment: str="", user=None, delete: bool=False): """ Given any model instance with registred content type, create new history entry of "change" type. This raises exception in case of object wasn't previously freezed. """ key = make_key_from_model_object(obj) with advisory_lock("history-"+key): typename = get_typename_for_model_class(obj.__class__) new_fobj = freeze_model_instance(obj) old_fobj, need_real_snapshot = get_last_snapshot_for_key(key) entry_model = apps.get_model("history", "HistoryEntry") user_id = None if user is None else user.id user_name = "" if user is None else user.get_full_name() # Determine history type if delete: entry_type = HistoryType.delete need_real_snapshot = True elif new_fobj and not old_fobj: entry_type = HistoryType.create elif new_fobj and old_fobj: entry_type = HistoryType.change else: raise RuntimeError("Unexpected condition") fdiff = make_diff(old_fobj, new_fobj) # If diff and comment are empty, do # not create empty history entry if (not fdiff.diff and not comment and old_fobj is not None and entry_type != HistoryType.delete): return None fvals = make_diff_values(typename, fdiff) if len(comment) > 0: is_hidden = False else: is_hidden = is_hidden_snapshot(fdiff) kwargs = { "user": {"pk": user_id, "name": user_name}, "project_id": getattr(obj, 'project_id', getattr(obj, 'id', None)), "key": key, "type": entry_type, "snapshot": fdiff.snapshot if need_real_snapshot else None, "diff": fdiff.diff, "values": fvals, "comment": comment, "comment_html": mdrender(obj.project, comment), "is_hidden": is_hidden, "is_snapshot": need_real_snapshot, } return entry_model.objects.create(**kwargs)
def add_block_multi(self, who, blocks): created = [] with advisory_lock("add_block"), transaction.atomic(): for block in blocks: b = self.add_block(who=who, **block) created.append(b) return created
def save(self, *args, **kwargs): if not self._importing or not self.modified_date: self.modified_date = timezone.now() if not self.is_backlog_activated: self.total_milestones = None self.total_story_points = None if not self.videoconferences: self.videoconferences_extra_data = None if not self.is_looking_for_people: self.looking_for_people_note = "" if self.anon_permissions is None: self.anon_permissions = [] if self.public_permissions is None: self.public_permissions = [] if not self.slug: with advisory_lock("project-creation"): base_slug = "{}-{}".format(self.owner.username, self.name) self.slug = slugify_uniquely(base_slug, self.__class__) super().save(*args, **kwargs) else: super().save(*args, **kwargs)
def save(self, *args, **kwargs): if not self.href: with advisory_lock("wiki-page-creation-{}".format(self.project_id)): wl_qs = self.project.wiki_links.all() self.href = slugify_uniquely_for_queryset(self.title, wl_qs, slugfield="href") super().save(*args, **kwargs) else: super().save(*args, **kwargs)
def decorator(self, *args, **kwargs): from taiga.base.utils.db import get_typename_for_model_class pk = self.kwargs.get(self.pk_url_kwarg, None) tn = get_typename_for_model_class(self.get_queryset().model) key = "{0}:{1}".format(tn, pk) with advisory_lock(key): return func(self, *args, **kwargs)
def save(self, *args, **kwargs): if not self._importing or not self.modified_date: self.modified_date = timezone.now() if not self.slug: with advisory_lock("milestone-creation-{}".format(self.project_id)): self.slug = slugify_uniquely(self.name, self.__class__) super().save(*args, **kwargs) else: super().save(*args, **kwargs)
def handle(self, *args, **options): with advisory_lock("send-notifications-command", wait=False) as acquired: if acquired: qs = HistoryChangeNotification.objects.all() for change_notification in iter_queryset(qs, itersize=100): try: send_sync_notifications(change_notification.pk) except HistoryChangeNotification.DoesNotExist: pass else: print("Other process already running")
def add_task(task): with advisory_lock(STATUS_LOCK_ID): task.status = 'waiting' count = Task.objects.filter(status='downloading').count() if count >= 3: task.save() return task.status = 'downloading' task.save() worker = DownloadWorker(task.pk) worker.start()
def update_in_bulk_with_ids(ids, list_of_new_values, model): """Update a table using a list of ids. :params ids: List of ids. :params new_values: List of dicts or duples where each dict/duple is the new data corresponding to the instance in the same index position as the dict. :param model: Model of the ids. """ tn = get_typename_for_model_class(model) for id, new_values in zip(ids, list_of_new_values): key = "{0}:{1}".format(tn, id) with advisory_lock(key) as acquired_key_lock: model.objects.filter(id=id).update(**new_values)
def run(self): while True: if self.pk == None: return self.download() self.pk = None with advisory_lock(STATUS_LOCK_ID): try: task = Task.objects.filter(status='waiting')[0] except: return task.status = 'downloading' task.save() self.pk = task.pk
def add_block(self, cidr, who, source, why, duration=None, unblock_at=None, skip_whitelist=False, extend=True, autoscale=False): if duration: duration = expand_time(duration) now = timezone.now() if duration and not unblock_at: unblock_at = now + datetime.timedelta(seconds=duration) with advisory_lock("add_block") as acquired, transaction.atomic(): b = self.get_block(cidr) if b: if extend is False or b.unblock_at is None or (unblock_at and unblock_at <= b.unblock_at): logger.info('DUPE IP=%s', cidr) return b b.unblock_at = unblock_at BlockEntry.objects.filter(block_id=b.id).update(unblock_at=unblock_at) logger.info('EXTEND IP=%s time extended UNTIL=%s DURATION=%s', cidr, unblock_at, duration) b.save() return b if duration and autoscale: lb = self.get_last_block(cidr) if lb and lb.duration: last_duration = lb.duration and lb.duration.total_seconds() or duration scaled_duration = max(duration, self.scale_duration(lb.age.total_seconds(), last_duration)) logger.info("Scaled duration from %d to %d", duration, scaled_duration) duration = scaled_duration unblock_at = now + datetime.timedelta(seconds=duration) b = Block(cidr=cidr, who=who, source=source, why=why, added=now, unblock_at=unblock_at, skip_whitelist=skip_whitelist) b.save() #It is possible that a block is added, and then after it expires, but before it is unblocked, a new block is added for that entry. #In that case, allow the new block (since we don't know if a backend may have already unblocked the old one) #but set the old record as already unblocked. This should prevent a "block,block,unblock" timeline that results in the address #ending up not actually blocked. pending_unblock_records = BlockEntry.objects.filter(removed__isnull=True, block__cidr=cidr).all() for e in pending_unblock_records: e.set_unblocked() e.save() quoted_why = quote(why.encode('ascii', 'ignore')) logger.info('BLOCK IP=%s WHO=%s SOURCE=%s WHY=%s UNTIL="%s" DURATION=%s', cidr, who, source, quoted_why, unblock_at, duration) return b
def add_vote(obj, user): """Add a vote to an object. If the user has already voted the object nothing happends, so this function can be considered idempotent. :param obj: Any Django model instance. :param user: User adding the vote. :class:`~taiga.users.models.User` instance. """ obj_type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(obj) with advisory_lock("vote-{}-{}".format(obj_type.id, obj.id)): vote, created = Vote.objects.get_or_create(content_type=obj_type, object_id=obj.id, user=user) if not created: return votes, _ = Votes.objects.get_or_create(content_type=obj_type, object_id=obj.id) votes.count = F('count') + 1 votes.save() return vote
def remove_vote(obj, user): """Remove an user vote from an object. If the user has not voted the object nothing happens so this function can be considered idempotent. :param obj: Any Django model instance. :param user: User removing her vote. :class:`~taiga.users.models.User` instance. """ obj_type = apps.get_model("contenttypes", "ContentType").objects.get_for_model(obj) with advisory_lock("vote-{}-{}".format(obj_type.id, obj.id)): qs = Vote.objects.filter(content_type=obj_type, object_id=obj.id, user=user) if not qs.exists(): return qs.delete() votes, _ = Votes.objects.get_or_create(content_type=obj_type, object_id=obj.id) votes.count = F('count') - 1 votes.save()
def store_summoner_spell_list(result): """ Callback that stores the result of RiotWatcher static_get_summoner_list calls. Since there are only a handful of spells, we replace all spells w/the new data. """ lock_id = 'store_spells' # Convert values to match model fields. for value in result['data'].values(): value['spell_id'] = value.pop('id') value['summoner_level'] = value.pop('summonerLevel') spell_objs = list(map(lambda kwargs: SummonerSpell(**kwargs), result['data'].values())) with advisory_lock(lock_id) as acquired: SummonerSpell.objects.all().delete() SummonerSpell.objects.bulk_create(spell_objs) logger.info('Stored %s summoner spells', len(result['data'])) return result
def cancel(self): with advisory_lock("delete-user"): self.username = slugify_uniquely("deleted-user", User, slugfield="username") self.email = "{}@taiga.io".format(self.username) self.is_active = False self.full_name = "Deleted user" self.color = "" self.bio = "" self.lang = "" self.theme = "" self.timezone = "" self.colorize_tags = True self.token = None self.set_unusable_password() self.photo = None self.save() self.auth_data.all().delete() # Blocking all owned projects self.owned_projects.update(blocked_code=BLOCKED_BY_OWNER_LEAVING) # Remove all memberships self.memberships.all().delete()
def complete_model_migration(self, search_app_name, new_mapping_hash): """ Completes a migration by performing a full resync, updating aliases and removing old indices. """ search_app = get_search_app(search_app_name) if search_app.es_model.get_target_mapping_hash() != new_mapping_hash: warning_message = f"""Unexpected target mapping hash. This indicates that the task was \ generated by either a newer or an older version of the app. This could happen during a blue-green \ deployment where a new app instance creates the task and it's picked up by an old Celery instance. Rescheduling the {search_app_name} search app migration to attempt to resolve the conflict... """ logger.warning(warning_message) raise self.retry() with advisory_lock(f'leeloo-resync_after_migrate-{search_app_name}', wait=False) as lock_held: if not lock_held: logger.warning( f'Another complete_model_migration task is in progress for the {search_app_name} ' f'search app. Aborting...', ) return resync_after_migrate(search_app)
def populate_interaction_dit_participant(batch_size=5000): """ Task that creates InteractionDITParticipant instances for interactions that do not already have one. Interactions that have neither a dit_adviser nor a dit_team are skipped. Usage example: populate_interaction_dit_participant.apply_async() """ lock_name = 'leeloo-populate_interaction_dit_participant' with advisory_lock(lock_name, wait=False) as lock_held: if not lock_held: logger.warning( f'Another populate_interaction_dit_participant task is in ' f'progress. Aborting...', ) return num_processed = _populate_interaction_dit_participant( batch_size=batch_size, ) # If there are definitely no more rows needing processing, return if num_processed < batch_size: return # Schedule another task to update another batch of rows. # # This must be outside of the atomic block, otherwise it will probably run before the # current changes have been committed. # # (Similarly, the lock should also be released before the next task is scheduled.) populate_interaction_dit_participant.apply_async( kwargs={'batch_size': batch_size}, )
def cancel(self): with advisory_lock("delete-user"): deleted_user_prefix = "deleted-user-{}".format(timestamp_ms()) self.username = slugify_uniquely(deleted_user_prefix, User, slugfield="username") self.email = "{}@taiga.io".format(self.username) self.is_active = False self.full_name = "Deleted user" self.color = "" self.bio = "" self.lang = "" self.theme = "" self.timezone = "" self.colorize_tags = True self.token = None self.set_unusable_password() self.photo = None self.save() self.auth_data.all().delete() # Blocking all owned projects self.owned_projects.update(blocked_code=BLOCKED_BY_OWNER_LEAVING) # Remove all memberships self.memberships.all().delete()
def test_basic_lock(self): self.assertNumLocks(0) with advisory_lock('test') as acquired: self.assertIsNone(acquired) self.assertNumLocks(1) self.assertNumLocks(0)
def add_block(self, cidr, who, source, why, duration=None, unblock_at=None, skip_whitelist=False, extend=True, autoscale=False): if duration: duration = expand_time(duration) now = timezone.now() if duration and not unblock_at: unblock_at = now + datetime.timedelta(seconds=duration) with advisory_lock("add_block") as acquired, transaction.atomic(): b = self.get_block(cidr) if b: if extend is False or b.unblock_at is None or ( unblock_at and unblock_at <= b.unblock_at): logger.info('DUPE IP=%s', cidr) return b b.unblock_at = unblock_at BlockEntry.objects.filter(block_id=b.id).update( unblock_at=unblock_at) logger.info('EXTEND IP=%s time extended UNTIL=%s DURATION=%s', cidr, unblock_at, duration) b.save() return b if duration and autoscale: lb = self.get_last_block(cidr) if lb and lb.duration: last_duration = lb.duration and lb.duration.total_seconds( ) or duration scaled_duration = max( duration, self.scale_duration(lb.age.total_seconds(), last_duration)) logger.info("Scaled duration from %d to %d", duration, scaled_duration) duration = scaled_duration unblock_at = now + datetime.timedelta(seconds=duration) b = Block(cidr=cidr, who=who, source=source, why=why, added=now, unblock_at=unblock_at, skip_whitelist=skip_whitelist) b.save() #It is possible that a block is added, and then after it expires, but before it is unblocked, a new block is added for that entry. #In that case, allow the new block (since we don't know if a backend may have already unblocked the old one) #but set the old record as already unblocked. This should prevent a "block,block,unblock" timeline that results in the address #ending up not actually blocked. pending_unblock_records = BlockEntry.objects.filter( removed__isnull=True, block__cidr=cidr).all() for e in pending_unblock_records: e.set_unblocked() e.save() quoted_why = quote(why.encode('ascii', 'ignore')) logger.info( 'BLOCK IP=%s WHO=%s SOURCE=%s WHY=%s UNTIL="%s" DURATION=%s', cidr, who, source, quoted_why, unblock_at, duration) return b
def handle(self, *args, **options): """Execute command.""" with advisory_lock('migrations'): super().handle(*args, **options)
def create(self, request, *args, **kwargs): project_id = request.DATA.get("project", 0) with advisory_lock("issue-status-creation-{}".format(project_id)): return super().create(request, *args, **kwargs)
def create(self, request, *args, **kwargs): project_id = request.DATA.get("project", 0) with advisory_lock("issue-due-date-creation-{}".format(project_id)): return super().create(request, *args, **kwargs)
def take_snapshot(obj: object, *, comment: str = "", user=None, delete: bool = False): """ Given any model instance with registred content type, create new history entry of "change" type. This raises exception in case of object wasn't previously freezed. """ key = make_key_from_model_object(obj) with advisory_lock("history-" + key): typename = get_typename_for_model_class(obj.__class__) new_fobj = freeze_model_instance(obj) old_fobj, need_real_snapshot = get_last_snapshot_for_key(key) # migrate diff to latest schema if old_fobj: old_fobj = migrate_to_last_version(typename, old_fobj) entry_model = apps.get_model("history", "HistoryEntry") user_id = None if user is None else user.id user_name = "" if user is None else user.get_full_name() # Determine history type if delete: entry_type = HistoryType.delete need_real_snapshot = True elif new_fobj and not old_fobj: entry_type = HistoryType.create elif new_fobj and old_fobj: entry_type = HistoryType.change else: raise RuntimeError("Unexpected condition") excluded_fields = get_excluded_fields(typename) fdiff = make_diff(old_fobj, new_fobj, excluded_fields) # If diff and comment are empty, do # not create empty history entry if (not fdiff.diff and not comment and old_fobj is not None and entry_type != HistoryType.delete): return None fvals = make_diff_values(typename, fdiff) if len(comment) > 0: is_hidden = False else: is_hidden = is_hidden_snapshot(fdiff) kwargs = { "user": { "pk": user_id, "name": user_name }, "project_id": getattr(obj, 'project_id', getattr(obj, 'id', None)), "key": key, "type": entry_type, "snapshot": fdiff.snapshot if need_real_snapshot else None, "diff": fdiff.diff, "values": fvals, "comment": comment, "comment_html": mdrender(obj.project, comment), "is_hidden": is_hidden, "is_snapshot": need_real_snapshot, } return entry_model.objects.create(**kwargs)
def test_basic_lock_tuple(self): self.assertNumLocks(0) with advisory_lock(123, 456) as acquired: self.assertTrue(acquired) self.assertNumLocks(1) self.assertNumLocks(0)
def create(self, request, *args, **kwargs): epic_id = request.DATA.get("epic", 0) with advisory_lock("epic-related-user-stories-creation-{}".format(epic_id)): return super().create(request, *args, **kwargs)
def test_basic_lock_shared_no_wait(self): self.assertNumLocks(0) with advisory_lock(123, shared=True, wait=False) as acquired: self.assertTrue(acquired) self.assertNumLocks(1) self.assertNumLocks(0)
def create(self, request, *args, **kwargs): epic_id = request.DATA.get("epic", 0) with advisory_lock( "epic-related-user-stories-creation-{}".format(epic_id)): return super().create(request, *args, **kwargs)
def post(self, request, format=None, **kwargs): serializer = GuestRegisterSerializer(data=request.data) if serializer.is_valid(): lock_id = "ipam-guest-register" with advisory_lock(lock_id): hostname_prefix = CONFIG.get("GUEST_HOSTNAME_FORMAT")[0] hostname_suffix = CONFIG.get("GUEST_HOSTNAME_FORMAT")[1] last_hostname = (Host.objects.filter( hostname__istartswith=hostname_prefix, hostname__iendswith=hostname_suffix, ).extra(select={ "hostname_length": "length(hostname)" }).order_by("-hostname_length", "-hostname").first()) hostname_index = int( last_hostname.hostname[len(hostname_prefix):last_hostname. hostname.find(hostname_suffix)]) guest_user = User.objects.get( username__iexact=CONFIG.get("GUEST_USER")) user_owner = serializer.valid_ticket.user description = serializer.data.get("description") name = serializer.data.get("name") ticket = serializer.data.get("ticket") mac_address = serializer.data.get("mac_address") try: hostname = "%s%s%s" % ( hostname_prefix, hostname_index + 1, hostname_suffix, ) # Check if instance already created. Bug in DHCP thats registering it twice?? instance = Host.objects.filter(hostname=hostname, mac=mac_address).first() # Add or update host Host.objects.add_or_update_host( user=guest_user, hostname=hostname, mac=mac_address, expires=serializer.valid_ticket.ends, description=description if description else "Name: %s; Ticket used: %s" % (name, ticket), pool=Pool.objects.get(name=CONFIG.get("GUEST_POOL")), user_owners=[user_owner], group_owners=[CONFIG.get("GUEST_GROUP")], instance=instance or None, ) except ValidationError as e: error_list = [] if hasattr(e, "error_dict"): for key, errors in list(e.message_dict.items()): for error in errors: error_list.append(error) else: error_list.append(e.message) return Response( {"non_field_errors": error_list}, status=status.HTTP_400_BAD_REQUEST, ) data = { "starts": serializer.valid_ticket.starts, "ends": serializer.valid_ticket.ends, } data.update(serializer.data) return Response(data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def create(self, request, *args, **kwargs): project_id = request.DATA.get("project", 0) with advisory_lock( "epic-user-story-status-creation-{}".format(project_id)): return super().create(request, *args, **kwargs)
def test_basic_lock_int(self): self.assertNumLocks(0) with advisory_lock(123) as acquired: self.assertTrue(acquired) self.assertNumLocks(1) self.assertNumLocks(0)
def handle(self, *args, **options): """Executes the command.""" apps = get_search_apps_by_name(options['model']) with advisory_lock('migrate-opensearch-lock-id'): migrate_apps(apps)
def test_basic_lock_tuple(self): self.assertNumLocks(0) with advisory_lock((123, 456)) as acquired: self.assertTrue(acquired) self.assertNumLocks(1) self.assertNumLocks(0)
def handle(self, *args, **options): with advisory_lock("reache_pages", wait=False) as acquired: if acquired: process_scheduled_recaches() else: print("Other recache process running")
def post(self, request, format=None, **kwargs): serializer = GuestRegisterSerializer(data=request.data) if serializer.is_valid(): lock_id = "ipam-guest-register" with advisory_lock(lock_id): hostname_prefix = CONFIG.get("GUEST_HOSTNAME_FORMAT")[0] hostname_suffix = CONFIG.get("GUEST_HOSTNAME_FORMAT")[1] last_hostname = ( Host.objects.filter( hostname__istartswith=hostname_prefix, hostname__iendswith=hostname_suffix, ) .extra(select={"hostname_length": "length(hostname)"}) .order_by("-hostname_length", "-hostname") .first() ) hostname_index = int( last_hostname.hostname[ len(hostname_prefix) : last_hostname.hostname.find( hostname_suffix ) ] ) guest_user = User.objects.get(username__iexact=CONFIG.get("GUEST_USER")) user_owner = serializer.valid_ticket.user description = serializer.data.get("description") name = serializer.data.get("name") ticket = serializer.data.get("ticket") mac_address = serializer.data.get("mac_address") try: hostname = "%s%s%s" % ( hostname_prefix, hostname_index + 1, hostname_suffix, ) # Check if instance already created. Bug in DHCP thats registering it twice?? instance = Host.objects.filter( hostname=hostname, mac=mac_address ).first() # Add or update host Host.objects.add_or_update_host( user=guest_user, hostname=hostname, mac=mac_address, expires=serializer.valid_ticket.ends, description=description if description else "Name: %s; Ticket used: %s" % (name, ticket), pool=Pool.objects.get(name=CONFIG.get("GUEST_POOL")), user_owners=[user_owner], group_owners=[CONFIG.get("GUEST_GROUP")], instance=instance or None, ) except ValidationError as e: error_list = [] if hasattr(e, "error_dict"): for key, errors in e.message_dict.items(): for error in errors: error_list.append(error) else: error_list.append(e.message) return Response( {"non_field_errors": error_list}, status=status.HTTP_400_BAD_REQUEST, ) data = { "starts": serializer.valid_ticket.starts, "ends": serializer.valid_ticket.ends, } data.update(serializer.data) return Response(data, status=status.HTTP_200_OK) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)