def remove_allocation(self, id=None, group=None): if id: master = self.allocation_by_id(id) allocations = [master] allocations.extend(self.allocation_mirrors_by_master(master)) elif group: allocations = self.allocations_by_group(group, masters_only=False) else: raise NotImplementedError for allocation in allocations: assert allocation.mirror_of == self.uuid, """ Trying to delete an allocation from a different resource than the scheduler and context. This is a serious error or someone trying to something funny with the POST parameters. """ if len(allocation.reserved_slots) > 0: raise AffectedReservationError(allocation.reserved_slots[0]) if allocation.pending_reservations.count(): raise AffectedPendingReservationError( allocation.pending_reservations[0] ) for allocation in allocations: if not allocation.is_transient: Session.delete(allocation)
def test_simple_add(self): # Add one slot together with a timespan allocation = Allocation(raster=15, resource=uuid()) allocation.start = datetime(2011, 1, 1, 15) allocation.end = datetime(2011, 1, 1, 15, 59) allocation.group = uuid() reservation = uuid() slot = ReservedSlot(resource=allocation.resource) slot.start = allocation.start slot.end = allocation.end slot.allocation = allocation slot.reservation = reservation allocation.reserved_slots.append(slot) # Ensure that the same slot cannot be doubly used anotherslot = ReservedSlot(resource=allocation.resource) anotherslot.start = allocation.start anotherslot.end = allocation.end anotherslot.allocation = allocation anotherslot.reservation = reservation Session.add(anotherslot) self.assertRaises(IntegrityError, Session.flush)
def remove_expired_reservation_sessions(expiration_date=None): """ Removes all reservations from all databases which have an expired session_id. Since this only concerns 'old' sessions it shouldn't be a problem however. """ expired_sessions = find_expired_reservation_sessions(expiration_date) # remove those session ids if expired_sessions: reservations = Session.query(Reservation) reservations = reservations.filter( Reservation.session_id.in_(expired_sessions) ) slots = Session.query(ReservedSlot) slots = slots.filter( ReservedSlot.reservation_token.in_( reservations.with_entities(Reservation.token).subquery() ) ) slots.delete('fetch') reservations.delete('fetch') return expired_sessions
def remove_reservation_from_session(session_id, token): """ Removes the reservation with the given session_id and token. """ assert token and session_id query = reservations_by_session(session_id) query = query.filter(Reservation.token == token) reservation = query.one() Session.delete(reservation) # if we get here the token must be valid, we should then check if the # token is used in the reserved slots, because with autoapproval these # slots may be created straight away. slots = Session.query(ReservedSlot).filter( ReservedSlot.reservation_token == token ) slots.delete('fetch') # we also update the timestamp of existing reservations within # the same session to ensure that we account for the user's activity # properly during the session expiration cronjob. Otherwise it is # possible that a user removes the latest reservations only to see # the rest of them vanish because his older reservations were # already old enough to be counted as expired. query = Session.query(Reservation) query = query.filter(Reservation.session_id == session_id) query.update({"modified": utils.utcnow()})
def add_something(resource=None): resource = resource or uuid() allocation = Allocation(raster=15, resource=resource, mirror_of=resource) allocation.start = datetime(2011, 1, 1, 15) allocation.end = datetime(2011, 1, 1, 15, 59) allocation.group = uuid() Session.add(allocation)
def _create_allocation(self): allocation = Allocation(raster=15, resource=uuid()) allocation.start = datetime(2011, 1, 1, 15, 00) allocation.end = datetime(2011, 1, 1, 16, 00) allocation.group = str(uuid()) allocation.mirror_of = allocation.resource Session.add(allocation) return allocation
def test_dirty_protection(self): Session.flush() # should not throw an exception serialized_call(lambda: None)() Session.flush() # nothing happened, no exception serialized_call(add_something)() self.assertRaises(DirtyReadOnlySession, lambda: Session.flush)
def approve_reservation(self, token): """ This function approves an existing reservation and writes the reserved slots accordingly. Returns a list with the reserved slots. """ reservation = self.reservation_by_token(token).one() # write out the slots slots_to_reserve = [] if reservation.target_type == u'group': dates = self.dates_by_group(reservation.target) else: dates = ((reservation.start, reservation.end),) # the reservation quota is simply implemented by multiplying the # dates which are approved dates = dates * reservation.quota for start, end in dates: for allocation in self.reservation_targets(start, end): for slot_start, slot_end in \ allocation.all_slots(start, end): slot = ReservedSlot() slot.start = slot_start slot.end = slot_end slot.resource = allocation.resource slot.reservation_token = token # the slots are written with the allocation allocation.reserved_slots.append(slot) slots_to_reserve.append(slot) # the allocation may be a fake one, in which case we # must make it realz yo if allocation.is_transient: Session.add(allocation) reservation.status = u'approved' if not slots_to_reserve: raise NotReservableError notify(ReservationApprovedEvent(reservation, self.language)) return slots_to_reserve
def deny_reservation(self, token): """ Denies a pending reservation, removing it from the records and sending an email to the reservee. """ query = self.reservation_by_token(token) query.filter(Reservation.status == u'pending') reservation = query.one() Session.delete(reservation) notify(ReservationDeniedEvent(reservation, self.language))
def reservations_by_session(session_id): # be sure to not query for all reservations. since a query should be # returned in any case we just use an impossible clause # this is mainly a security feature if not session_id: log.warn('Empty session id') return Session.query(Reservation).filter("0=1") query = Session.query(Reservation) query = query.filter(Reservation.session_id == session_id) query = query.order_by(Reservation.created) return query
def remove_reservation(self, token): """ Removes all reserved slots of the given reservation token. Note that removing a reservation does not let the reservee know that his reservation has been removed. If you want to let the reservee know what happened, use revoke_reservation. """ slots = self.reserved_slots_by_reservation(token) for slot in slots: Session.delete(slot) self.reservation_by_token(token).delete()
def test_simple_add(self): # Test a simple add allocation = Allocation(raster=15, resource=uuid()) allocation.start = datetime(2011, 1, 1, 15) allocation.end = datetime(2011, 1, 1, 15, 59) allocation.group = str(uuid()) allocation.mirror_of = allocation.resource Session.add(allocation) self.assertEqual(Session.query(Allocation).count(), 1) # Test failing add allocation = Allocation(raster=15) Session.add(allocation) self.assertRaises(IntegrityError, Session.flush)
def managed_reserved_slots(self): """ The reserved_slots managed by this scheduler / resource. """ uuids = self.managed_allocations().with_entities(Allocation.resource) query = Session.query(ReservedSlot) query = query.filter(ReservedSlot.resource.in_(uuids)) return query
def in_group(self): """True if the event is in any group.""" query = Session.query(Allocation.id) query = query.filter(Allocation.resource == self.resource) query = query.filter(Allocation.group == self.group) query = query.limit(2) return len(query.all()) > 1
def upgrade_1017_to_1018(context): # seantis.reservation before 1.0.12 left behind reserved slots when # removing reservations of expired sessions. These need to be cleaned for # the allocation usage to be right. # all slots need a connected reservation all_reservations = Session.query(Reservation) # orphan slots are therefore all slots.. orphan_slots = Session.query(ReservedSlot) # ..with tokens not found in the reservations table orphan_slots = orphan_slots.filter( not_(ReservedSlot.reservation_token.in_(all_reservations.with_entities(Reservation.token).subquery())) ) log.info("Removing {} reserved slots with no linked reservations".format(orphan_slots.count())) orphan_slots.delete("fetch")
def pending_reservations(self): """ Returns the pending reservations query for this allocation. As the pending reservations target the group and not a specific allocation this function returns the same value for masters and mirrors. """ Reservation = self.models.Reservation query = Session.query(Reservation.id) query = query.filter(Reservation.target == self.group) query = query.filter(Reservation.status == u"pending") return query
def fetch_records(resources): """ Returns the records used for the dataset. """ query = Session.query(Reservation) query = query.filter(Reservation.resource.in_(resources.keys())) query = query.order_by( Reservation.resource, Reservation.status, Reservation.start, Reservation.email, Reservation.token, ) return query.all()
def all_allocations_in_range(start, end): # Query version of utils.overlaps return Session.query(Allocation).filter( or_( and_( Allocation._start <= start, start <= Allocation._end ), and_( start <= Allocation._start, Allocation._start <= end ) ) )
def latest_reservations(resources, daterange, reservations='*'): query = Session().query(Reservation) query = query.filter(Reservation.resource.in_(resources.keys())) query = query.filter(Reservation.created > daterange[0]) query = query.filter(Reservation.created <= daterange[1]) query = query.order_by(desc(Reservation.created)) if reservations != '*': query = query.filter(Reservation.token.in_(reservations)) result = utils.OrderedDict() for reservation in query.all(): if reservation.token in result: result[reservation.token].append(reservation) else: result[reservation.token] = [reservation] return result
def _target_allocations(self): """ Returns the allocations this reservation is targeting. This should NOT be confused with db.allocations_by_reservation. The method in the db module returns the actual allocations belonging to an approved reservation. This method only returns the master allocations to get information about timespans and other properties. If you don't know exactly what you're doing you do not want to use this method as misuse might be dangerous. """ Allocation = self.models.Allocation query = Session.query(Allocation) query = query.filter(Allocation.group == self.target) # master allocations only query = query.filter(Allocation.resource == Allocation.mirror_of) return query
def siblings(self, imaginary=True): """Returns the master/mirrors group this allocation is part of. If 'imaginary' is true, inexistant mirrors are created on the fly. those mirrors are transient (see self.is_transient) """ # this function should always have itself in the result if not imaginary and self.is_transient: assert False, \ 'the resulting list would not contain this allocation' if self.quota == 1: assert(self.is_master) return [self] query = Session.query(Allocation) query = query.filter(Allocation.mirror_of == self.mirror_of) query = query.filter(Allocation._start == self._start) existing = dict(((e.resource, e) for e in query)) master = self.is_master and self or existing[self.mirror_of] existing[master.resource] = master uuids = utils.generate_uuids(master.resource, master.quota) imaginary = imaginary and (master.quota - len(existing)) or 0 siblings = [master] for uuid in uuids: if uuid in existing: siblings.append(existing[uuid]) elif imaginary > 0: allocation = master.copy() allocation.resource = uuid siblings.append(allocation) imaginary -= 1 return siblings
def find_expired_reservation_sessions(expiration_date): """ Goes through all reservations and returns the session ids of the unconfirmed ones which are older than the given expiration date. By default the expiration date is now - 15 minutes. Note that this method goes through ALL RESERVATIONS OF THE DATABASE. If this is not desired have a look at buildout/database.cfg.example to setup each site with its own database. """ expiration_date = expiration_date or ( utils.utcnow() - timedelta(minutes=15) ) # first get the session ids which are expired query = Session.query( Reservation.session_id, func.max(Reservation.created), func.max(Reservation.modified) ) query = query.group_by(Reservation.session_id) # != null() because != None is not allowed by PEP8 query = query.filter(Reservation.session_id != null()) # the idea is to remove all reservations belonging to sessions whose # latest update is expired - either delete the whole session or let # all of it be expired_sessions = [] for session_id, created, modified in query.all(): modified = modified or created assert created and modified if max(created, modified) < expiration_date: expired_sessions.append(session_id) return expired_sessions
def latest_reservations(resources, reservations='*', days=30): schedulers = {} for uuid in resources.keys(): schedulers[uuid] = db.Scheduler(uuid) since = utils.utcnow() - timedelta(days=days) query = Session.query(Reservation) query = query.filter(Reservation.resource.in_(resources.keys())) query = query.filter(Reservation.created > since) query = query.order_by(desc(Reservation.created)) if reservations != '*': query = query.filter(Reservation.token.in_(reservations)) result = utils.OrderedDict() for reservation in query.all(): if reservation.token in result: result.append(reservation) else: result[reservation.token] = [reservation] return result
def _approved_timespans(self, start, end): ReservedSlot = self.models.ReservedSlot query = Session.query(ReservedSlot)\ .filter_by(reservation_token=self.token) if start: query = query.filter(ReservedSlot.start >= start) if end: query = query.filter(ReservedSlot.end <= end) # find the slots that are still reserved result = [] for start, end in self.target_dates(): reserved_slot = query.filter(ReservedSlot.start <= end)\ .filter(ReservedSlot.end >= start)\ .first() if not reserved_slot: continue # build tuple containing necessary info for deletion links timespan = Timespan(start=start, end=end + timedelta(microseconds=1), allocation_id=reserved_slot.allocation_id, token=self.token) result.append(timespan) return result
def fetch_records(resources, year, month): """ Returns the records used for the dataset. """ if not resources.keys(): return [] query = Session().query(Reservation) query = query.filter(Reservation.resource.in_(resources.keys())) if year != 'all': query = query.filter(extract('year', Reservation.start) == int(year)) if month != 'all': query = query.filter(extract('month', Reservation.start) == int(month)) query = query.order_by( Reservation.resource, Reservation.status, Reservation.start, Reservation.email, Reservation.token, ) return query.all()
def _query_blocked_periods(self): query = Session.query(self.models.BlockedPeriod) query = query.filter_by(resource=self.resource) return query
def read_allocation(): allocation = Session.query(Allocation).one() allocation.resource
def drop(): Session.query(Allocation).delete() transaction.commit()
def reserve(self, email, dates=None, group=None, data=None, session_id=None, quota=1): """ First step of the reservation. Seantis.reservation uses a two-step reservation process. The first step is reserving what is either an open spot or a place on the waiting list. The second step is to actually write out the reserved slots, which is done by approving an existing reservation. Most checks are done in the reserve functions. The approval step only fails if there's no open spot. This function returns a reservation token which can be used to approve the reservation in approve_reservation. """ assert (dates or group) and not (dates and group) validate_email(email) if group: dates = self.dates_by_group(group) dates = utils.pairs(dates) # First, the request is checked for saneness. If any requested # date cannot be reserved the request as a whole fails. for start, end in dates: # are the parameters valid? if abs((end - start).days) >= 1: raise ReservationTooLong if start > end or (end - start).seconds < 5 * 60: raise ReservationParametersInvalid # can all allocations be reserved? for allocation in self.allocations_in_range(start, end): # start and end are not rasterized, so we need this check if not allocation.overlaps(start, end): continue assert allocation.is_master # with manual approval the reservation ends up on the # waitinglist and does not yet need a spot if not allocation.approve_manually: if not self.find_spot(allocation, start, end): raise AlreadyReservedError free = self.free_allocations_count(allocation, start, end) if free < quota: raise AlreadyReservedError if allocation.reservation_quota_limit > 0: if allocation.reservation_quota_limit < quota: raise QuotaOverLimit if allocation.quota < quota: raise QuotaImpossible if quota < 1: raise InvalidQuota # ok, we're good to go token = new_uuid() found = 0 # groups are reserved by group-identifier - so all members of a group # or none of them. As such there's no start / end date which is defined # implicitly by the allocation if group: found = 1 reservation = Reservation() reservation.token = token reservation.target = group reservation.status = u'pending' reservation.target_type = u'group' reservation.resource = self.uuid reservation.data = data reservation.session_id = session_id reservation.email = email reservation.quota = quota Session.add(reservation) else: groups = [] for start, end in dates: for allocation in self.allocations_in_range(start, end): if not allocation.overlaps(start, end): continue found += 1 reservation = Reservation() reservation.token = token reservation.start, reservation.end = rasterize_span( start, end, allocation.raster ) reservation.target = allocation.group reservation.status = u'pending' reservation.target_type = u'allocation' reservation.resource = self.uuid reservation.data = data reservation.session_id = session_id reservation.email = email reservation.quota = quota Session.add(reservation) groups.append(allocation.group) # check if no group reservation is made with this request. # reserve by group in this case (or make this function # do that automatically) assert len(groups) == len(set(groups)), \ 'wrongly trying to reserve a group' if found: notify(ReservationMadeEvent(reservation, self.language)) else: raise InvalidReservationError return token
def change_quota(self, master, new_quota): """ Changes the quota of a master allocation. Fails if the quota is already exhausted. When the quota is decreased a reorganization of the mirrors is triggered. Reorganizing means eliminating gaps in the chain of mirrors that emerge when reservations are removed: Initial State: 1 (master) Free 2 (mirror) Free 3 (mirror) Free Reservations are made: 1 (master) Reserved 2 (mirror) Reserved 3 (mirror) Reserved A reservation is deleted: 1 (master) Reserved 2 (mirror) Free <-- !! 3 (mirror) Reserved Reorganization is performed: 1 (master) Reserved 2 (mirror) Reserved <-- !! 3 (mirror) Free <-- !! The quota is decreased: 1 (master) Reserved 2 (mirror) Reserved In other words, the reserved allocations are moved to the beginning, the free allocations moved at the end. This is done to ensure that the sequence of generated uuids for the mirrors always represent all possible keys. Without the reorganization we would see the following after decreasing the quota: The quota is decreased: 1 (master) Reserved 3 (mirror) Reserved This would make it impossible to calculate the mirror keys. Instead the existing keys would have to queried from the database. """ assert new_quota > 0, "Quota must be greater than 0" if new_quota == master.quota: return if new_quota > master.quota: master.quota = new_quota return # Make sure that the quota can be decreased mirrors = self.allocation_mirrors_by_master(master) allocations = [master] + mirrors free_allocations = [a for a in allocations if a.is_available()] required = master.quota - new_quota if len(free_allocations) < required: raise AffectedReservationError(None) # get a map pointing from the existing uuid to the newly assigned uuid reordered = self.reordered_keylist(allocations, new_quota) # unused keys are the ones not present in the newly assignd uuid list unused = set(reordered.keys()) - set(reordered.values()) - set((None,)) # get a map for resource_uuid -> allocation.id ids = dict(((a.resource, a.id) for a in allocations)) for allocation in allocations: # change the quota for all allocations allocation.quota = new_quota # the value is None if the allocation is not mapped to a new uuid new_resource = reordered[allocation.resource] if not new_resource: continue # move all slots to the mapped allocation id new_id = ids[new_resource] for slot in allocation.reserved_slots: # build a query here as the manipulation of mapped objects in # combination with the delete query below seems a bit # unpredictable given the cascading of changes query = Session.query(ReservedSlot) query = query.filter(and_( ReservedSlot.resource == slot.resource, ReservedSlot.allocation_id == slot.allocation_id, ReservedSlot.start == slot.start )) query.update( { ReservedSlot.resource: new_resource, ReservedSlot.allocation_id: new_id } ) # get rid of the unused allocations (always preserving the master) if unused: query = Session.query(Allocation) query = query.filter(Allocation.resource.in_(unused)) query = query.filter(Allocation.id != master.id) query = query.filter(Allocation._start == master._start) query.delete('fetch')
def allocate(self, dates, raster=15, quota=None, partly_available=False, grouped=False, approve_manually=True, reservation_quota_limit=0, whole_day=False ): """Allocates a spot in the calendar. An allocation defines a timerange which can be reserved. No reservations can exist outside of existing allocations. In fact any reserved slot will link to an allocation. An allocation may be available as a whole (to reserve all or nothing). It may also be partly available which means reservations can be made for parts of the allocation. If an allocation is partly available a raster defines the granularity with which a reservation can be made (e.g. a raster of 15min will ensure that reservations are at least 15 minutes long and start either at :00, :15, :30 or :45) The reason for the raster is mainly to ensure that different reservations trying to reserve overlapping times need the same keys in the reserved_slots table, ensuring integrity at the database level. Allocations may have a quota, which determines how many times an allocation may be reserved. Quotas are enabled using a master-mirrors relationship. The master is the first allocation to be created. The mirrors copies of that allocation. See Scheduler.__doc__ """ dates = utils.pairs(dates) group = new_uuid() quota = quota or 1 # if the allocation is not partly available the raster is set to lowest # possible raster value raster = partly_available and raster or MIN_RASTER_VALUE # the whole day option results in the dates being aligned to # the beginning of the day / end of it -> not timezone aware! if whole_day: for ix, (start, end) in enumerate(dates): dates[ix] = utils.align_range_to_day(start, end) # Ensure that the list of dates contains no overlaps inside for start, end in dates: if utils.count_overlaps(dates, start, end) > 1: raise InvalidAllocationError # Make sure that this span does not overlap another master for start, end in dates: start, end = rasterize_span(start, end, raster) existing = self.allocations_in_range(start, end).first() if existing: raise OverlappingAllocationError(start, end, existing) # Write the master allocations allocations = [] for start, end in dates: allocation = Allocation(raster=raster) allocation.start = start allocation.end = end allocation.resource = self.uuid allocation.quota = quota allocation.mirror_of = self.uuid allocation.partly_available = partly_available allocation.approve_manually = approve_manually allocation.reservation_quota_limit = reservation_quota_limit if grouped: allocation.group = group else: allocation.group = new_uuid() allocations.append(allocation) Session.add_all(allocations) return allocations
def monthly_report(year, month, resources, reservations='*'): schedulers, titles = dict(), dict() for uuid in resources.keys(): schedulers[uuid] = db.Scheduler(uuid) titles[uuid] = utils.get_resource_title(resources[uuid]) # this order is used for every day in the month ordered_uuids = [i[0] for i in sorted(titles.items(), key=lambda i: i[1])] # build the hierarchical structure of the report data report = utils.OrderedDict() last_day = 28 for d in sorted((d for d in calendar.itermonthdates(year, month))): if not d.month == month: continue day = d.day last_day = max(last_day, day) report[day] = utils.OrderedDict() for uuid in ordered_uuids: report[day][uuid] = dict() report[day][uuid][u'title'] = titles[uuid] report[day][uuid][u'approved'] = list() report[day][uuid][u'pending'] = list() report[day][uuid][u'url'] = resources[uuid].absolute_url() report[day][uuid][u'lists'] = { u'approved': _(u'Approved'), u'pending': _(u'Pending'), } # gather the reservations with as much bulk loading as possible period_start = date(year, month, 1) period_end = date(year, month, last_day) # get a list of relevant allocations in the given period query = Session.query(Allocation) query = query.filter(period_start <= Allocation._start) query = query.filter(Allocation._start <= period_end) query = query.filter(Allocation.resource == Allocation.mirror_of) query = query.filter(Allocation.resource.in_(resources.keys())) allocations = query.all() # quit if there are no allocations at this point if not allocations: return {} # store by group as it will be needed multiple times over later groups = dict() for allocation in allocations: groups.setdefault(allocation.group, list()).append(allocation) # using the groups get the relevant reservations query = Session.query(Reservation) query = query.filter(Reservation.target.in_(groups.keys())) if reservations != '*': query = query.filter(Reservation.token.in_(reservations)) query = query.order_by(Reservation.status) reservations = query.all() reservation_urls = ReservationUrls() @utils.memoize def json_timespans(start, end): return json.dumps([dict(start=start, end=end)]) used_days = dict([(i, False) for i in range(1, 32)]) def add_reservation(start, end, reservation): day = start.day used_days[day] = True end += timedelta(microseconds=1) start, end = start.strftime('%H:%M'), end.strftime('%H:%M') context = resources[utils.string_uuid(reservation.resource)] if reservation.status == u'approved': rightside_urls = [( _(u'Delete'), reservation_urls.revoke_all_url(reservation.token, context) )] elif reservation.status == u'pending': rightside_urls = [ ( _(u'Approve'), reservation_urls.approve_all_url( reservation.token, context ) ), ( _(u'Deny'), reservation_urls.deny_all_url(reservation.token, context) ), ] else: raise NotImplementedError reservation_lists = report[day][utils.string_uuid( reservation.resource )] reservation_lists[reservation.status].append( dict( start=start, end=end, email=reservation.email, data=reservation.data, timespans=json_timespans(start, end), rightside_urls=rightside_urls, token=reservation.token, quota=utils.get_reservation_quota_statement(reservation.quota), resource=context ) ) for reservation in reservations: if reservation.target_type == u'allocation': add_reservation(reservation.start, reservation.end, reservation) else: for allocation in groups[reservation.target]: add_reservation(allocation.start, allocation.end, reservation) # remove unused days for day in report: if not used_days[day]: del report[day] return report
def managed_allocations(self): """ The allocations managed by this scheduler / resource. """ query = Session.query(Allocation) query = query.filter(Allocation.mirror_of == self.uuid) return query
def managed_reservations(self): """ The reservations managed by this scheduler / resource. """ query = Session.query(Reservation) query = query.filter(Reservation.resource == self.uuid) return query
def change_allocation(): allocation = Session.query(Allocation).one() allocation.group = uuid()