def remove_reservation_from_session(session_id, token):
""" Removes the reservation with the given session_id and token. """
assert token and session_id
query = reservations_by_session(session_id)
query = query.filter(Reservation.token == token)
reservation = query.one()
Session.delete(reservation)
# if we get here the token must be valid, we should then check if the
# token is used in the reserved slots, because with autoapproval these
# slots may be created straight away.
slots = Session.query(ReservedSlot).filter(
ReservedSlot.reservation_token == token
)
slots.delete('fetch')
# we also update the timestamp of existing reservations within
# the same session to ensure that we account for the user's activity
# properly during the session expiration cronjob. Otherwise it is
# possible that a user removes the latest reservations only to see
# the rest of them vanish because his older reservations were
# already old enough to be counted as expired.
query = Session.query(Reservation)
query = query.filter(Reservation.session_id == session_id)
query.update({"modified": utils.utcnow()})
def remove_expired_reservation_sessions(expiration_date=None):
""" Removes all reservations from all databases which have an
expired session_id.
Since this only concerns 'old' sessions it shouldn't be a problem
however.
"""
expired_sessions = find_expired_reservation_sessions(expiration_date)
# remove those session ids
if expired_sessions:
reservations = Session.query(Reservation)
reservations = reservations.filter(
Reservation.session_id.in_(expired_sessions)
)
slots = Session.query(ReservedSlot)
slots = slots.filter(
ReservedSlot.reservation_token.in_(
reservations.with_entities(Reservation.token).subquery()
)
)
slots.delete('fetch')
reservations.delete('fetch')
return expired_sessions
def reservations_by_session(session_id):
# be sure to not query for all reservations. since a query should be
# returned in any case we just use an impossible clause
# this is mainly a security feature
if not session_id:
log.warn('Empty session id')
return Session.query(Reservation).filter("0=1")
query = Session.query(Reservation)
query = query.filter(Reservation.session_id == session_id)
query = query.order_by(Reservation.created)
return query
def managed_reserved_slots(self):
""" The reserved_slots managed by this scheduler / resource. """
uuids = self.managed_allocations().with_entities(Allocation.resource)
query = Session.query(ReservedSlot)
query = query.filter(ReservedSlot.resource.in_(uuids))
return query
def in_group(self):
"""True if the event is in any group."""
query = Session.query(Allocation.id)
query = query.filter(Allocation.resource == self.resource)
query = query.filter(Allocation.group == self.group)
query = query.limit(2)
return len(query.all()) > 1
def in_group(self):
"""True if the event is in any group."""
query = Session.query(Allocation.id)
query = query.filter(Allocation.resource == self.resource)
query = query.filter(Allocation.group == self.group)
query = query.limit(2)
return len(query.all()) > 1
def upgrade_1017_to_1018(context):
# seantis.reservation before 1.0.12 left behind reserved slots when
# removing reservations of expired sessions. These need to be cleaned for
# the allocation usage to be right.
# all slots need a connected reservation
all_reservations = Session.query(Reservation)
# orphan slots are therefore all slots..
orphan_slots = Session.query(ReservedSlot)
# ..with tokens not found in the reservations table
orphan_slots = orphan_slots.filter(
not_(ReservedSlot.reservation_token.in_(all_reservations.with_entities(Reservation.token).subquery()))
)
log.info("Removing {} reserved slots with no linked reservations".format(orphan_slots.count()))
orphan_slots.delete("fetch")
def fetch_records(resources):
""" Returns the records used for the dataset. """
query = Session.query(Reservation)
query = query.filter(Reservation.resource.in_(resources.keys()))
query = query.order_by(
Reservation.resource,
Reservation.status,
Reservation.start,
Reservation.email,
Reservation.token,
)
return query.all()
def upgrade_1017_to_1018(context):
# seantis.reservation before 1.0.12 left behind reserved slots when
# removing reservations of expired sessions. These need to be cleaned for
# the allocation usage to be right.
# all slots need a connected reservation
all_reservations = Session.query(Reservation)
# orphan slots are therefore all slots..
orphan_slots = Session.query(ReservedSlot)
# ..with tokens not found in the reservations table
orphan_slots = orphan_slots.filter(
not_(
ReservedSlot.reservation_token.in_(
all_reservations.with_entities(Reservation.token).subquery())))
log.info('Removing {} reserved slots with no linked reservations'.format(
orphan_slots.count()))
orphan_slots.delete('fetch')
def pending_reservations(self):
""" Returns the pending reservations query for this allocation.
As the pending reservations target the group and not a specific
allocation this function returns the same value for masters and
mirrors.
"""
Reservation = self.models.Reservation
query = Session.query(Reservation.id)
query = query.filter(Reservation.target == self.group)
query = query.filter(Reservation.status == u'pending')
return query
def pending_reservations(self):
""" Returns the pending reservations query for this allocation.
As the pending reservations target the group and not a specific
allocation this function returns the same value for masters and
mirrors.
"""
Reservation = self.models.Reservation
query = Session.query(Reservation.id)
query = query.filter(Reservation.target == self.group)
query = query.filter(Reservation.status == u"pending")
return query
def all_allocations_in_range(start, end):
# Query version of utils.overlaps
return Session.query(Allocation).filter(
or_(
and_(
Allocation._start <= start,
start <= Allocation._end
),
and_(
start <= Allocation._start,
Allocation._start <= end
)
)
)
def test_simple_add(self):
# Test a simple add
allocation = Allocation(raster=15, resource=uuid())
allocation.start = datetime(2011, 1, 1, 15)
allocation.end = datetime(2011, 1, 1, 15, 59)
allocation.group = str(uuid())
allocation.mirror_of = allocation.resource
Session.add(allocation)
self.assertEqual(Session.query(Allocation).count(), 1)
# Test failing add
allocation = Allocation(raster=15)
Session.add(allocation)
self.assertRaises(IntegrityError, Session.flush)
def test_simple_add(self):
# Test a simple add
allocation = Allocation(raster=15, resource=uuid())
allocation.start = datetime(2011, 1, 1, 15)
allocation.end = datetime(2011, 1, 1, 15, 59)
allocation.group = str(uuid())
allocation.mirror_of = allocation.resource
Session.add(allocation)
self.assertEqual(Session.query(Allocation).count(), 1)
# Test failing add
allocation = Allocation(raster=15)
Session.add(allocation)
self.assertRaises(IntegrityError, Session.flush)
def _target_allocations(self):
""" Returns the allocations this reservation is targeting. This should
NOT be confused with db.allocations_by_reservation. The method in
the db module returns the actual allocations belonging to an approved
reservation.
This method only returns the master allocations to get information
about timespans and other properties. If you don't know exactly
what you're doing you do not want to use this method as misuse might
be dangerous.
"""
Allocation = self.models.Allocation
query = Session.query(Allocation)
query = query.filter(Allocation.group == self.target)
# master allocations only
query = query.filter(Allocation.resource == Allocation.mirror_of)
return query
def siblings(self, imaginary=True):
"""Returns the master/mirrors group this allocation is part of.
If 'imaginary' is true, inexistant mirrors are created on the fly.
those mirrors are transient (see self.is_transient)
"""
# this function should always have itself in the result
if not imaginary and self.is_transient:
assert False, \
'the resulting list would not contain this allocation'
if self.quota == 1:
assert(self.is_master)
return [self]
query = Session.query(Allocation)
query = query.filter(Allocation.mirror_of == self.mirror_of)
query = query.filter(Allocation._start == self._start)
existing = dict(((e.resource, e) for e in query))
master = self.is_master and self or existing[self.mirror_of]
existing[master.resource] = master
uuids = utils.generate_uuids(master.resource, master.quota)
imaginary = imaginary and (master.quota - len(existing)) or 0
siblings = [master]
for uuid in uuids:
if uuid in existing:
siblings.append(existing[uuid])
elif imaginary > 0:
allocation = master.copy()
allocation.resource = uuid
siblings.append(allocation)
imaginary -= 1
return siblings
def siblings(self, imaginary=True):
"""Returns the master/mirrors group this allocation is part of.
If 'imaginary' is true, inexistant mirrors are created on the fly.
those mirrors are transient (see self.is_transient)
"""
# this function should always have itself in the result
if not imaginary and self.is_transient:
assert False, \
'the resulting list would not contain this allocation'
if self.quota == 1:
assert(self.is_master)
return [self]
query = Session.query(Allocation)
query = query.filter(Allocation.mirror_of == self.mirror_of)
query = query.filter(Allocation._start == self._start)
existing = dict(((e.resource, e) for e in query))
master = self.is_master and self or existing[self.mirror_of]
existing[master.resource] = master
uuids = utils.generate_uuids(master.resource, master.quota)
imaginary = imaginary and (master.quota - len(existing)) or 0
siblings = [master]
for uuid in uuids:
if uuid in existing:
siblings.append(existing[uuid])
elif imaginary > 0:
allocation = master.copy()
allocation.resource = uuid
siblings.append(allocation)
imaginary -= 1
return siblings
def find_expired_reservation_sessions(expiration_date):
""" Goes through all reservations and returns the session ids of the
unconfirmed ones which are older than the given expiration date.
By default the expiration date is now - 15 minutes.
Note that this method goes through ALL RESERVATIONS OF THE DATABASE. If
this is not desired have a look at buildout/database.cfg.example to
setup each site with its own database.
"""
expiration_date = expiration_date or (
utils.utcnow() - timedelta(minutes=15)
)
# first get the session ids which are expired
query = Session.query(
Reservation.session_id,
func.max(Reservation.created),
func.max(Reservation.modified)
)
query = query.group_by(Reservation.session_id)
# != null() because != None is not allowed by PEP8
query = query.filter(Reservation.session_id != null())
# the idea is to remove all reservations belonging to sessions whose
# latest update is expired - either delete the whole session or let
# all of it be
expired_sessions = []
for session_id, created, modified in query.all():
modified = modified or created
assert created and modified
if max(created, modified) < expiration_date:
expired_sessions.append(session_id)
return expired_sessions
def latest_reservations(resources, reservations='*', days=30):
schedulers = {}
for uuid in resources.keys():
schedulers[uuid] = db.Scheduler(uuid)
since = utils.utcnow() - timedelta(days=days)
query = Session.query(Reservation)
query = query.filter(Reservation.resource.in_(resources.keys()))
query = query.filter(Reservation.created > since)
query = query.order_by(desc(Reservation.created))
if reservations != '*':
query = query.filter(Reservation.token.in_(reservations))
result = utils.OrderedDict()
for reservation in query.all():
if reservation.token in result:
result.append(reservation)
else:
result[reservation.token] = [reservation]
return result
def _approved_timespans(self, start, end):
ReservedSlot = self.models.ReservedSlot
query = Session.query(ReservedSlot)\
.filter_by(reservation_token=self.token)
if start:
query = query.filter(ReservedSlot.start >= start)
if end:
query = query.filter(ReservedSlot.end <= end)
# find the slots that are still reserved
result = []
for start, end in self.target_dates():
reserved_slot = query.filter(ReservedSlot.start <= end)\
.filter(ReservedSlot.end >= start)\
.first()
if not reserved_slot:
continue
# build tuple containing necessary info for deletion links
timespan = Timespan(start=start,
end=end + timedelta(microseconds=1),
allocation_id=reserved_slot.allocation_id,
token=self.token)
result.append(timespan)
return result
def latest_reservations(resources, reservations='*', days=30):
schedulers = {}
for uuid in resources.keys():
schedulers[uuid] = db.Scheduler(uuid)
since = utils.utcnow() - timedelta(days=days)
query = Session.query(Reservation)
query = query.filter(Reservation.resource.in_(resources.keys()))
query = query.filter(Reservation.created > since)
query = query.order_by(desc(Reservation.created))
if reservations != '*':
query = query.filter(Reservation.token.in_(reservations))
result = utils.OrderedDict()
for reservation in query.all():
if reservation.token in result:
result.append(reservation)
else:
result[reservation.token] = [reservation]
return result
def drop():
Session.query(Allocation).delete()
transaction.commit()
def _query_blocked_periods(self):
query = Session.query(self.models.BlockedPeriod)
query = query.filter_by(resource=self.resource)
return query
def monthly_report(year, month, resources, reservations="*"):
schedulers, titles = dict(), dict()
for uuid in resources.keys():
schedulers[uuid] = db.Scheduler(uuid)
titles[uuid] = utils.get_resource_title(resources[uuid])
# this order is used for every day in the month
ordered_uuids = [i[0] for i in sorted(titles.items(), key=lambda i: i[1])]
# build the hierarchical structure of the report data
report = utils.OrderedDict()
last_day = 28
for d in sorted((d for d in calendar.itermonthdates(year, month))):
if not d.month == month:
continue
day = d.day
last_day = max(last_day, day)
report[day] = utils.OrderedDict()
for uuid in ordered_uuids:
report[day][uuid] = dict()
report[day][uuid][u"title"] = titles[uuid]
report[day][uuid][u"approved"] = list()
report[day][uuid][u"pending"] = list()
report[day][uuid][u"url"] = resources[uuid].absolute_url()
report[day][uuid][u"lists"] = {u"approved": _(u"Approved"), u"pending": _(u"Pending")}
# gather the reservations with as much bulk loading as possible
period_start = date(year, month, 1)
period_end = date(year, month, last_day)
# get a list of relevant allocations in the given period
query = Session.query(Allocation)
query = query.filter(period_start <= Allocation._start)
query = query.filter(Allocation._start <= period_end)
query = query.filter(Allocation.resource == Allocation.mirror_of)
query = query.filter(Allocation.resource.in_(resources.keys()))
allocations = query.all()
# quit if there are no allocations at this point
if not allocations:
return {}
# store by group as it will be needed multiple times over later
groups = dict()
for allocation in allocations:
groups.setdefault(allocation.group, list()).append(allocation)
# using the groups get the relevant reservations
query = Session.query(Reservation)
query = query.filter(Reservation.target.in_(groups.keys()))
if reservations != "*":
query = query.filter(Reservation.token.in_(reservations))
query = query.order_by(Reservation.status)
reservations = query.all()
reservation_urls = ReservationUrls()
@utils.memoize
def json_timespans(start, end):
return json.dumps([dict(start=start, end=end)])
used_days = dict([(i, False) for i in range(1, 32)])
def add_reservation(start, end, reservation):
day = start.day
used_days[day] = True
end += timedelta(microseconds=1)
start, end = start.strftime("%H:%M"), end.strftime("%H:%M")
context = resources[utils.string_uuid(reservation.resource)]
if reservation.status == u"approved":
rightside_urls = [(_(u"Delete"), reservation_urls.revoke_all_url(reservation.token, context))]
elif reservation.status == u"pending":
rightside_urls = [
(_(u"Approve"), reservation_urls.approve_all_url(reservation.token, context)),
(_(u"Deny"), reservation_urls.deny_all_url(reservation.token, context)),
]
else:
raise NotImplementedError
reservation_lists = report[day][utils.string_uuid(reservation.resource)]
reservation_lists[reservation.status].append(
dict(
start=start,
end=end,
email=reservation.email,
data=reservation.data,
timespans=json_timespans(start, end),
rightside_urls=rightside_urls,
token=reservation.token,
quota=utils.get_reservation_quota_statement(reservation.quota),
resource=context,
)
)
for reservation in reservations:
if reservation.target_type == u"allocation":
add_reservation(reservation.start, reservation.end, reservation)
else:
for allocation in groups[reservation.target]:
add_reservation(allocation.start, allocation.end, reservation)
# remove unused days
for day in report:
if not used_days[day]:
del report[day]
return report
def change_quota(self, master, new_quota):
""" Changes the quota of a master allocation.
Fails if the quota is already exhausted.
When the quota is decreased a reorganization of the mirrors is
triggered. Reorganizing means eliminating gaps in the chain of mirrors
that emerge when reservations are removed:
Initial State:
1 (master) Free
2 (mirror) Free
3 (mirror) Free
Reservations are made:
1 (master) Reserved
2 (mirror) Reserved
3 (mirror) Reserved
A reservation is deleted:
1 (master) Reserved
2 (mirror) Free <-- !!
3 (mirror) Reserved
Reorganization is performed:
1 (master) Reserved
2 (mirror) Reserved <-- !!
3 (mirror) Free <-- !!
The quota is decreased:
1 (master) Reserved
2 (mirror) Reserved
In other words, the reserved allocations are moved to the beginning,
the free allocations moved at the end. This is done to ensure that
the sequence of generated uuids for the mirrors always represent all
possible keys.
Without the reorganization we would see the following after
decreasing the quota:
The quota is decreased:
1 (master) Reserved
3 (mirror) Reserved
This would make it impossible to calculate the mirror keys. Instead the
existing keys would have to queried from the database.
"""
assert new_quota > 0, "Quota must be greater than 0"
if new_quota == master.quota:
return
if new_quota > master.quota:
master.quota = new_quota
return
# Make sure that the quota can be decreased
mirrors = self.allocation_mirrors_by_master(master)
allocations = [master] + mirrors
free_allocations = [a for a in allocations if a.is_available()]
required = master.quota - new_quota
if len(free_allocations) < required:
raise AffectedReservationError(None)
# get a map pointing from the existing uuid to the newly assigned uuid
reordered = self.reordered_keylist(allocations, new_quota)
# unused keys are the ones not present in the newly assignd uuid list
unused = set(reordered.keys()) - set(reordered.values()) - set((None,))
# get a map for resource_uuid -> allocation.id
ids = dict(((a.resource, a.id) for a in allocations))
for allocation in allocations:
# change the quota for all allocations
allocation.quota = new_quota
# the value is None if the allocation is not mapped to a new uuid
new_resource = reordered[allocation.resource]
if not new_resource:
continue
# move all slots to the mapped allocation id
new_id = ids[new_resource]
for slot in allocation.reserved_slots:
# build a query here as the manipulation of mapped objects in
# combination with the delete query below seems a bit
# unpredictable given the cascading of changes
query = Session.query(ReservedSlot)
query = query.filter(and_(
ReservedSlot.resource == slot.resource,
ReservedSlot.allocation_id == slot.allocation_id,
ReservedSlot.start == slot.start
))
query.update(
{
ReservedSlot.resource: new_resource,
ReservedSlot.allocation_id: new_id
}
)
# get rid of the unused allocations (always preserving the master)
if unused:
query = Session.query(Allocation)
query = query.filter(Allocation.resource.in_(unused))
query = query.filter(Allocation.id != master.id)
query = query.filter(Allocation._start == master._start)
query.delete('fetch')
def _query_blocked_periods(self):
query = Session.query(self.models.BlockedPeriod)
query = query.filter_by(resource=self.resource)
return query
def change_allocation():
allocation = Session.query(Allocation).one()
allocation.group = uuid()
def managed_allocations(self):
""" The allocations managed by this scheduler / resource. """
query = Session.query(Allocation)
query = query.filter(Allocation.mirror_of == self.uuid)
return query
def managed_reservations(self):
""" The reservations managed by this scheduler / resource. """
query = Session.query(Reservation)
query = query.filter(Reservation.resource == self.uuid)
return query
def read_allocation():
allocation = Session.query(Allocation).one()
allocation.resource
def drop():
Session.query(Allocation).delete()
transaction.commit()
def read_allocation():
allocation = Session.query(Allocation).one()
allocation.resource
def monthly_report(year, month, resources, reservations='*'):
schedulers, titles = dict(), dict()
for uuid in resources.keys():
schedulers[uuid] = db.Scheduler(uuid)
titles[uuid] = utils.get_resource_title(resources[uuid])
# this order is used for every day in the month
ordered_uuids = [i[0] for i in sorted(titles.items(), key=lambda i: i[1])]
# build the hierarchical structure of the report data
report = utils.OrderedDict()
last_day = 28
for d in sorted((d for d in calendar.itermonthdates(year, month))):
if not d.month == month:
continue
day = d.day
last_day = max(last_day, day)
report[day] = utils.OrderedDict()
for uuid in ordered_uuids:
report[day][uuid] = dict()
report[day][uuid][u'title'] = titles[uuid]
report[day][uuid][u'approved'] = list()
report[day][uuid][u'pending'] = list()
report[day][uuid][u'url'] = resources[uuid].absolute_url()
report[day][uuid][u'lists'] = {
u'approved': _(u'Approved'),
u'pending': _(u'Pending'),
}
# gather the reservations with as much bulk loading as possible
period_start = date(year, month, 1)
period_end = date(year, month, last_day)
# get a list of relevant allocations in the given period
query = Session.query(Allocation)
query = query.filter(period_start <= Allocation._start)
query = query.filter(Allocation._start <= period_end)
query = query.filter(Allocation.resource == Allocation.mirror_of)
query = query.filter(Allocation.resource.in_(resources.keys()))
allocations = query.all()
# quit if there are no allocations at this point
if not allocations:
return {}
# store by group as it will be needed multiple times over later
groups = dict()
for allocation in allocations:
groups.setdefault(allocation.group, list()).append(allocation)
# using the groups get the relevant reservations
query = Session.query(Reservation)
query = query.filter(Reservation.target.in_(groups.keys()))
if reservations != '*':
query = query.filter(Reservation.token.in_(reservations))
query = query.order_by(Reservation.status)
reservations = query.all()
reservation_urls = ReservationUrls()
@utils.memoize
def json_timespans(start, end):
return json.dumps([dict(start=start, end=end)])
used_days = dict([(i, False) for i in range(1, 32)])
def add_reservation(start, end, reservation):
day = start.day
used_days[day] = True
end += timedelta(microseconds=1)
start, end = start.strftime('%H:%M'), end.strftime('%H:%M')
context = resources[utils.string_uuid(reservation.resource)]
if reservation.status == u'approved':
rightside_urls = [(
_(u'Delete'),
reservation_urls.revoke_all_url(reservation.token, context)
)]
elif reservation.status == u'pending':
rightside_urls = [
(
_(u'Approve'),
reservation_urls.approve_all_url(
reservation.token, context
)
),
(
_(u'Deny'),
reservation_urls.deny_all_url(reservation.token, context)
),
]
else:
raise NotImplementedError
reservation_lists = report[day][utils.string_uuid(
reservation.resource
)]
reservation_lists[reservation.status].append(
dict(
start=start,
end=end,
email=reservation.email,
data=reservation.data,
timespans=json_timespans(start, end),
rightside_urls=rightside_urls,
token=reservation.token,
quota=utils.get_reservation_quota_statement(reservation.quota),
resource=context
)
)
for reservation in reservations:
if reservation.target_type == u'allocation':
add_reservation(reservation.start, reservation.end, reservation)
else:
for allocation in groups[reservation.target]:
add_reservation(allocation.start, allocation.end, reservation)
# remove unused days
for day in report:
if not used_days[day]:
del report[day]
return report
def change_allocation():
allocation = Session.query(Allocation).one()
allocation.group = uuid()