def doCreate(result):
if result.find("FATAL:") != -1:
log.error(result)
raise InternalDataStoreError(
"Unable to initialize postgres database: {}".format(
result))
self.startDatabase()
def _connectorFor_pg8000(dbmodule, **kwargs):
"""
Turn properties into pg8000 kwargs
"""
params = DBAPIParameters(**kwargs)
dbkwargs = {
"user": params.user,
"password": params.password,
"database": params.database,
}
if params.ssl:
dbkwargs["ssl"] = params.ssl
if params.unixsocket:
dbkwargs["unix_sock"] = params.unixsocket
# We're using a socket file
socketFP = CachingFilePath(dbkwargs["unix_sock"])
if socketFP.isdir():
# We have been given the directory, not the actual socket file
socketFP = socketFP.child(".s.PGSQL.{}".format(params.port if params.port else "5432"))
dbkwargs["unix_sock"] = socketFP.path
if not socketFP.isSocket():
raise InternalDataStoreError(
"No such socket file: {}".format(socketFP.path)
)
else:
dbkwargs["host"] = params.host
if params.port:
dbkwargs["port"] = int(params.port)
if "txnTimeoutSeconds" in kwargs:
dbkwargs["txnTimeoutSeconds"] = kwargs["txnTimeoutSeconds"]
return DBAPIConnector(dbmodule, pg8000Preflight, **dbkwargs)
def _connectorFor_module(dbmodule, **kwargs):
m = getattr(DBAPIConnector, "_connectorFor_{}".format(dbmodule.__name__), None)
if m is None:
raise InternalDataStoreError(
"Unknown DBAPI module: {}".format(dbmodule)
)
return m(dbmodule, **kwargs)
def connectorFor(dbtype, **kwargs):
if dbtype == "postgres":
return DBAPIConnector._connectorFor_module(postgres, **kwargs)
elif dbtype == "oracle":
return DBAPIConnector._connectorFor_module(cx_Oracle, **kwargs)
else:
raise InternalDataStoreError(
"Unknown database type: {}".format(dbtype))
def iCalendar(self):
text = self.fp.open().read()
try:
component = Component.fromString(text)
# Fix any bogus data we can
component.validCalendarData()
component.validCalendarForCalDAV(methodAllowed=False)
except InvalidICalendarDataError, e:
raise InternalDataStoreError(
"File corruption detected (%s) in file: %s" %
(e, self._path.path))
def component(self):
"""
Read address data and validate/fix it. Do not raise a store error here if there are unfixable
errors as that could prevent the overall request to fail. Instead we will hand bad data off to
the caller - that is not ideal but in theory we should have checked everything on the way in and
only allowed in good data.
"""
text = self._text()
try:
component = VComponent.fromString(text)
except InvalidVCardDataError, e:
# This is a really bad situation, so do raise
raise InternalDataStoreError(
"File corruption detected (%s) in file: %s" %
(e, self._path.path))
def createConnection():
try:
createDatabaseConn = self.produceConnection(
"schema creation", "postgres")
except postgres.DatabaseError as e:
log.error(
"Unable to connect to database for schema creation:"
" {error}",
error=e)
raise
createDatabaseCursor = createDatabaseConn.cursor()
if postgres.__name__ == "pg8000":
createDatabaseConn.realConnection.autocommit = True
elif postgres.__name__ == "pgdb":
createDatabaseCursor.execute("commit")
else:
raise InternalDataStoreError(
"Unknown Postgres DBM module: {}".format(postgres))
return createDatabaseConn, createDatabaseCursor
def _matchCalendarResources(self, calresource):
# Get the timezone property from the collection.
tz = calresource.getTimezone()
# Try cache
aggregated_resources = (yield FBCacheEntry.getCacheEntry(calresource, self.attendee_uid, self.timerange)) if config.EnableFreeBusyCache else None
if aggregated_resources is None:
if self.accountingItems is not None:
self.accountingItems["fb-uncached"] = self.accountingItems.get("fb-uncached", 0) + 1
caching = False
if config.EnableFreeBusyCache:
# Log extended item
if self.logItems is not None:
self.logItems["fb-uncached"] = self.logItems.get("fb-uncached", 0) + 1
# We want to cache a large range of time based on the current date
cache_start = normalizeToUTC(DateTime.getToday() + Duration(days=0 - config.FreeBusyCacheDaysBack))
cache_end = normalizeToUTC(DateTime.getToday() + Duration(days=config.FreeBusyCacheDaysForward))
# If the requested time range would fit in our allowed cache range, trigger the cache creation
if compareDateTime(self.timerange.getStart(), cache_start) >= 0 and compareDateTime(self.timerange.getEnd(), cache_end) <= 0:
cache_timerange = Period(cache_start, cache_end)
caching = True
#
# What we do is a fake calendar-query for VEVENT/VFREEBUSYs in the specified time-range.
# We then take those results and merge them into one VFREEBUSY component
# with appropriate FREEBUSY properties, and return that single item as iCal data.
#
# Create fake filter element to match time-range
tr = TimeRange(
start=(cache_timerange if caching else self.timerange).getStart().getText(),
end=(cache_timerange if caching else self.timerange).getEnd().getText(),
)
filter = caldavxml.Filter(
caldavxml.ComponentFilter(
caldavxml.ComponentFilter(
tr,
name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
),
name="VCALENDAR",
)
)
filter = Filter(filter)
tzinfo = filter.settimezone(tz)
if self.accountingItems is not None:
self.accountingItems["fb-query-timerange"] = (str(tr.start), str(tr.end),)
try:
resources = yield calresource.search(filter, useruid=self.attendee_uid, fbtype=True)
aggregated_resources = {}
for name, uid, comptype, test_organizer, float, start, end, fbtype, transp in resources:
if transp == 'T' and fbtype != '?':
fbtype = 'F'
aggregated_resources.setdefault((name, uid, comptype, test_organizer,), []).append((
float,
tupleFromDateTime(parseSQLTimestampToPyCalendar(start)),
tupleFromDateTime(parseSQLTimestampToPyCalendar(end)),
fbtype,
))
if caching:
yield FBCacheEntry.makeCacheEntry(calresource, self.attendee_uid, cache_timerange, aggregated_resources)
except IndexedSearchException:
raise InternalDataStoreError("Invalid indexedSearch query")
else:
if self.accountingItems is not None:
self.accountingItems["fb-cached"] = self.accountingItems.get("fb-cached", 0) + 1
# Log extended item
if self.logItems is not None:
self.logItems["fb-cached"] = self.logItems.get("fb-cached", 0) + 1
# Determine appropriate timezone (UTC is the default)
tzinfo = tz.gettimezone() if tz is not None else Timezone.UTCTimezone
filter = None
returnValue((aggregated_resources, tzinfo, filter,))
fh = self._path.open()
except IOError, e:
if e[0] == ENOENT:
raise NoSuchObjectResourceError(self)
else:
raise
try:
text = fh.read()
finally:
fh.close()
if not (text.startswith("BEGIN:VCARD\r\n")
or text.endswith("\r\nEND:VCARD\r\n")):
raise InternalDataStoreError(
"File corruption detected (improper start) in file: %s" %
(self._path.path, ))
self._objectText = text
return text
def uid(self):
if not hasattr(self, "_uid"):
self._uid = self.component().resourceUID()
return self._uid
# IDataStoreObject
def contentType(self):
"""
The content type of Addressbook objects is text/x-vcard.
"""
def locateCommand(name, cmd):
for found in which(cmd):
return found
raise InternalDataStoreError(
"Unable to locate {} command: {}".format(name, cmd))
def _internalGenerateFreeBusyInfo(
calresource,
fbinfo,
timerange,
matchtotal,
excludeuid=None,
organizer=None,
organizerPrincipal=None,
same_calendar_user=False,
servertoserver=False,
event_details=None,
logItems=None,
accountingItems=None,
):
"""
Run a free busy report on the specified calendar collection
accumulating the free busy info for later processing.
@param calresource: the L{Calendar} for a calendar collection.
@param fbinfo: the array of busy periods to update.
@param timerange: the L{TimeRange} for the query.
@param matchtotal: the running total for the number of matches.
@param excludeuid: a C{str} containing a UID value to exclude any
components with that UID from contributing to free-busy.
@param organizer: a C{str} containing the value of the ORGANIZER property
in the VFREEBUSY request. This is used in conjunction with the UID
value to process exclusions.
@param same_calendar_user: a C{bool} indicating whether the calendar user
requesting the free-busy information is the same as the calendar user
being targeted.
@param servertoserver: a C{bool} indicating whether we are doing a local or
remote lookup request.
@param event_details: a C{list} into which to store extended VEVENT details if not C{None}
@param logItems: a C{dict} to store logging info to
@param accountingItems: a C{dict} to store accounting info to
"""
# First check the privilege on this collection
# TODO: for server-to-server we bypass this right now as we have no way to authorize external users.
# TODO: actually we by pass altogether by assuming anyone can check anyone else's freebusy
# May need organizer principal
organizer_record = (yield calresource.directoryService(
).recordWithCalendarUserAddress(organizer)) if organizer else None
organizer_uid = organizer_record.uid if organizer_record else ""
# Free busy is per-user
attendee_uid = calresource.viewerHome().uid()
attendee_record = yield calresource.directoryService().recordWithUID(
attendee_uid.decode("utf-8"))
# Get the timezone property from the collection.
tz = calresource.getTimezone()
# Look for possible extended free busy information
rich_options = {
"organizer": False,
"delegate": False,
"resource": False,
}
do_event_details = False
if event_details is not None and organizer_record is not None and attendee_record is not None:
# Get the principal of the authorized user which may be different from the organizer if a delegate of
# the organizer is making the request
authz_uid = organizer_uid
authz_record = organizer_record
if calresource._txn._authz_uid is not None and calresource._txn._authz_uid != organizer_uid:
authz_uid = calresource._txn._authz_uid
authz_record = yield calresource.directoryService().recordWithUID(
authz_uid.decode("utf-8"))
# Check if attendee is also the organizer or the delegate doing the request
if attendee_uid in (organizer_uid, authz_uid):
do_event_details = True
rich_options["organizer"] = True
# Check if authorized user is a delegate of attendee
proxy = (yield authz_record.isProxyFor(attendee_record))
if config.Scheduling.Options.DelegeteRichFreeBusy and proxy:
do_event_details = True
rich_options["delegate"] = True
# Check if attendee is room or resource
if config.Scheduling.Options.RoomResourceRichFreeBusy and attendee_record.getCUType(
) in (
"RESOURCE",
"ROOM",
):
do_event_details = True
rich_options["resource"] = True
# Try cache
resources = (yield FBCacheEntry.getCacheEntry(
calresource, attendee_uid,
timerange)) if config.EnableFreeBusyCache else None
if resources is None:
if accountingItems is not None:
accountingItems["fb-uncached"] = accountingItems.get(
"fb-uncached", 0) + 1
caching = False
if config.EnableFreeBusyCache:
# Log extended item
if logItems is not None:
logItems["fb-uncached"] = logItems.get("fb-uncached", 0) + 1
# We want to cache a large range of time based on the current date
cache_start = normalizeToUTC(DateTime.getToday() + Duration(
days=0 - config.FreeBusyCacheDaysBack))
cache_end = normalizeToUTC(DateTime.getToday() + Duration(
days=config.FreeBusyCacheDaysForward))
# If the requested time range would fit in our allowed cache range, trigger the cache creation
if compareDateTime(timerange.start,
cache_start) >= 0 and compareDateTime(
timerange.end, cache_end) <= 0:
cache_timerange = TimeRange(start=cache_start.getText(),
end=cache_end.getText())
caching = True
#
# What we do is a fake calendar-query for VEVENT/VFREEBUSYs in the specified time-range.
# We then take those results and merge them into one VFREEBUSY component
# with appropriate FREEBUSY properties, and return that single item as iCal data.
#
# Create fake filter element to match time-range
filter = caldavxml.Filter(
caldavxml.ComponentFilter(
caldavxml.ComponentFilter(
cache_timerange if caching else timerange,
name=("VEVENT", "VFREEBUSY", "VAVAILABILITY"),
),
name="VCALENDAR",
))
filter = Filter(filter)
tzinfo = filter.settimezone(tz)
if accountingItems is not None:
tr = cache_timerange if caching else timerange
accountingItems["fb-query-timerange"] = (
str(tr.start),
str(tr.end),
)
try:
resources = yield calresource.search(filter,
useruid=attendee_uid,
fbtype=True)
if caching:
yield FBCacheEntry.makeCacheEntry(calresource, attendee_uid,
cache_timerange, resources)
except IndexedSearchException:
raise InternalDataStoreError("Invalid indexedSearch query")
else:
if accountingItems is not None:
accountingItems["fb-cached"] = accountingItems.get("fb-cached",
0) + 1
# Log extended item
if logItems is not None:
logItems["fb-cached"] = logItems.get("fb-cached", 0) + 1
# Determine appropriate timezone (UTC is the default)
tzinfo = tz.gettimezone() if tz is not None else Timezone(utc=True)
# We care about separate instances for VEVENTs only
aggregated_resources = {}
for name, uid, type, test_organizer, float, start, end, fbtype, transp in resources:
if transp == 'T' and fbtype != '?':
fbtype = 'F'
aggregated_resources.setdefault((
name,
uid,
type,
test_organizer,
), []).append((
float,
start,
end,
fbtype,
))
if accountingItems is not None:
accountingItems["fb-resources"] = {}
for k, v in aggregated_resources.items():
name, uid, type, test_organizer = k
accountingItems["fb-resources"][uid] = []
for float, start, end, fbtype in v:
fbstart = parseSQLTimestampToPyCalendar(start)
if float == 'Y':
fbstart.setTimezone(tzinfo)
else:
fbstart.setTimezone(Timezone(utc=True))
fbend = parseSQLTimestampToPyCalendar(end)
if float == 'Y':
fbend.setTimezone(tzinfo)
else:
fbend.setTimezone(Timezone(utc=True))
accountingItems["fb-resources"][uid].append((
float,
str(fbstart),
str(fbend),
fbtype,
))
# Cache directory record lookup outside this loop as it is expensive and will likely
# always end up being called with the same organizer address.
recordUIDCache = {}
for key in aggregated_resources.iterkeys():
name, uid, type, test_organizer = key
# Short-cut - if an fbtype exists we can use that
if type == "VEVENT" and aggregated_resources[key][0][3] != '?':
matchedResource = False
# Look at each instance
for float, start, end, fbtype in aggregated_resources[key]:
# Ignore free time or unknown
if fbtype in ('F', '?'):
continue
# Ignore ones of this UID
if excludeuid:
# See if we have a UID match
if (excludeuid == uid):
if test_organizer:
test_uid = recordUIDCache.get(test_organizer)
if test_uid is None:
test_record = (yield
calresource.directoryService(
).recordWithCalendarUserAddress(
test_organizer))
test_uid = test_record.uid if test_record else ""
recordUIDCache[test_organizer] = test_uid
else:
test_uid = ""
# Check that ORGANIZER's match (security requirement)
if (organizer is None) or (organizer_uid == test_uid):
continue
# Check for no ORGANIZER and check by same calendar user
elif (test_uid == "") and same_calendar_user:
continue
# Apply a timezone to any floating times
fbstart = parseSQLTimestampToPyCalendar(start)
if float == 'Y':
fbstart.setTimezone(tzinfo)
else:
fbstart.setTimezone(Timezone(utc=True))
fbend = parseSQLTimestampToPyCalendar(end)
if float == 'Y':
fbend.setTimezone(tzinfo)
else:
fbend.setTimezone(Timezone(utc=True))
# Clip instance to time range
clipped = clipPeriod(Period(fbstart, duration=fbend - fbstart),
Period(timerange.start, timerange.end))
# Double check for overlap
if clipped:
matchedResource = True
fbinfo[fbtype_index_mapper.get(fbtype, 0)].append(clipped)
if matchedResource:
# Check size of results is within limit
matchtotal += 1
if matchtotal > config.MaxQueryWithDataResults:
raise QueryMaxResources(config.MaxQueryWithDataResults,
matchtotal)
# Add extended details
if do_event_details:
child = (yield calresource.calendarObjectWithName(name))
# Only add fully public events
if not child.accessMode or child.accessMode == Component.ACCESS_PUBLIC:
calendar = (yield child.componentForUser())
_addEventDetails(calendar, event_details, rich_options,
timerange, tzinfo)
else:
child = (yield calresource.calendarObjectWithName(name))
calendar = (yield child.componentForUser())
# The calendar may come back as None if the resource is being changed, or was deleted
# between our initial index query and getting here. For now we will ignore this error, but in
# the longer term we need to implement some form of locking, perhaps.
if calendar is None:
log.error(
"Calendar %s is missing from calendar collection %r" %
(name, calresource))
continue
# Ignore ones of this UID
if excludeuid:
# See if we have a UID match
if (excludeuid == uid):
test_organizer = calendar.getOrganizer()
if test_organizer:
test_uid = recordUIDCache.get(test_organizer)
if test_uid is None:
test_record = (yield calresource.directoryService(
).recordWithCalendarUserAddress(test_organizer))
test_uid = test_record.uid if test_record else ""
recordUIDCache[test_organizer] = test_uid
else:
test_uid = ""
# Check that ORGANIZER's match (security requirement)
if (organizer is None) or (organizer_uid == test_uid):
continue
# Check for no ORGANIZER and check by same calendar user
elif (test_organizer is None) and same_calendar_user:
continue
if accountingItems is not None:
accountingItems.setdefault("fb-filter-match", []).append(uid)
if filter.match(calendar, None):
if accountingItems is not None:
accountingItems.setdefault("fb-filter-matched",
[]).append(uid)
# Check size of results is within limit
matchtotal += 1
if matchtotal > config.MaxQueryWithDataResults:
raise QueryMaxResources(config.MaxQueryWithDataResults,
matchtotal)
if calendar.mainType() == "VEVENT":
processEventFreeBusy(calendar, fbinfo, timerange, tzinfo)
elif calendar.mainType() == "VFREEBUSY":
processFreeBusyFreeBusy(calendar, fbinfo, timerange)
elif calendar.mainType() == "VAVAILABILITY":
processAvailabilityFreeBusy(calendar, fbinfo, timerange)
else:
assert "Free-busy query returned unwanted component: %s in %r", (
name,
calresource,
)
# Add extended details
if calendar.mainType() == "VEVENT" and do_event_details:
child = (yield calresource.calendarObjectWithName(name))
# Only add fully public events
if not child.accessMode or child.accessMode == Component.ACCESS_PUBLIC:
calendar = (yield child.componentForUser())
_addEventDetails(calendar, event_details, rich_options,
timerange, tzinfo)
returnValue(matchtotal)
