def send(self, request, response):
"""This method takes a http request and sends a catalog
to the client. If the client is capable of receiving an
incremental update, we'll send that. Otherwise, it calls
into the catalog to send a full copy."""
modified = request.headers.get("If-Modified-Since", None)
ts = None
if modified:
try:
ts = catalog.ts_to_datetime(modified)
except ValueError:
ts = None
# Incremental catalog updates
response.headers['Content-type'] = 'text/plain'
if ts and self.up_to_date(ts):
response.status = httplib.NOT_MODIFIED
response.headers['Last-Modified'] = \
self.catalog.last_modified()
response.headers['X-Catalog-Type'] = 'incremental'
return
elif ts and self.enough_history(ts):
response.headers['Last-Modified'] = \
self.catalog.last_modified()
response.headers['X-Catalog-Type'] = 'incremental'
return self._send_updates(ts, None, response)
else:
# Not enough history, or full catalog requested
response.headers['X-Catalog-Type'] = 'full'
return self.catalog.send(None, response)
def send(self, request, response):
"""This method takes a http request and sends a catalog
to the client. If the client is capable of receiving an
incremental update, we'll send that. Otherwise, it calls
into the catalog to send a full copy."""
modified = request.headers.get("If-Modified-Since", None)
ts = None
if modified:
try:
ts = catalog.ts_to_datetime(modified)
except ValueError:
ts = None
# Incremental catalog updates
response.headers['Content-type'] = 'text/plain'
if ts and self.up_to_date(ts):
response.status = httplib.NOT_MODIFIED
response.headers['Last-Modified'] = \
self.catalog.last_modified()
response.headers['X-Catalog-Type'] = 'incremental'
return
elif ts and self.enough_history(ts):
response.headers['Last-Modified'] = \
self.catalog.last_modified()
response.headers['X-Catalog-Type'] = 'incremental'
return self._send_updates(ts, None, response)
else:
# Not enough history, or full catalog requested
response.headers['X-Catalog-Type'] = 'full'
return self.catalog.send(None, response)
def catalog(self, last_modified=None):
"""Returns a generator object containing an incremental update
if 'last_modified' is provided. If 'last_modified' is not
provided, a generator object for the full version of the catalog
will be returned instead. 'last_modified' should be a datetime
object or an ISO8601 formatted string."""
self.scfg.inc_catalog()
if isinstance(last_modified, basestring):
last_modified = catalog.ts_to_datetime(last_modified)
# Incremental catalog updates
c = self.scfg.catalog
ul = self.scfg.updatelog
if last_modified:
if not ul.up_to_date(last_modified) and \
ul.enough_history(last_modified):
for line in ul._gen_updates(last_modified,
self.scfg):
yield line
else:
raise RepositoryCatalogNoUpdatesError(
"incremental", c.last_modified())
return
# Full catalog request.
# Return attributes first.
for line in c.attrs_as_lines():
yield line
# Return the contents last.
for line in c.as_lines(self.scfg):
yield line
def _setup_logfiles(self):
"""Scans the directory containing the update log's files.
Sets up any necessary state for the UpdateLog."""
# Store names of logfiles as integers for easier comparison
self.logfiles = [f for f in os.listdir(self.rootdir)]
self.logfiles.sort()
self.curfiles = len(self.logfiles)
if self.curfiles == 0:
self.last_update = None
self.first_update = None
return
# Find the last update by opening the most recent logfile
# and finding its last entry
filenm = self.logfiles[self.curfiles - 1]
logf = file(os.path.join(self.rootdir, filenm), "r")
last_update = None
for ln in logf:
lspl = ln.split(" ", 4)
if len(lspl) < 4:
continue
current_ts = catalog.ts_to_datetime(lspl[1])
if not last_update or current_ts > last_update:
last_update = current_ts
logf.close()
self.last_update = last_update
self.first_update = datetime.datetime(
*time.strptime(self.logfiles[0], "%Y%m%d%H")[0:6])
def _setup_logfiles(self):
"""Scans the directory containing the update log's files.
Sets up any necessary state for the UpdateLog."""
# Store names of logfiles as integers for easier comparison
self.logfiles = [f for f in os.listdir(self.rootdir)]
self.logfiles.sort()
self.curfiles = len(self.logfiles)
if self.curfiles == 0:
self.last_update = None
self.first_update = None
return
# Find the last update by opening the most recent logfile
# and finding its last entry
filenm = self.logfiles[self.curfiles - 1]
logf = file(os.path.join(self.rootdir, filenm), "r")
last_update = None
for ln in logf:
lspl = ln.split(" ", 4)
if len(lspl) < 4:
continue
current_ts = catalog.ts_to_datetime(lspl[1])
if not last_update or current_ts > last_update:
last_update = current_ts
logf.close()
self.last_update = last_update
self.first_update = datetime.datetime(
*time.strptime(self.logfiles[0], "%Y%m%d%H")[0:6])
def _recv_updates(filep, path, cts):
"""A static method that takes a file-like object, a path, and a
timestamp. It reads a stream as an incoming updatelog and
modifies the catalog on disk."""
if not os.path.exists(path):
os.makedirs(path)
# Build a list of FMRIs that this update would add, check to
# make sure that they aren't present in the catalog, then
# append the fmris.
mts = catalog.ts_to_datetime(cts)
cts = mts
pts = mts
added = 0
npkgs = 0
add_lines = []
unknown_lines = []
bad_fmri = None
attrs = {}
for s in filep:
l = s.split(None, 3)
if len(l) < 4:
continue
elif l[2] not in catalog.known_prefixes:
# Add unknown line directly to catalog.
# This can be post-processed later, when it
# becomes known.
#
# XXX Notify user that unknown entry was added?
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
line = "{0} {1}\n".format(l[2], l[3])
unknown_lines.append(line)
elif l[0] == "+":
# This is a known entry type.
# Create a list of FMRIs to add, since
# additional inspection is required
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
# The format for C and V records
# is described in the Catalog's
# docstring.
if l[2] in tuple("CV"):
try:
f = fmri.PkgFmri(l[3])
except fmri.IllegalFmri as e:
bad_fmri = e
mts = pts
continue
line = "{0} {1} {2} {3}\n".format(
l[2], "pkg", f.pkg_name, f.version)
add_lines.append(line)
added += 1
# If we got a parse error on FMRIs and transfer
# wasn't truncated, raise a retryable transport
if bad_fmri:
raise bad_fmri
# Verify that they aren't already in the catalog
catpath = os.path.normpath(os.path.join(path, "catalog"))
tmp_num, tmpfile = tempfile.mkstemp(dir=path)
tfile = os.fdopen(tmp_num, 'w')
try:
pfile = file(catpath, "rb")
except IOError as e:
if e.errno == errno.ENOENT:
# Creating an empty file
file(catpath, "wb").close()
pfile = file(catpath, "rb")
else:
tfile.close()
portable.remove(tmpfile)
raise
pfile.seek(0)
for c in pfile:
if c[0] in tuple("CV"):
npkgs += 1
if c in add_lines:
pfile.close()
tfile.close()
portable.remove(tmpfile)
raise UpdateLogException(
"Package {0} is already in the catalog".format(c))
tfile.write(c)
# Write the new entries to the catalog
tfile.seek(0, os.SEEK_END)
tfile.writelines(add_lines)
if len(unknown_lines) > 0:
tfile.writelines(unknown_lines)
tfile.close()
pfile.close()
os.chmod(tmpfile, catalog.ServerCatalog.file_mode)
portable.rename(tmpfile, catpath)
# Now re-write npkgs and Last-Modified in attributes file
afile = file(os.path.normpath(os.path.join(path, "attrs")), "r")
attrre = re.compile('^S ([^:]*): (.*)')
for entry in afile:
m = attrre.match(entry)
if m != None:
attrs[m.group(1)] = m.group(2)
afile.close()
# Update the attributes we care about
attrs["npkgs"] = npkgs + added
attrs["Last-Modified"] = mts.isoformat()
# Write attributes back out
apath = os.path.normpath(os.path.join(path, "attrs"))
tmp_num, tmpfile = tempfile.mkstemp(dir=path)
tfile = os.fdopen(tmp_num, 'w')
for a in attrs.keys():
s = "S {0}: {1}\n".format(a, attrs[a])
tfile.write(s)
tfile.close()
os.chmod(tmpfile, catalog.ServerCatalog.file_mode)
portable.rename(tmpfile, apath)
return True
def _recv_updates(filep, path, cts):
"""A static method that takes a file-like object,
a path, and a timestamp. This is the other half of
send_updates(). It reads a stream as an incoming updatelog and
modifies the catalog on disk."""
if not os.path.exists(path):
os.makedirs(path)
# Build a list of FMRIs that this update would add, check to
# make sure that they aren't present in the catalog, then
# append the fmris.
mts = catalog.ts_to_datetime(cts)
cts = mts
pts = mts
added = 0
npkgs = 0
add_lines = []
unknown_lines = []
bad_fmri = None
attrs = {}
for s in filep:
l = s.split(None, 3)
if len(l) < 4:
continue
elif l[2] not in catalog.known_prefixes:
# Add unknown line directly to catalog.
# This can be post-processed later, when it
# becomes known.
#
# XXX Notify user that unknown entry was added?
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
line = "%s %s\n" % (l[2], l[3])
unknown_lines.append(line)
elif l[0] == "+":
# This is a known entry type.
# Create a list of FMRIs to add, since
# additional inspection is required
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
# The format for C and V records
# is described in the Catalog's
# docstring.
if l[2] in tuple("CV"):
try:
f = fmri.PkgFmri(l[3])
except fmri.IllegalFmri, e:
bad_fmri = e
mts = pts
continue
line = "%s %s %s %s\n" % \
(l[2], "pkg", f.pkg_name,
f.version)
add_lines.append(line)
added += 1
def _recv_updates(filep, path, cts):
"""A static method that takes a file-like object,
a path, and a timestamp. This is the other half of
send_updates(). It reads a stream as an incoming updatelog and
modifies the catalog on disk."""
if not os.path.exists(path):
os.makedirs(path)
# Build a list of FMRIs that this update would add, check to
# make sure that they aren't present in the catalog, then
# append the fmris.
mts = catalog.ts_to_datetime(cts)
cts = mts
pts = mts
added = 0
npkgs = 0
add_lines = []
unknown_lines = []
bad_fmri = None
attrs = {}
for s in filep:
l = s.split(None, 3)
if len(l) < 4:
continue
elif l[2] not in catalog.known_prefixes:
# Add unknown line directly to catalog.
# This can be post-processed later, when it
# becomes known.
#
# XXX Notify user that unknown entry was added?
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
line = "%s %s\n" % (l[2], l[3])
unknown_lines.append(line)
elif l[0] == "+":
# This is a known entry type.
# Create a list of FMRIs to add, since
# additional inspection is required
ts = catalog.ts_to_datetime(l[1])
if ts > cts:
if ts > mts:
pts = mts
mts = ts
# The format for C and V records
# is described in the Catalog's
# docstring.
if l[2] in tuple("CV"):
try:
f = fmri.PkgFmri(l[3])
except fmri.IllegalFmri, e:
bad_fmri = e
mts = pts
continue
line = "%s %s %s %s\n" % \
(l[2], "pkg", f.pkg_name,
f.version)
add_lines.append(line)
added += 1
