def send(self, request, response): """This method takes a http request and sends a catalog to the client. If the client is capable of receiving an incremental update, we'll send that. Otherwise, it calls into the catalog to send a full copy.""" modified = request.headers.get("If-Modified-Since", None) ts = None if modified: try: ts = catalog.ts_to_datetime(modified) except ValueError: ts = None # Incremental catalog updates response.headers['Content-type'] = 'text/plain' if ts and self.up_to_date(ts): response.status = httplib.NOT_MODIFIED response.headers['Last-Modified'] = \ self.catalog.last_modified() response.headers['X-Catalog-Type'] = 'incremental' return elif ts and self.enough_history(ts): response.headers['Last-Modified'] = \ self.catalog.last_modified() response.headers['X-Catalog-Type'] = 'incremental' return self._send_updates(ts, None, response) else: # Not enough history, or full catalog requested response.headers['X-Catalog-Type'] = 'full' return self.catalog.send(None, response)
def catalog(self, last_modified=None): """Returns a generator object containing an incremental update if 'last_modified' is provided. If 'last_modified' is not provided, a generator object for the full version of the catalog will be returned instead. 'last_modified' should be a datetime object or an ISO8601 formatted string.""" self.scfg.inc_catalog() if isinstance(last_modified, basestring): last_modified = catalog.ts_to_datetime(last_modified) # Incremental catalog updates c = self.scfg.catalog ul = self.scfg.updatelog if last_modified: if not ul.up_to_date(last_modified) and \ ul.enough_history(last_modified): for line in ul._gen_updates(last_modified, self.scfg): yield line else: raise RepositoryCatalogNoUpdatesError( "incremental", c.last_modified()) return # Full catalog request. # Return attributes first. for line in c.attrs_as_lines(): yield line # Return the contents last. for line in c.as_lines(self.scfg): yield line
def _setup_logfiles(self): """Scans the directory containing the update log's files. Sets up any necessary state for the UpdateLog.""" # Store names of logfiles as integers for easier comparison self.logfiles = [f for f in os.listdir(self.rootdir)] self.logfiles.sort() self.curfiles = len(self.logfiles) if self.curfiles == 0: self.last_update = None self.first_update = None return # Find the last update by opening the most recent logfile # and finding its last entry filenm = self.logfiles[self.curfiles - 1] logf = file(os.path.join(self.rootdir, filenm), "r") last_update = None for ln in logf: lspl = ln.split(" ", 4) if len(lspl) < 4: continue current_ts = catalog.ts_to_datetime(lspl[1]) if not last_update or current_ts > last_update: last_update = current_ts logf.close() self.last_update = last_update self.first_update = datetime.datetime( *time.strptime(self.logfiles[0], "%Y%m%d%H")[0:6])
def _recv_updates(filep, path, cts): """A static method that takes a file-like object, a path, and a timestamp. It reads a stream as an incoming updatelog and modifies the catalog on disk.""" if not os.path.exists(path): os.makedirs(path) # Build a list of FMRIs that this update would add, check to # make sure that they aren't present in the catalog, then # append the fmris. mts = catalog.ts_to_datetime(cts) cts = mts pts = mts added = 0 npkgs = 0 add_lines = [] unknown_lines = [] bad_fmri = None attrs = {} for s in filep: l = s.split(None, 3) if len(l) < 4: continue elif l[2] not in catalog.known_prefixes: # Add unknown line directly to catalog. # This can be post-processed later, when it # becomes known. # # XXX Notify user that unknown entry was added? ts = catalog.ts_to_datetime(l[1]) if ts > cts: if ts > mts: pts = mts mts = ts line = "{0} {1}\n".format(l[2], l[3]) unknown_lines.append(line) elif l[0] == "+": # This is a known entry type. # Create a list of FMRIs to add, since # additional inspection is required ts = catalog.ts_to_datetime(l[1]) if ts > cts: if ts > mts: pts = mts mts = ts # The format for C and V records # is described in the Catalog's # docstring. if l[2] in tuple("CV"): try: f = fmri.PkgFmri(l[3]) except fmri.IllegalFmri as e: bad_fmri = e mts = pts continue line = "{0} {1} {2} {3}\n".format( l[2], "pkg", f.pkg_name, f.version) add_lines.append(line) added += 1 # If we got a parse error on FMRIs and transfer # wasn't truncated, raise a retryable transport if bad_fmri: raise bad_fmri # Verify that they aren't already in the catalog catpath = os.path.normpath(os.path.join(path, "catalog")) tmp_num, tmpfile = tempfile.mkstemp(dir=path) tfile = os.fdopen(tmp_num, 'w') try: pfile = file(catpath, "rb") except IOError as e: if e.errno == errno.ENOENT: # Creating an empty file file(catpath, "wb").close() pfile = file(catpath, "rb") else: tfile.close() portable.remove(tmpfile) raise pfile.seek(0) for c in pfile: if c[0] in tuple("CV"): npkgs += 1 if c in add_lines: pfile.close() tfile.close() portable.remove(tmpfile) raise UpdateLogException( "Package {0} is already in the catalog".format(c)) tfile.write(c) # Write the new entries to the catalog tfile.seek(0, os.SEEK_END) tfile.writelines(add_lines) if len(unknown_lines) > 0: tfile.writelines(unknown_lines) tfile.close() pfile.close() os.chmod(tmpfile, catalog.ServerCatalog.file_mode) portable.rename(tmpfile, catpath) # Now re-write npkgs and Last-Modified in attributes file afile = file(os.path.normpath(os.path.join(path, "attrs")), "r") attrre = re.compile('^S ([^:]*): (.*)') for entry in afile: m = attrre.match(entry) if m != None: attrs[m.group(1)] = m.group(2) afile.close() # Update the attributes we care about attrs["npkgs"] = npkgs + added attrs["Last-Modified"] = mts.isoformat() # Write attributes back out apath = os.path.normpath(os.path.join(path, "attrs")) tmp_num, tmpfile = tempfile.mkstemp(dir=path) tfile = os.fdopen(tmp_num, 'w') for a in attrs.keys(): s = "S {0}: {1}\n".format(a, attrs[a]) tfile.write(s) tfile.close() os.chmod(tmpfile, catalog.ServerCatalog.file_mode) portable.rename(tmpfile, apath) return True
def _recv_updates(filep, path, cts): """A static method that takes a file-like object, a path, and a timestamp. This is the other half of send_updates(). It reads a stream as an incoming updatelog and modifies the catalog on disk.""" if not os.path.exists(path): os.makedirs(path) # Build a list of FMRIs that this update would add, check to # make sure that they aren't present in the catalog, then # append the fmris. mts = catalog.ts_to_datetime(cts) cts = mts pts = mts added = 0 npkgs = 0 add_lines = [] unknown_lines = [] bad_fmri = None attrs = {} for s in filep: l = s.split(None, 3) if len(l) < 4: continue elif l[2] not in catalog.known_prefixes: # Add unknown line directly to catalog. # This can be post-processed later, when it # becomes known. # # XXX Notify user that unknown entry was added? ts = catalog.ts_to_datetime(l[1]) if ts > cts: if ts > mts: pts = mts mts = ts line = "%s %s\n" % (l[2], l[3]) unknown_lines.append(line) elif l[0] == "+": # This is a known entry type. # Create a list of FMRIs to add, since # additional inspection is required ts = catalog.ts_to_datetime(l[1]) if ts > cts: if ts > mts: pts = mts mts = ts # The format for C and V records # is described in the Catalog's # docstring. if l[2] in tuple("CV"): try: f = fmri.PkgFmri(l[3]) except fmri.IllegalFmri, e: bad_fmri = e mts = pts continue line = "%s %s %s %s\n" % \ (l[2], "pkg", f.pkg_name, f.version) add_lines.append(line) added += 1