def note_seed_redirect(self, url): canon_seed_redirect = brozzler.site_surt_canon(url) canon_seed = brozzler.site_surt_canon(self.seed) # if http://foo.com/ redirects to https://foo.com/a/b/c let's also # put all of https://foo.com/ in scope if (canon_seed_redirect.authority == canon_seed.authority and canon_seed_redirect.scheme != canon_seed.scheme): canon_seed.scheme = canon_seed_redirect.scheme self._accept_ssurt_if_not_redundant( canon_seed.ssurt().decode('ascii')) self._accept_ssurt_if_not_redundant( canon_seed_redirect.ssurt().decode('ascii'))
def populate_defaults(self): if not "status" in self: self.status = "ACTIVE" if not "claimed" in self: self.claimed = False if not "last_disclaimed" in self: self.last_disclaimed = brozzler.EPOCH_UTC if not "last_claimed" in self: self.last_claimed = brozzler.EPOCH_UTC if not "scope" in self: self.scope = {} if not "surt" in self.scope and self.seed: self.scope["surt"] = brozzler.site_surt_canon( self.seed).surt().decode('ascii') if not "starts_and_stops" in self: if self.get("start_time"): # backward compatibility self.starts_and_stops = [{ "start":self.get("start_time"),"stop":None}] if self.get("status") != "ACTIVE": self.starts_and_stops[0]["stop"] = self.last_disclaimed del self["start_time"] else: self.starts_and_stops = [ {"start":doublethink.utcnow(),"stop":None}]
def populate_defaults(self): if not "status" in self: self.status = "ACTIVE" if not "claimed" in self: self.claimed = False if not "last_disclaimed" in self: self.last_disclaimed = brozzler.EPOCH_UTC if not "last_claimed" in self: self.last_claimed = brozzler.EPOCH_UTC if not "scope" in self: self.scope = {} # backward compatibility if "surt" in self.scope: if not "accepts" in self.scope: self.scope["accepts"] = [] self.scope["accepts"].append({"surt": self.scope["surt"]}) del self.scope["surt"] # backward compatibility if ("max_hops_off_surt" in self.scope and not "max_hops_off" in self.scope): self.scope["max_hops_off"] = self.scope["max_hops_off_surt"] if "max_hops_off_surt" in self.scope: del self.scope["max_hops_off_surt"] if self.seed: self._accept_ssurt_if_not_redundant( brozzler.site_surt_canon(self.seed).ssurt().decode('ascii')) if not "starts_and_stops" in self: if self.get("start_time"): # backward compatibility self.starts_and_stops = [{ "start": self.get("start_time"), "stop": None }] if self.get("status") != "ACTIVE": self.starts_and_stops[0]["stop"] = self.last_disclaimed del self["start_time"] else: self.starts_and_stops = [{ "start": doublethink.utcnow(), "stop": None }]
def populate_defaults(self): if not "status" in self: self.status = "ACTIVE" if not "claimed" in self: self.claimed = False if not "last_disclaimed" in self: self.last_disclaimed = brozzler.EPOCH_UTC if not "last_claimed" in self: self.last_claimed = brozzler.EPOCH_UTC if not "scope" in self: self.scope = {} # backward compatibility if "surt" in self.scope: if not "accepts" in self.scope: self.scope["accepts"] = [] self.scope["accepts"].append({"surt": self.scope["surt"]}) del self.scope["surt"] # backward compatibility if ("max_hops_off_surt" in self.scope and not "max_hops_off" in self.scope): self.scope["max_hops_off"] = self.scope["max_hops_off_surt"] if "max_hops_off_surt" in self.scope: del self.scope["max_hops_off_surt"] if self.seed: self._accept_ssurt_if_not_redundant( brozzler.site_surt_canon(self.seed).ssurt().decode('ascii')) if not "starts_and_stops" in self: if self.get("start_time"): # backward compatibility self.starts_and_stops = [{ "start":self.get("start_time"),"stop":None}] if self.get("status") != "ACTIVE": self.starts_and_stops[0]["stop"] = self.last_disclaimed del self["start_time"] else: self.starts_and_stops = [ {"start":doublethink.utcnow(),"stop":None}]
def note_seed_redirect(self, url): self._accept_ssurt_if_not_redundant( brozzler.site_surt_canon(url).ssurt().decode('ascii'))
def note_seed_redirect(self, url): new_scope_surt = brozzler.site_surt_canon(url).surt().decode("ascii") if not new_scope_surt.startswith(self.scope["surt"]): self.logger.info("changing site scope surt from {} to {}".format( self.scope["surt"], new_scope_surt)) self.scope["surt"] = new_scope_surt