def note_seed_redirect(self, url):
canon_seed_redirect = brozzler.site_surt_canon(url)
canon_seed = brozzler.site_surt_canon(self.seed)
# if http://foo.com/ redirects to https://foo.com/a/b/c let's also
# put all of https://foo.com/ in scope
if (canon_seed_redirect.authority == canon_seed.authority
and canon_seed_redirect.scheme != canon_seed.scheme):
canon_seed.scheme = canon_seed_redirect.scheme
self._accept_ssurt_if_not_redundant(
canon_seed.ssurt().decode('ascii'))
self._accept_ssurt_if_not_redundant(
canon_seed_redirect.ssurt().decode('ascii'))
def note_seed_redirect(self, url):
canon_seed_redirect = brozzler.site_surt_canon(url)
canon_seed = brozzler.site_surt_canon(self.seed)
# if http://foo.com/ redirects to https://foo.com/a/b/c let's also
# put all of https://foo.com/ in scope
if (canon_seed_redirect.authority == canon_seed.authority
and canon_seed_redirect.scheme != canon_seed.scheme):
canon_seed.scheme = canon_seed_redirect.scheme
self._accept_ssurt_if_not_redundant(
canon_seed.ssurt().decode('ascii'))
self._accept_ssurt_if_not_redundant(
canon_seed_redirect.ssurt().decode('ascii'))
def populate_defaults(self):
if not "status" in self:
self.status = "ACTIVE"
if not "claimed" in self:
self.claimed = False
if not "last_disclaimed" in self:
self.last_disclaimed = brozzler.EPOCH_UTC
if not "last_claimed" in self:
self.last_claimed = brozzler.EPOCH_UTC
if not "scope" in self:
self.scope = {}
if not "surt" in self.scope and self.seed:
self.scope["surt"] = brozzler.site_surt_canon(
self.seed).surt().decode('ascii')
if not "starts_and_stops" in self:
if self.get("start_time"): # backward compatibility
self.starts_and_stops = [{
"start":self.get("start_time"),"stop":None}]
if self.get("status") != "ACTIVE":
self.starts_and_stops[0]["stop"] = self.last_disclaimed
del self["start_time"]
else:
self.starts_and_stops = [
{"start":doublethink.utcnow(),"stop":None}]
def populate_defaults(self):
if not "status" in self:
self.status = "ACTIVE"
if not "claimed" in self:
self.claimed = False
if not "last_disclaimed" in self:
self.last_disclaimed = brozzler.EPOCH_UTC
if not "last_claimed" in self:
self.last_claimed = brozzler.EPOCH_UTC
if not "scope" in self:
self.scope = {}
# backward compatibility
if "surt" in self.scope:
if not "accepts" in self.scope:
self.scope["accepts"] = []
self.scope["accepts"].append({"surt": self.scope["surt"]})
del self.scope["surt"]
# backward compatibility
if ("max_hops_off_surt" in self.scope
and not "max_hops_off" in self.scope):
self.scope["max_hops_off"] = self.scope["max_hops_off_surt"]
if "max_hops_off_surt" in self.scope:
del self.scope["max_hops_off_surt"]
if self.seed:
self._accept_ssurt_if_not_redundant(
brozzler.site_surt_canon(self.seed).ssurt().decode('ascii'))
if not "starts_and_stops" in self:
if self.get("start_time"): # backward compatibility
self.starts_and_stops = [{
"start": self.get("start_time"),
"stop": None
}]
if self.get("status") != "ACTIVE":
self.starts_and_stops[0]["stop"] = self.last_disclaimed
del self["start_time"]
else:
self.starts_and_stops = [{
"start": doublethink.utcnow(),
"stop": None
}]
def populate_defaults(self):
if not "status" in self:
self.status = "ACTIVE"
if not "claimed" in self:
self.claimed = False
if not "last_disclaimed" in self:
self.last_disclaimed = brozzler.EPOCH_UTC
if not "last_claimed" in self:
self.last_claimed = brozzler.EPOCH_UTC
if not "scope" in self:
self.scope = {}
# backward compatibility
if "surt" in self.scope:
if not "accepts" in self.scope:
self.scope["accepts"] = []
self.scope["accepts"].append({"surt": self.scope["surt"]})
del self.scope["surt"]
# backward compatibility
if ("max_hops_off_surt" in self.scope
and not "max_hops_off" in self.scope):
self.scope["max_hops_off"] = self.scope["max_hops_off_surt"]
if "max_hops_off_surt" in self.scope:
del self.scope["max_hops_off_surt"]
if self.seed:
self._accept_ssurt_if_not_redundant(
brozzler.site_surt_canon(self.seed).ssurt().decode('ascii'))
if not "starts_and_stops" in self:
if self.get("start_time"): # backward compatibility
self.starts_and_stops = [{
"start":self.get("start_time"),"stop":None}]
if self.get("status") != "ACTIVE":
self.starts_and_stops[0]["stop"] = self.last_disclaimed
del self["start_time"]
else:
self.starts_and_stops = [
{"start":doublethink.utcnow(),"stop":None}]
def note_seed_redirect(self, url):
self._accept_ssurt_if_not_redundant(
brozzler.site_surt_canon(url).ssurt().decode('ascii'))
def note_seed_redirect(self, url):
new_scope_surt = brozzler.site_surt_canon(url).surt().decode("ascii")
if not new_scope_surt.startswith(self.scope["surt"]):
self.logger.info("changing site scope surt from {} to {}".format(
self.scope["surt"], new_scope_surt))
self.scope["surt"] = new_scope_surt
