def add_domain(cls, url, db, publish_method, config): from holmes.models import Domain domain_name, domain_url = get_domain_from_url(url) domains = db.query(Domain).filter(or_( Domain.name == domain_name, Domain.name == domain_name.rstrip('/'), Domain.name == "%s/" % domain_name )).all() if not domains: domain = None else: domain = domains[0] if not domain: url_hash = hashlib.sha512(domain_url).hexdigest() domain = Domain(url=domain_url, url_hash=url_hash, name=domain_name) db.add(domain) db.flush() db.commit() publish_method(dumps({ 'type': 'new-domain', 'domainUrl': str(domain_url) })) from holmes.models import Limiter connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS Limiter.add_or_update_limiter(db, domain_url, connections) return domain
def post(self): access_token = self.request.headers.get('X-AUTH-HOLMES', None) if access_token is None: self.set_status(403) self.write_json({'reason': 'Empty access token'}) return result = yield User.authenticate( access_token, self.application.http_client.fetch, self.db, self.application.config ) if result and result.get('user', None) is None: self.set_status(403) self.write_json({'reason': 'Not authorized user.'}) return post_data = loads(self.request.body) url = post_data.get('url', None) connections = self.application.config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS value = post_data.get('value', connections) if not url and not value: self.set_status(400) self.write_json({'reason': 'Not url or value'}) return result = Limiter.add_or_update_limiter(self.db, url, value) yield self.cache.remove_domain_limiters_key() self.write_json(result)
def add_domain(cls, url, db, publish_method, config, girl, default_violations_values, violation_definitions, cache): from holmes.models import Domain, DomainsViolationsPrefs from holmes.material import expire_materials domain_name, domain_url = get_domain_from_url(url) domains = ( db.query(Domain) .filter( or_( Domain.name == domain_name, Domain.name == domain_name.rstrip("/"), Domain.name == "%s/" % domain_name, ) ) .all() ) if not domains: domain = None else: domain = domains[0] if not domain: url_hash = hashlib.sha512(domain_url).hexdigest() domain = Domain(url=domain_url, url_hash=url_hash, name=domain_name) db.add(domain) db.flush() expire_materials(girl) publish_method(dumps({"type": "new-domain", "domainUrl": str(domain_url)})) keys = default_violations_values.keys() DomainsViolationsPrefs.insert_default_violations_values_for_domain( db, domain, keys, violation_definitions, cache ) from holmes.models import Limiter connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS Limiter.add_or_update_limiter(db, domain_url, connections) return domain
def add_domain(cls, url, db, publish_method, config, girl, default_violations_values, violation_definitions, cache): from holmes.models import Domain, DomainsViolationsPrefs from holmes.material import expire_materials domain_name, domain_url = get_domain_from_url(url) domains = db.query(Domain).filter( or_(Domain.name == domain_name, Domain.name == domain_name.rstrip('/'), Domain.name == "%s/" % domain_name)).all() if not domains: domain = None else: domain = domains[0] if not domain: url_hash = hashlib.sha512(domain_url).hexdigest() domain = Domain(url=domain_url, url_hash=url_hash, name=domain_name) db.add(domain) db.flush() expire_materials(girl) publish_method( dumps({ 'type': 'new-domain', 'domainUrl': str(domain_url) })) keys = default_violations_values.keys() DomainsViolationsPrefs.insert_default_violations_values_for_domain( db, domain, keys, violation_definitions, cache) from holmes.models import Limiter connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS Limiter.add_or_update_limiter(db, domain_url, connections) return domain
def test_get_limiters_for_domains(self): self.db.query(Limiter).delete() self.db.query(Domain).delete() for i in range(2): DomainFactory.create() value = 2 url = 'http://globo.com/' Limiter.add_or_update_limiter(self.db, url, value) DomainFactory.create(url=url, name='globo.com') active_domains = self.db.query(Domain).all() domains = Limiter.get_limiters_for_domains(self.db, active_domains) expect(domains).to_length(1) expect(str(domains[0])).to_equal(url)
def test_can_add_or_update_limiter(self): self.db.query(Limiter).delete() limiters = Limiter.get_all(self.db) expect(limiters).to_equal([]) # Add url = 'http://globo.com/' value = 2 Limiter.add_or_update_limiter(self.db, url, value) limiter = Limiter.by_url(url, self.db) expect(limiter.value).to_equal(2) # Update url = 'http://globo.com/' value = 3 Limiter.add_or_update_limiter(self.db, url, value) limiter = Limiter.by_url(url, self.db) expect(limiter.value).to_equal(3)
def post(self): if not self.validate_superuser(): return post_data = loads(self.request.body) url = post_data.get('url', None) connections = self.config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS value = post_data.get('value', connections) if not url and not value: self.set_status(400) self.write_json({ 'reason': 'Not url or value', 'description': self._('Not url or value') }) return result = Limiter.add_or_update_limiter(self.db, url, value) yield self.cache.remove_domain_limiters_key() self.write_json(result)