def add_domain(cls, url, db, publish_method, config):
from holmes.models import Domain
domain_name, domain_url = get_domain_from_url(url)
domains = db.query(Domain).filter(or_(
Domain.name == domain_name,
Domain.name == domain_name.rstrip('/'),
Domain.name == "%s/" % domain_name
)).all()
if not domains:
domain = None
else:
domain = domains[0]
if not domain:
url_hash = hashlib.sha512(domain_url).hexdigest()
domain = Domain(url=domain_url, url_hash=url_hash, name=domain_name)
db.add(domain)
db.flush()
db.commit()
publish_method(dumps({
'type': 'new-domain',
'domainUrl': str(domain_url)
}))
from holmes.models import Limiter
connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS
Limiter.add_or_update_limiter(db, domain_url, connections)
return domain
def post(self):
access_token = self.request.headers.get('X-AUTH-HOLMES', None)
if access_token is None:
self.set_status(403)
self.write_json({'reason': 'Empty access token'})
return
result = yield User.authenticate(
access_token,
self.application.http_client.fetch,
self.db,
self.application.config
)
if result and result.get('user', None) is None:
self.set_status(403)
self.write_json({'reason': 'Not authorized user.'})
return
post_data = loads(self.request.body)
url = post_data.get('url', None)
connections = self.application.config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS
value = post_data.get('value', connections)
if not url and not value:
self.set_status(400)
self.write_json({'reason': 'Not url or value'})
return
result = Limiter.add_or_update_limiter(self.db, url, value)
yield self.cache.remove_domain_limiters_key()
self.write_json(result)
def add_domain(cls, url, db, publish_method, config, girl, default_violations_values, violation_definitions, cache):
from holmes.models import Domain, DomainsViolationsPrefs
from holmes.material import expire_materials
domain_name, domain_url = get_domain_from_url(url)
domains = (
db.query(Domain)
.filter(
or_(
Domain.name == domain_name,
Domain.name == domain_name.rstrip("/"),
Domain.name == "%s/" % domain_name,
)
)
.all()
)
if not domains:
domain = None
else:
domain = domains[0]
if not domain:
url_hash = hashlib.sha512(domain_url).hexdigest()
domain = Domain(url=domain_url, url_hash=url_hash, name=domain_name)
db.add(domain)
db.flush()
expire_materials(girl)
publish_method(dumps({"type": "new-domain", "domainUrl": str(domain_url)}))
keys = default_violations_values.keys()
DomainsViolationsPrefs.insert_default_violations_values_for_domain(
db, domain, keys, violation_definitions, cache
)
from holmes.models import Limiter
connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS
Limiter.add_or_update_limiter(db, domain_url, connections)
return domain
def add_domain(cls, url, db, publish_method, config, girl,
default_violations_values, violation_definitions, cache):
from holmes.models import Domain, DomainsViolationsPrefs
from holmes.material import expire_materials
domain_name, domain_url = get_domain_from_url(url)
domains = db.query(Domain).filter(
or_(Domain.name == domain_name,
Domain.name == domain_name.rstrip('/'),
Domain.name == "%s/" % domain_name)).all()
if not domains:
domain = None
else:
domain = domains[0]
if not domain:
url_hash = hashlib.sha512(domain_url).hexdigest()
domain = Domain(url=domain_url,
url_hash=url_hash,
name=domain_name)
db.add(domain)
db.flush()
expire_materials(girl)
publish_method(
dumps({
'type': 'new-domain',
'domainUrl': str(domain_url)
}))
keys = default_violations_values.keys()
DomainsViolationsPrefs.insert_default_violations_values_for_domain(
db, domain, keys, violation_definitions, cache)
from holmes.models import Limiter
connections = config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS
Limiter.add_or_update_limiter(db, domain_url, connections)
return domain
def test_get_limiters_for_domains(self):
self.db.query(Limiter).delete()
self.db.query(Domain).delete()
for i in range(2):
DomainFactory.create()
value = 2
url = 'http://globo.com/'
Limiter.add_or_update_limiter(self.db, url, value)
DomainFactory.create(url=url, name='globo.com')
active_domains = self.db.query(Domain).all()
domains = Limiter.get_limiters_for_domains(self.db, active_domains)
expect(domains).to_length(1)
expect(str(domains[0])).to_equal(url)
def test_can_add_or_update_limiter(self):
self.db.query(Limiter).delete()
limiters = Limiter.get_all(self.db)
expect(limiters).to_equal([])
# Add
url = 'http://globo.com/'
value = 2
Limiter.add_or_update_limiter(self.db, url, value)
limiter = Limiter.by_url(url, self.db)
expect(limiter.value).to_equal(2)
# Update
url = 'http://globo.com/'
value = 3
Limiter.add_or_update_limiter(self.db, url, value)
limiter = Limiter.by_url(url, self.db)
expect(limiter.value).to_equal(3)
def post(self):
if not self.validate_superuser():
return
post_data = loads(self.request.body)
url = post_data.get('url', None)
connections = self.config.DEFAULT_NUMBER_OF_CONCURRENT_CONNECTIONS
value = post_data.get('value', connections)
if not url and not value:
self.set_status(400)
self.write_json({
'reason': 'Not url or value',
'description': self._('Not url or value')
})
return
result = Limiter.add_or_update_limiter(self.db, url, value)
yield self.cache.remove_domain_limiters_key()
self.write_json(result)
