def all_list(request):
""".. :py:method::
webanan的主页views方法
"""
domains_list = []
domain_list = Domain.objects().all()
for do in domain_list:
domains_list.append(do.name)
title = u'各主机文章数量'
per_dict = {}
domains = Domain.objects().all()
article_all = Article.objects().count()
for domain in domains:
categories = Category.objects(belong_Domain=domain).all()
domain_count = 0
for cate in categories:
article_num = Article.objects(belong_cate=cate).count()
domain_count = domain_count + article_num
per_dict[domain.name] = float(u"%.2f" % (float(domain_count)/article_all))
per_dict = OrderedDict(sorted(per_dict.items(), key=lambda t: t[1]))
return render_to_response('webanan_list.html',{
'title': title,
'per_dict':per_dict,
'domains':domains_list,
})
def sec_list(request,domain=None):
""".. :py:method::
webanan的各下属分类的views方法
"""
domains_list = []
domain_list = Domain.objects().all()
for do in domain_list:
domains_list.append(do.name)
title = u'{}分类文章数量'.format(domain)
per_dict = {}
domain_obj = Domain.objects(name=domain).first()
if domain_obj:
categories = Category.objects(belong_Domain=domain_obj).all()
article_all = 0
for cate in categories:
article_all = article_all + Article.objects(belong_cate=cate).count()
for cate in categories:
article_num = Article.objects(belong_cate=cate).count()
per_dict[cate.name] = float(u"%.2f" % (float(article_num)/article_all))
per_dict = OrderedDict(sorted(per_dict.items(), key=lambda t: t[1]))
else:
raise Http404
return render_to_response('webanan_list.html',{
'title': title,
'per_dict':per_dict,
'domains':domains_list,
})
def upsert_author(info):
with db_session:
a = Author.get(author_id=info['author_id'])
if a:
logger.debug('author has existed, author_id=%s', info['author_id'])
return
# logger.debug(info)
author = Author(author_id=info['author_id'])
author.name = info['name'] if info['name'] else ''
author.image_url = info['image_url'] if info['image_url'] else ''
author.organization = info['organization'] if info[
'organization'] else ''
author.home_page = info['home_page'] if info['home_page'] else ''
author.paper_count = info['paper_count']
author.citied_count = info['cited_count']
for d in info['domains']:
if d['id'] == '':
continue
ds = Domain.get(domain_id=d['id'])
if ds:
author.domains.add(ds)
else:
domain = Domain(domain_id=d['id'])
domain.name = d['name']
author.domains.add(domain)
def make_url_model(url, site): """ This should on occur once per newly created URL, the linked count is set to zero if it is a new site added to database """ now = datetime.now() base_url = 'http://links.ep.io/' url_model = Url() url_model.url = url url_short = url try: domain = Domain.objects.get(site=site) domain.linked_count += 1 domain.date_updated = now domain.save() except: domain = Domain(site=site, linked_count=1, date_updated= now) domain.save() url_model.site = domain url_model.date_time_created = datetime.now() url_model.linked_count = 1 url_model.save() url_model.url_shortened = base_url + encode_62(url_model.pk) print url_model.url_shortened url_model.save() return url_model
def test_job_should_emit_status_for_all_urls(self):
first_domain = Domain("http://example.com")
first_domain.crawled = True
second_domain = Domain("http://example.com/2")
job = Job([first_domain,second_domain])
self.assertEqual(job.get_status()["completed"], 1)
self.assertEqual(job.get_status()["inprogress"], 1)
def add_domain(permission):
body = request.get_json()
try:
domain = Domain(body['domain_name'])
domain.insert()
return jsonify({'success': True, 'domain_id': domain.id})
except:
abort(422)
def run(self):
# Fixtures
if Domain.objects.count() == 0:
domain = Domain(scheme='http', netloc='sametmax.com')
url = Url(path='/')
domain.urls.append(url)
domain.save()
for domain in Domain.objects:
self._find_links(domain)
def post(self, domain_name):
domain = get_domain_or_404(domain_name, allow_none=True)
if not domain:
domain = Domain(name=domain_name, clickcount=0, money=0., status=0.)
# example content:
# "firstvisit":"center","secondvisit":"center","heading":"Vielen Dank!","subheading":"Dein Klick auf domain.hiv hat soeben einen Gegenwert von 1 ct ausgelöst.","claim":"Wir sind Teil der Bewegung","about":"Über dotHIV","vote":"Vote","activated":"Bisher aktiviert:","currency":"€","corresponding":"entspricht","clicks":"Klicks"
domain.content = self.request.body
domain.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.set_status(204)
def create(self):
domain = Domain(
name = self.data.get('name'),
is_private = self.data.get('is_private')
)
if domain.is_https() and domain.is_valid_address():
domain.save()
return True
else:
return False
def index(request):
form = DomainForm()
if request.method == "POST":
form = DomainForm(data=request.POST)
if form.is_valid():
domain = form.cleaned_data["domain"]
d = Domain(user=request.user, domain=domain)
d.save()
scan.delay(d.id)
messages.add_message(request, messages.INFO, 'Domain başarılı bir şekilde alındı. '
'Durum takibini Taramalar sayfasından yapabilirsiniz')
return render(request, "index.html", locals())
def index(request):
form = DomainForm()
if request.method == "POST":
form = DomainForm(data=request.POST)
if form.is_valid():
domain = form.cleaned_data["domain"]
d = Domain(user=request.user, domain=domain)
d.save()
scan.delay(d.id)
messages.add_message(
request, messages.INFO, 'Domain başarılı bir şekilde alındı. '
'Durum takibini Taramalar sayfasından yapabilirsiniz')
return render(request, "index.html", locals())
def test_domains(self):
protein_data = [
('NM_000184', 'HBG2'),
('NM_000517', 'HBA2'),
('NM_024694', 'ADGB')
]
proteins: Dict[str, Protein] = {} # gene_name -> protein
for refseq, gene_name in protein_data:
g = Gene(name=gene_name)
proteins[gene_name] = Protein(gene=g, refseq=refseq)
sub_domain = InterproDomain(
accession='IPR012292',
description='Globin_dom',
occurrences=[
Domain(start=3, end=142, protein=proteins['HBA2']),
Domain(start=784, end=869, protein=proteins['ADGB'])
]
)
domain = InterproDomain(
accession='IPR009050',
description='Globin-like',
children=[sub_domain],
occurrences=[
Domain(start=3, end=147, protein=proteins['HBG2']),
Domain(start=1, end=142, protein=proteins['HBA2'])
]
)
db.session.add_all([domain, sub_domain, *proteins.values()])
with NamedTemporaryFile('w+') as f:
gmt_from_domains(path=f.name, include_sub_types=True)
accepted_lines = [
'IPR009050\tGlobin-like\t' + '\t'.join(genes) + '\n'
for genes in permutations(['HBA2', 'HBG2', 'ADGB'])
] + [
'IPR012292\tGlobin_dom\t' + '\t'.join(genes) + '\n'
for genes in permutations(['HBA2', 'ADGB'])
]
gmt_lines = f.readlines()
assert all(line in accepted_lines for line in gmt_lines)
assert len(gmt_lines) == 2
def list_redirects_json(self):
redirects = []
domains = Domain.query(Domain.redirect_enabled == True)
for domain in domains.iter():
host = urlparse(domain.redirect_url)
rule = dict(hosts=[host.netloc])
rule["rules"] = [
{"from": "^" + re.escape(domain.redirect_url), "to": "http://" + domain.name + "/"}]
if re.match('^www\.', host.netloc):
# Add rule without www
rule["hosts"].append(host.netloc[4:])
nowww_redirect_url = re.sub('//www\.', '//', domain.redirect_url)
rule["rules"].append({"from": "^" + re.escape(nowww_redirect_url), "to": "http://" + domain.name + "/"})
else:
# Add rule with www
rule["hosts"].append("www." + host.netloc)
www_redirect_url = re.sub('(https*://)', '\g<1>www.', domain.redirect_url)
rule["rules"].append({"from": "^" + re.escape(www_redirect_url), "to": "http://" + domain.name + "/"})
rule["exceptions"] = []
redirects.append(rule)
self.response.headers['Content-Type'] = 'application/json'
self.response.write(json.encode(redirects))
def addomain():
form = FormDomain()
if request.method == 'GET':
return render_template('addomain.html', form=form)
if request.method == 'POST':
if form.validate_on_submit():
form = FormDomain(request.form)
mycompany = form.company.data
mydomain = form.domain.data
dominio = Domain.query.filter_by(name=mydomain).first()
if dominio:
flash('Domínio já existe', 'notification is-danger')
else:
query = Domain(company=mycompany, name=mydomain)
db.session.add(query)
flash("Domínio cadastrado", 'notification is-success')
return redirect(url_for('addomain'))
else:
flash('Fala ao cadastrar o domínio', 'notification is-danger')
return render_template('addomain.html', form=form)
def test_callback_new_domain(self):
self.expect_indieauth_check()
self.expect_site_fetch()
self.mox.ReplayAll()
resp = self.callback()
self.assertEqual('http://localhost/#!Authorized you for snarfed.org.',
urllib.parse.unquote_plus(resp.headers['Location']))
self.assert_entities_equal([
Domain(id='snarfed.org',
tokens=['towkin'],
auth=self.auth_entity.key),
],
Domain.query().fetch(),
ignore=('created', 'updated'))
def auth(self):
"""Loads the source and token and checks that they're valid.
Expects token in the `token` query param, source in `key` or `username`.
Raises: :class:`HTTPException` with HTTP 400 if the token or source are
missing or invalid
Returns: BrowserSource or None
"""
# Load source
source = util.load_source(self, param='key')
if not source:
self.abort(
404,
f'No account found for {self.gr_source().NAME} user {key or username}'
)
# Load and check token
token = util.get_required_param(self, 'token')
for domain in Domain.query(Domain.tokens == token):
if domain.key.id() in source.domains:
return source
self.abort(
403,
f'Token {token} is not authorized for any of: {source.domains}')
def create_domain(self, domain):
domain_query_runner = self.db.query(Domain).filter(
Domain.domain == domain)
if domain_query_runner.count() == 0:
domain_entity = Domain(domain=domain)
self.db.add(domain_entity)
self.db.commit()
def update_clicks(domainname):
"""
Updates the memcached total clickcount value
"""
domain = Domain.query(Domain.name == str(domainname)).get()
if domain:
memcache.set(get_cache_key(domain), domain.clickcount)
def setUp(self):
super().setUp()
self.actor['numeric_id'] = '212038'
self.source = Facebook.new(actor=self.actor)
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
self.auth = f'token=towkin&key={self.source.key.urlsafe().decode()}'
self.mox.StubOutWithMock(gr_facebook, 'now_fn')
def update_total_clicks():
"""
Updates the memcached total clickcount value
"""
clicks = 0
for domain in Domain.query():
clicks = clicks + domain.clickcount
memcache.set('clicks_total', clicks)
def post(self):
token = util.get_required_param(self, 'token')
domains = [d.key.id() for d in Domain.query(Domain.tokens == token)]
if not domains:
self.abort(404, f'No registered domains for token {token}')
self.output(domains)
def _getDbDomains(self, cur):
domains = []
db_domains = cur.execute("SELECT * from dbd$domains").fetchall()
domain_metadata = self._getMetadata(cur, "dbd$domains")
for domain in db_domains:
domain_dictionary = dict(zip(domain_metadata, list(domain)))
tmp = Domain(domain_dictionary)
data_type = cur.execute(
"SELECT * FROM dbd$data_types WHERE id = :type_id", {
"type_id": tmp.data_type_id
}).fetchall()
tmp.type = data_type[0][1]
domains.append(tmp)
return domains
def saveDomainAndReturnId(domain_name,domain_cname,domain_type,
domain_status,disId, etag, project_id,
username, new_ip_list,test_url,ignore_param_req):
domain = Domain(domain_name=domain_name,
domain_cname=domain_cname,
domain_type=domain_type,
domain_status=domain_status,
distribution_id=disId,
etag=etag,
project_id=project_id,
username=username,
ip_list=new_ip_list,
test_url=test_url,
ignore_param_req=ignore_param_req)
domain.save()
domain_id = Domain.objects.latest('id').id
return domain_id
def welcome():
try:
domains = list(Domain.select())
except IndexError:
domains = []
about = settings.ABOUT
default_domain = settings.MY_DOMAINS[0]
return render_template('welcome.html', **locals())
def _save(self, results):
for href in results:
uri = urlparse(href)
try:
domain = Domain.objects.get(netloc=uri.netloc)
except DoesNotExist as e:
domain = Domain(scheme=uri.scheme, netloc=uri.netloc)
path = uri.path
if not path:
path = '/'
url = Url(path=path)
domain.urls.append(url)
domain.save()
def get_or_create_domain(self, domain: str) -> Tuple[Domain, bool]:
# we need create redis cache
try:
return self.store.get_cache(domain), False
except ValueError as exc:
instance = Domain(domain)
self.store.set_cache(domain, instance, instance.cache_time_out)
return instance, True
def welcome():
try:
domains = list(Domain.select())
except IndexError:
domains = []
about = settings.ABOUT
default_domain = settings.MY_DOMAINS[0]
return render_template('welcome.html', **locals())
async def letsencrypt_fork(self, instance: Domain):
domain: str = instance.domain
command = [
'letsencrypt',
'certonly',
f'--cert-name "{domain}"',
'--manual',
# f'--manual-auth-hook {self.dns_acme_auth}',
f'--manual-auth-hook ./dns_acme_auth.py',
# f'--deploy-hook {self.dns_acme_deploy}',
f'--deploy-hook ./dns_acme_deploy.py',
'--force-renewal',
'--preferred-challenges=dns',
'--register-unsafely-without-email',
'--manual-public-ip-logging-ok',
'--server https://acme-v02.api.letsencrypt.org/directory',
'--agree-tos',
'--quiet',
f'-d "{domain}"',
f'-d "*.{domain}"',
]
shell_command = " ".join(command)
process = await asyncio.create_subprocess_shell(
shell_command,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
# if success process.returncode = 0
# if error process.returncode = 1
stdout, stderr = await process.communicate()
instance: Domain = self.store.get_cache(domain)
instance.on_success = stdout.decode().replace("\n", " ").strip()
instance.on_error = stderr.decode().replace("\n", " ").strip()
logger.debug(
f"process.communicate: {process.returncode} {instance.serialize()}"
)
if process.returncode == 0:
instance.status = SUCCESS
else:
instance.status = FAILED
self.store.set_cache(domain, instance, instance.cache_time_out)
def setUp(self):
super().setUp()
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
FakeBrowserSource.gr_source = FakeGrSource()
self.actor['fbs_id'] = '222yyy'
self.source = FakeBrowserSource.new(actor=self.actor).put()
self.auth = f'token=towkin&key={self.source.urlsafe().decode()}'
self.other_source = FakeBrowserSource(id='333zzz', domains=['foo.com']).put()
for a in self.activities:
a['object']['author'] = self.actor
self.activities_no_replies = copy.deepcopy(self.activities)
for a in self.activities_no_replies:
del a['object']['replies']
del a['object']['tags']
def domain_acquire(self, request):
user = self.authenticate(request)
validator = Validator(request)
user_domain = validator.new_user_domain()
device_mac_address = validator.device_mac_address()
device_name = validator.string('device_name', required=True)
device_title = validator.string('device_title', required=True)
check_validator(validator)
with self.create_storage() as storage:
domain = storage.get_domain_by_name(user_domain)
if domain and domain.user_id != user.id:
raise servicesexceptions.parameter_error(
'user_domain', 'User domain name is already in use')
update_token = util.create_token()
if not domain:
domain = Domain(user_domain,
device_mac_address,
device_name,
device_title,
update_token=update_token)
domain.user = user
storage.add(domain)
else:
domain.update_token = update_token
domain.device_mac_address = device_mac_address
domain.device_name = device_name
domain.device_title = device_title
return domain
def setUp(self):
super(InstagramTest, self).setUp()
self.handler.messages = []
self.inst = Instagram.new(
self.handler,
actor={
'objectType': 'person',
'id': 'tag:instagram.com,2013:420973239',
'username': '******',
'displayName': 'Ryan Barrett',
'url': 'https://snarfed.org/',
'image': {
'url': 'http://pic.ture/url'
},
# ...
})
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
def get_domain_or_404(name, allow_none=False):
if not name:
webapp2.abort(404)
domain = Domain.query(Domain.name == name).get()
if not domain and not allow_none:
webapp2.abort(404)
return domain
def list_redirects_csv(self):
self.response.headers['Content-Type'] = 'text/csv'
writer = csv.writer(self.response, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
domains = Domain.query(Domain.redirect_enabled == True)
for domain in domains.iter():
writer.writerow([domain.name, domain.redirect_url])
def test_get(self):
with db_session:
a = Author.get(author_id='ded2d8cd-73c6-41c0-bcf7-db2983805231')
logger.debug(a.organization)
self.assertEqual(a.name, 'Abraham Silberschatz')
d = Domain.get(domain_id='1.1')
logger.debug(d.name)
self.assertEqual(d.name, '算法与理论')
def get_or_create_domain(domain_string):
"""
check in the instance of the specific domain is saved .
If yes return an instance of domain, if not save an instance and return it
Params
domain_string - the domain name (ex. facebook.com): string
Return
an instance of domain object : Domain
"""
domain = db.Query(Domain).filter('name = ', domain_string).get()
if not domain:
domain = Domain()
domain.name = domain_string
db.put(domain)
return domain
def add_or_update_domain():
domain = Domain.get_or_insert(
util.domain_from_link(
util.replace_test_domains_with_localhost(
auth_entity.key.id())))
domain.auth = auth_entity.key
if state not in domain.tokens:
domain.tokens.append(state)
domain.put()
flash(f'Authorized you for {domain.key.id()}.')
def test_creating_new_domains(self):
domain = Domain()
domain.id = 42
domain.name = "Test VM"
domain.state = domain.RUNNING
domain.max_memory = 2*2024*2024
domain.memory = domain.max_memory
domain.vcpu = 1
domain.save()
all_domains_in_database = Domain.objects.all()
self.assertEquals(len(all_domains_in_database), 1)
only_domain_in_database = all_domains_in_database[0]
self.assertEquals(only_domain_in_database, domain)
self.assertEquals(only_domain_in_database.name, "Test VM")
def create_new_user(self, request):
validator = Validator(request)
email = validator.email()
password = validator.password()
user_domain = validator.new_user_domain(error_if_missing=False)
errors = validator.errors
if errors:
message = ", ".join(errors)
raise servicesexceptions.bad_request(message)
user = None
action = None
with self.create_storage() as storage:
by_email = storage.get_user_by_email(email)
if by_email and by_email.email == email:
raise servicesexceptions.conflict('Email is already registered')
if user_domain:
by_domain = storage.get_domain_by_name(user_domain)
if by_domain and by_domain.user_domain == user_domain:
raise servicesexceptions.conflict('User domain name is already in use')
update_token = util.create_token()
user = User(email, util.hash(password), not self.activate_by_email)
if user_domain:
domain = Domain(user_domain, None, update_token)
domain.user = user
user.domains.append(domain)
storage.add(domain)
if self.activate_by_email:
action = user.enable_action(ActionType.ACTIVATE)
storage.add(user)
if self.activate_by_email:
self.mail.send_activate(self.main_domain, user.email, action.token)
return user
def domain_acquire(self, request):
user = self.authenticate(request)
validator = Validator(request)
user_domain = validator.new_user_domain()
with self.create_storage() as storage:
domain = storage.get_domain_by_name(user_domain)
if domain and domain.user_id != user.id:
raise servicesexceptions.conflict('User domain name is already in use')
update_token = util.create_token()
if not domain:
domain = Domain(user_domain, None, update_token)
domain.user = user
storage.add(domain)
else:
domain.update_token = update_token
return domain
def dispatch_request(self):
token = request.values['token']
domains = [d.key.id() for d in Domain.query(Domain.tokens == token)]
if not domains:
indieauth_start = util.host_url(f'/indieauth/start?token={token}')
self.error(
f'Not connected to Bridgy. <a href="{indieauth_start}" target="_blank"">Connect now!</a>',
404)
return jsonify(domains)
def domain_acquire(self, request):
user = self.authenticate(request)
validator = Validator(request)
user_domain = validator.new_user_domain()
device_mac_address = validator.device_mac_address()
device_name = validator.string('device_name', required=True)
device_title = validator.string('device_title', required=True)
check_validator(validator)
with self.create_storage() as storage:
domain = storage.get_domain_by_name(user_domain)
if domain and domain.user_id != user.id:
raise servicesexceptions.parameter_error('user_domain', 'User domain name is already in use')
update_token = util.create_token()
if not domain:
domain = Domain(user_domain, device_mac_address, device_name, device_title, update_token=update_token)
domain.user = user
storage.add(domain)
else:
domain.update_token = update_token
domain.device_mac_address = device_mac_address
domain.device_name = device_name
domain.device_title = device_title
return domain
def _getFields(self, xml):
if xml.nodeName != "table":
raise ValueError("Is not a table")
fields = []
for field in xml.getElementsByTagName("field"):
tmp = Field()
domain = Domain()
for attributeName, attributeValue in field.attributes.items():
if attributeName.lower() == "name":
tmp.name = attributeValue
elif attributeName.lower() == "rname":
tmp.rname = attributeValue
elif attributeName == "domain":
tmp.domain = attributeValue
elif attributeName.lower() == "domain.name":
domain.name = attributeValue
elif attributeName.lower() == "domain.char_length":
domain.char_length = attributeValue
elif attributeName.lower() == "domain.precision":
domain.precision = attributeValue
elif attributeName.lower() == "domain.scale":
domain.scale = attributeValue
elif attributeName.lower() == "domain.type":
domain.type = attributeValue
elif attributeName.lower() == "description":
tmp.description = attributeValue
elif attributeName.lower() == "props":
for prop in attributeValue.split(", "):
if prop == "input":
tmp.input = True
elif prop == "edit":
tmp.edit = True
elif prop == "show_in_grid":
tmp.show_in_grid = True
elif prop == "show_in_details":
tmp.show_in_details = True
elif prop == "is_mean":
tmp.is_mean = True
elif prop == "autocalculated":
tmp.autocalculated = True
elif prop == "required":
tmp.required = True
else:
raise ValueError(
"Invalid format of props string: {}".format(
attributeValue))
else:
raise ValueError(
"Incorrect attribute name \"{}\" in tag \"{}\" ".
format(attributeName, field.nodeName))
if tmp.domain is None:
tmp.domain = domain
fields.append(tmp)
return fields
def finish(self, auth_entity, state=None):
if not auth_entity:
return
assert state
domain = Domain.get_or_insert(util.domain_from_link(auth_entity.key.id()))
domain.auth = auth_entity.key
if state not in domain.tokens:
domain.tokens.append(state)
domain.put()
self.messages.add(f'Authorized you for {domain.key.id()}.')
self.redirect('/')
def get_domain_or_404(name, allow_none=False):
"""Gets a domain entity for the given name.
Aborts with 404 if entity doesn't exist and if that is not allowed.
"""
if not name:
webapp2.abort(404)
domain = Domain.query(Domain.name==name).get()
if not domain and not allow_none:
webapp2.abort(404)
return domain
def get(self):
if self.request.headers.get('X-AppEngine-Cron') != "true":
self.error(403)
return
total = 0
for domain in Domain.query():
mem_count = memcache.get(get_cache_key(domain))
if mem_count:
total = total + mem_count
else:
total = total + domain.clickcount
ct = models.ClickcountDate()
ct.clickcount = total
ct.put()
def main():
unvisited_links = session.query(Link).\
filter(Link.visited_at == None).all()
if len(unvisited_links) == 0:
print("Nothing to visit right now.")
for link in unvisited_links:
try:
r = requests.get(link)
soup = BeautifulSoup(r.text, 'html.parser')
for site_url in set([o.get('href') for o in soup.find_all('a')]):
if site_url is None:
continue
url = site_url
if not is_url(site_url):
url = urljoin(link.get_domain(), site_url)
print('Found: {}'.format(url))
l = session.query(Link).\
filter(Link.url == url).first()
if l is not None:
continue
l = Link(url=url)
domain = l.get_domain()
domain_in_db = session.query(Domain).\
filter(Domain.url == domain).\
first()
if domain_in_db in None:
print("Found new domain: {}".format(domain))
domain_in_db = Domain(url=domain)
save(domain_in_db)
l.domain = domain_in_db
save(l)
except:
print('Something went wrong.')
finally:
link.visited_at = datetime.now()
save(link)
def add_new_domain(domain):
""" Add a new domain to be perpetually resolved
"""
# Check to see if this domain is already being tracked
exists = Domain.search().filter('term', domain=domain).execute()
if exists:
raise DomainExists(
f"Supplied domain {domain} is already being tracked by Darkstar")
# This is a new domain, lets try to resolve it for the first time
try:
hostname, aliastlist, ipaddrlist = gethostbyname_ex(domain)
except gaierror as e:
current_app.logger.exception(e)
raise e
def process_item(self, item, spider):
if 'URLPipeline' not in getattr(spider, "pipelines"):
return item
if self.is_subdomain(item['url']):
print 'subdomain: ', item['url']
return item
try:
dmain = Domain(url_domain=item['url'],
start_url=item['source'],
status_code=item['status'],
domain_ext=item['domain_ext'])
db.session.add(dmain)
db.session.commit()
except IntegrityError:
db.session.rollback()
return item
def post(self):
reqs = request.get_json(force=True)
if not reqs:
raise JsonRequiredError()
try:
# check if domain pair existed
u = Domain.query.filter_by(url=reqs['url'],
port=reqs['port']).first()
if not (u is None):
raise UserExistedError()
reqs['user_id'] = current_identity.id
current_identity.num_domains += 1
reqs['id'] = current_identity.num_domains
u = Domain(**reqs)
db.session.add(u)
db.session.commit()
return {}, 201, {'Location': '/domains/%d' % u.relative_id}
except KeyError:
raise JsonInvalidError()
def get(self):
if self.request.headers.get('X-AppEngine-Cron') != "true":
self.error(403)
return
for domain in Domain.query():
# TODO: Maybe maintain a list of domains which do have a memcache counter?
k = get_cache_key(domain)
new_count = memcache.get(k)
if not new_count:
logging.info("No clickcount in memcache for: %s", domain.name)
elif new_count < domain.clickcount:
logging.error("New clickcount for %s would be %d, is %d", domain.name, new_count, domain.clickcount)
memcache.delete(k)
elif new_count == domain.clickcount:
logging.info("clickcount for %s unchanged: %d", domain.name, new_count)
else:
old_count = domain.clickcount
domain.clickcount = new_count
domain.put()
logging.info("clickcount for %s: %d -> %d", domain.name, old_count, new_count)
def post(self, domain_name):
domain = get_domain_or_404(domain_name, allow_none=True)
if not domain:
domain = Domain(name=domain_name, clickcount=0)
data = self.request.body.strip()
config = json.decode(data) if len(data) > 0 else {}
config['__locales'] = []
if 'strings' in config:
for locale in config['strings']:
config['__locales'].append(locale)
domain.content = json.encode(config)
# Redirects
domain.redirect_enabled = False
domain.redirect_url = None
if 'redirect_url' in config:
domain.redirect_enabled = True
domain.redirect_url = config['redirect_url']
domain.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.set_status(204)
def get_domains(self):
"""return the all domains as a generator, from start to finish, inclusive.
"""
return Domain.filter(self.rd)
def post(self):
params = self.request.params
if not 'domain' in params:
self.abort(404)
cache_key_domain = 'domain:d:' + params['domain']
domain = memcache.get(cache_key_domain)
if domain == 0:
self.abort(404)
return
if not domain:
domain = Domain.query(Domain.name == params['domain']).get()
if not domain:
# store for 60 seconds
memcache.set(cache_key_domain, 0, 60)
self.abort(404)
return
else:
memcache.set(cache_key_domain, domain, 60)
# count clicks from outside the iframe that have a previous (pt) and current (ct) visit timestamp
# previous timestamp may be empty if the client has no record of this user having visited the domain before)
# current timestamp may not be empty
# for simplicity timestamps are expressed as integers (unix timestamps) in the users timezone
try:
if 'from' in params and 'pt' in params and 'ct' in params:
if params['from'] == 'outside':
current_visit = int(params['ct']) if params['ct'] != "" else 0
if current_visit < 1397340000000:
raise Error("Invalid value for 'ct': '%s'" % params['ct'])
previous_visit = int(params['pt']) if params['pt'] != "" else current_visit - COUNT_THRESHOLD
if previous_visit < 1397340000000:
raise Error("Invalid value for 'pt': '%s'" % params['pt'])
if current_visit < previous_visit:
raise Error("Value '%s' for 'ct' must be greater or equal than value '%s' for 'pt'" % (params['ct'], params['pt']))
if current_visit - previous_visit >= COUNT_THRESHOLD:
if memcache.incr(jobs.get_cache_key(domain)) is None:
jobs.init_clicks(domain)
if memcache.incr("clicks_total") is None:
jobs.init_clicks_total()
except Error as e:
self.response.headers['Content-Type'] = 'application/api-problem+json'
self.response.set_status(403)
self.response.write(
json.encode(
{
"problemType": "https://github.com/dothiv/clickcounter-backend/wiki/problem-invalid-request",
"title": "%s" % e.message
}
)
)
return
# explicit request to have content-type application/json
self.response.headers['Content-Type'] = 'application/json'
# allow origin
if 'Origin' in self.request.headers:
self.response.headers['Access-Control-Allow-Origin'] = self.request.headers['Origin']
config = createDomainConfig(domain, getClientLocales(self.request.headers.get('accept-language')))
self.response.write(json.encode(config))
def delete(self, domain_name):
domain = Domain.query(Domain.name == domain_name).get()
if domain:
domain.key.delete()
self.response.headers['Content-Type'] = 'text/plain'
self.response.set_status(204)
#!/usr/bin/env python
# coding:utf-8
import redis
from torndb import Connection
from models import Domain, Flow, Rule
import settings
rd = redis.Redis(settings.REDIS_HOST, settings.REDIS_PORT)
db = Connection("127.0.0.1", "spider", user="******", password="******")
domains = db.query("select * from domain")
for d in domains:
domain = Domain(rd, d["domain"])
domain.update(d)
if "scheduled" not in domain:
domain["scheduled"] = 0
rule_datas = db.query("select * from rule where domain = %s", d["domain"])
for rule_data in rule_datas:
rule = Rule(rd, d["domain"])
rule.update(rule_data)
flows = db.query("select * from flow where domain = %s", d["domain"])
for f in flows:
flow = Flow(rd, f["flow"])
flow.update(f)
if "scheduled" not in flow:
flow["scheduled"] = 0
def get(self):
self.render('schedule.html', domains=Domain.filter(self.rdb))
def test_job_should_emit_status_for_crawled_urls(self):
first_domain = Domain("http://example.com")
first_domain.crawled = True
job = Job([first_domain])
self.assertEqual(job.get_status()["completed"], 1)