示例#1
0
def all_list(request):
    """.. :py:method::
    webanan的主页views方法
    """
    domains_list = []
    domain_list = Domain.objects().all()
    for do in domain_list:
        domains_list.append(do.name)
    title = u'各主机文章数量'
    per_dict = {}
    domains = Domain.objects().all()
    article_all = Article.objects().count()
    for domain in domains:
        categories = Category.objects(belong_Domain=domain).all()
        domain_count = 0
        
        for cate in categories:
            article_num = Article.objects(belong_cate=cate).count()
            domain_count = domain_count + article_num
        per_dict[domain.name] = float(u"%.2f" % (float(domain_count)/article_all))
    per_dict = OrderedDict(sorted(per_dict.items(), key=lambda t: t[1]))
    return render_to_response('webanan_list.html',{
        'title': title,
        'per_dict':per_dict,
        'domains':domains_list,
        })
示例#2
0
def sec_list(request,domain=None):
    """.. :py:method::
    webanan的各下属分类的views方法
    """
    domains_list = []
    domain_list = Domain.objects().all()
    for do in domain_list:
        domains_list.append(do.name)
    title = u'{}分类文章数量'.format(domain)
    per_dict = {}
    domain_obj = Domain.objects(name=domain).first()
    if domain_obj:
        categories = Category.objects(belong_Domain=domain_obj).all()
        article_all = 0
        for cate in categories:
            article_all = article_all + Article.objects(belong_cate=cate).count()
        for cate in categories:
            article_num = Article.objects(belong_cate=cate).count()
            per_dict[cate.name] = float(u"%.2f" % (float(article_num)/article_all))
        per_dict = OrderedDict(sorted(per_dict.items(), key=lambda t: t[1]))
    else:
        raise Http404
    return render_to_response('webanan_list.html',{
                'title': title,
                'per_dict':per_dict,
                'domains':domains_list,
                })
示例#3
0
文件: db.py 项目: GivenZeng/sospider
def upsert_author(info):
    with db_session:
        a = Author.get(author_id=info['author_id'])
        if a:
            logger.debug('author has existed, author_id=%s', info['author_id'])
            return
        # logger.debug(info)
        author = Author(author_id=info['author_id'])
        author.name = info['name'] if info['name'] else ''
        author.image_url = info['image_url'] if info['image_url'] else ''
        author.organization = info['organization'] if info[
            'organization'] else ''
        author.home_page = info['home_page'] if info['home_page'] else ''
        author.paper_count = info['paper_count']
        author.citied_count = info['cited_count']

        for d in info['domains']:
            if d['id'] == '':
                continue
            ds = Domain.get(domain_id=d['id'])
            if ds:
                author.domains.add(ds)
            else:
                domain = Domain(domain_id=d['id'])
                domain.name = d['name']
                author.domains.add(domain)
示例#4
0
文件: views.py 项目: myusuf3/links
def make_url_model(url, site): 
	""" This should on occur once per newly created URL, the linked count is set to zero if it
		is a new site added to database
	"""
	now = datetime.now()
	base_url = 'http://links.ep.io/'
	url_model = Url()
	url_model.url = url
	url_short  = url
	try: 
		domain = Domain.objects.get(site=site)
		domain.linked_count += 1
		domain.date_updated = now
		domain.save()
	except:
		domain = Domain(site=site, linked_count=1, date_updated= now)
		domain.save()
	url_model.site = domain
	url_model.date_time_created = datetime.now()
	url_model.linked_count = 1
	url_model.save()
	url_model.url_shortened = base_url + encode_62(url_model.pk)
	print url_model.url_shortened
	url_model.save()
	return url_model
示例#5
0
  def test_job_should_emit_status_for_all_urls(self):
    first_domain = Domain("http://example.com")
    first_domain.crawled = True
    second_domain = Domain("http://example.com/2")
    job = Job([first_domain,second_domain])

    self.assertEqual(job.get_status()["completed"], 1)
    self.assertEqual(job.get_status()["inprogress"], 1)
 def add_domain(permission):
     body = request.get_json()
     try:
         domain = Domain(body['domain_name'])
         domain.insert()
         return jsonify({'success': True, 'domain_id': domain.id})
     except:
         abort(422)
示例#7
0
    def run(self):
        # Fixtures
        if Domain.objects.count() == 0:
            domain = Domain(scheme='http', netloc='sametmax.com')
            url = Url(path='/')
            domain.urls.append(url)
            domain.save()

        for domain in Domain.objects:
            self._find_links(domain)
示例#8
0
  def post(self, domain_name):
    domain = get_domain_or_404(domain_name, allow_none=True)
    if not domain:
      domain = Domain(name=domain_name, clickcount=0, money=0., status=0.)
    # example content:
    # "firstvisit":"center","secondvisit":"center","heading":"Vielen Dank!","subheading":"Dein Klick auf domain.hiv hat soeben einen Gegenwert von 1 ct ausgelöst.","claim":"Wir sind Teil der Bewegung","about":"Über dotHIV","vote":"Vote","activated":"Bisher aktiviert:","currency":"€","corresponding":"entspricht","clicks":"Klicks"
    domain.content = self.request.body
    domain.put()

    self.response.headers['Content-Type'] = 'text/plain'
    self.response.set_status(204)
示例#9
0
	def create(self):
		domain = Domain(
			name = self.data.get('name'),
			is_private = self.data.get('is_private')
		)
		
		if domain.is_https() and domain.is_valid_address(): 
			domain.save()
			return True
		else:
			return False
示例#10
0
def index(request):
    form = DomainForm()
    if request.method == "POST":
        form = DomainForm(data=request.POST)
        if form.is_valid():
            domain = form.cleaned_data["domain"]
            d = Domain(user=request.user, domain=domain)
            d.save()
            scan.delay(d.id)
            messages.add_message(request, messages.INFO, 'Domain başarılı bir şekilde alındı. '
                                                         'Durum takibini Taramalar sayfasından yapabilirsiniz')
    return render(request, "index.html", locals())
示例#11
0
def index(request):
    form = DomainForm()
    if request.method == "POST":
        form = DomainForm(data=request.POST)
        if form.is_valid():
            domain = form.cleaned_data["domain"]
            d = Domain(user=request.user, domain=domain)
            d.save()
            scan.delay(d.id)
            messages.add_message(
                request, messages.INFO, 'Domain başarılı bir şekilde alındı. '
                'Durum takibini Taramalar sayfasından yapabilirsiniz')
    return render(request, "index.html", locals())
示例#12
0
    def test_domains(self):

        protein_data = [
            ('NM_000184', 'HBG2'),
            ('NM_000517', 'HBA2'),
            ('NM_024694', 'ADGB')
        ]
        proteins: Dict[str, Protein] = {}    # gene_name -> protein

        for refseq, gene_name in protein_data:
            g = Gene(name=gene_name)
            proteins[gene_name] = Protein(gene=g, refseq=refseq)

        sub_domain = InterproDomain(
            accession='IPR012292',
            description='Globin_dom',
            occurrences=[
                Domain(start=3, end=142, protein=proteins['HBA2']),
                Domain(start=784, end=869, protein=proteins['ADGB'])
            ]
        )

        domain = InterproDomain(
            accession='IPR009050',
            description='Globin-like',
            children=[sub_domain],
            occurrences=[
                Domain(start=3, end=147, protein=proteins['HBG2']),
                Domain(start=1, end=142, protein=proteins['HBA2'])
            ]
        )

        db.session.add_all([domain, sub_domain, *proteins.values()])

        with NamedTemporaryFile('w+') as f:

            gmt_from_domains(path=f.name, include_sub_types=True)

            accepted_lines = [
                'IPR009050\tGlobin-like\t' + '\t'.join(genes) + '\n'
                for genes in permutations(['HBA2', 'HBG2', 'ADGB'])
            ] + [
                'IPR012292\tGlobin_dom\t' + '\t'.join(genes) + '\n'
                for genes in permutations(['HBA2', 'ADGB'])
            ]
            gmt_lines = f.readlines()

            assert all(line in accepted_lines for line in gmt_lines)
            assert len(gmt_lines) == 2
示例#13
0
    def list_redirects_json(self):

        redirects = []

        domains = Domain.query(Domain.redirect_enabled == True)
        for domain in domains.iter():
            host = urlparse(domain.redirect_url)
            rule = dict(hosts=[host.netloc])
            rule["rules"] = [
                {"from": "^" + re.escape(domain.redirect_url), "to": "http://" + domain.name + "/"}]

            if re.match('^www\.', host.netloc):
                # Add rule without www
                rule["hosts"].append(host.netloc[4:])
                nowww_redirect_url = re.sub('//www\.', '//', domain.redirect_url)
                rule["rules"].append({"from": "^" + re.escape(nowww_redirect_url), "to": "http://" + domain.name + "/"})
            else:
                # Add rule with www
                rule["hosts"].append("www." + host.netloc)
                www_redirect_url = re.sub('(https*://)', '\g<1>www.', domain.redirect_url)
                rule["rules"].append({"from": "^" + re.escape(www_redirect_url), "to": "http://" + domain.name + "/"})

            rule["exceptions"] = []
            redirects.append(rule)

        self.response.headers['Content-Type'] = 'application/json'
        self.response.write(json.encode(redirects))
示例#14
0
def addomain():
    form = FormDomain()

    if request.method == 'GET':
        return render_template('addomain.html', form=form)

    if request.method == 'POST':
        if form.validate_on_submit():
            form = FormDomain(request.form)
            mycompany = form.company.data
            mydomain = form.domain.data

            dominio = Domain.query.filter_by(name=mydomain).first()

            if dominio:
                flash('Domínio já existe', 'notification is-danger')
            else:
                query = Domain(company=mycompany, name=mydomain)
                db.session.add(query)

                flash("Domínio cadastrado", 'notification is-success')
                return redirect(url_for('addomain'))
        else:
            flash('Fala ao cadastrar o domínio', 'notification is-danger')

        return render_template('addomain.html', form=form)
示例#15
0
    def test_callback_new_domain(self):
        self.expect_indieauth_check()
        self.expect_site_fetch()
        self.mox.ReplayAll()

        resp = self.callback()
        self.assertEqual('http://localhost/#!Authorized you for snarfed.org.',
                         urllib.parse.unquote_plus(resp.headers['Location']))

        self.assert_entities_equal([
            Domain(id='snarfed.org',
                   tokens=['towkin'],
                   auth=self.auth_entity.key),
        ],
                                   Domain.query().fetch(),
                                   ignore=('created', 'updated'))
示例#16
0
    def auth(self):
        """Loads the source and token and checks that they're valid.

    Expects token in the `token` query param, source in `key` or `username`.

    Raises: :class:`HTTPException` with HTTP 400 if the token or source are
      missing or invalid

    Returns: BrowserSource or None
    """
        # Load source
        source = util.load_source(self, param='key')
        if not source:
            self.abort(
                404,
                f'No account found for {self.gr_source().NAME} user {key or username}'
            )

        # Load and check token
        token = util.get_required_param(self, 'token')
        for domain in Domain.query(Domain.tokens == token):
            if domain.key.id() in source.domains:
                return source

        self.abort(
            403,
            f'Token {token} is not authorized for any of: {source.domains}')
示例#17
0
 def create_domain(self, domain):
     domain_query_runner = self.db.query(Domain).filter(
         Domain.domain == domain)
     if domain_query_runner.count() == 0:
         domain_entity = Domain(domain=domain)
         self.db.add(domain_entity)
         self.db.commit()
示例#18
0
def update_clicks(domainname):
    """
    Updates the memcached total clickcount value
    """
    domain = Domain.query(Domain.name == str(domainname)).get()
    if domain:
        memcache.set(get_cache_key(domain), domain.clickcount)
示例#19
0
 def setUp(self):
     super().setUp()
     self.actor['numeric_id'] = '212038'
     self.source = Facebook.new(actor=self.actor)
     self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
     self.auth = f'token=towkin&key={self.source.key.urlsafe().decode()}'
     self.mox.StubOutWithMock(gr_facebook, 'now_fn')
示例#20
0
def update_total_clicks():
    """
    Updates the memcached total clickcount value
    """
    clicks = 0
    for domain in Domain.query():
        clicks = clicks + domain.clickcount
    memcache.set('clicks_total', clicks)
示例#21
0
    def post(self):
        token = util.get_required_param(self, 'token')

        domains = [d.key.id() for d in Domain.query(Domain.tokens == token)]
        if not domains:
            self.abort(404, f'No registered domains for token {token}')

        self.output(domains)
示例#22
0
    def _getDbDomains(self, cur):
        domains = []

        db_domains = cur.execute("SELECT * from dbd$domains").fetchall()
        domain_metadata = self._getMetadata(cur, "dbd$domains")

        for domain in db_domains:
            domain_dictionary = dict(zip(domain_metadata, list(domain)))
            tmp = Domain(domain_dictionary)
            data_type = cur.execute(
                "SELECT * FROM dbd$data_types WHERE id = :type_id", {
                    "type_id": tmp.data_type_id
                }).fetchall()
            tmp.type = data_type[0][1]
            domains.append(tmp)

        return domains
示例#23
0
文件: utils.py 项目: JerryDog/cdn
def saveDomainAndReturnId(domain_name,domain_cname,domain_type,
                          domain_status,disId, etag, project_id,
                          username, new_ip_list,test_url,ignore_param_req):
    domain = Domain(domain_name=domain_name,
                    domain_cname=domain_cname,
                    domain_type=domain_type,
                    domain_status=domain_status,
                    distribution_id=disId,
                    etag=etag,
                    project_id=project_id,
                    username=username,
                    ip_list=new_ip_list,
                    test_url=test_url,
                    ignore_param_req=ignore_param_req)
    domain.save()
    domain_id = Domain.objects.latest('id').id
    return domain_id
示例#24
0
def welcome():
	try:
		domains = list(Domain.select())
	except IndexError:
		domains = []
	about = settings.ABOUT
	default_domain = settings.MY_DOMAINS[0]
	return render_template('welcome.html', **locals())
示例#25
0
    def _save(self, results):

        for href in results:
            uri = urlparse(href)

            try:
                domain = Domain.objects.get(netloc=uri.netloc)
            except DoesNotExist as e:
                domain = Domain(scheme=uri.scheme, netloc=uri.netloc)

            path = uri.path
            if not path:
                path = '/'

            url = Url(path=path)
            domain.urls.append(url)
            domain.save()
示例#26
0
 def get_or_create_domain(self, domain: str) -> Tuple[Domain, bool]:
     # we need create redis cache
     try:
         return self.store.get_cache(domain), False
     except ValueError as exc:
         instance = Domain(domain)
         self.store.set_cache(domain, instance, instance.cache_time_out)
         return instance, True
示例#27
0
def welcome():
    try:
        domains = list(Domain.select())
    except IndexError:
        domains = []
    about = settings.ABOUT
    default_domain = settings.MY_DOMAINS[0]
    return render_template('welcome.html', **locals())
示例#28
0
    async def letsencrypt_fork(self, instance: Domain):
        domain: str = instance.domain

        command = [
            'letsencrypt',
            'certonly',
            f'--cert-name "{domain}"',
            '--manual',
            # f'--manual-auth-hook {self.dns_acme_auth}',
            f'--manual-auth-hook ./dns_acme_auth.py',
            # f'--deploy-hook {self.dns_acme_deploy}',
            f'--deploy-hook ./dns_acme_deploy.py',
            '--force-renewal',
            '--preferred-challenges=dns',
            '--register-unsafely-without-email',
            '--manual-public-ip-logging-ok',
            '--server https://acme-v02.api.letsencrypt.org/directory',
            '--agree-tos',
            '--quiet',
            f'-d "{domain}"',
            f'-d "*.{domain}"',
        ]

        shell_command = " ".join(command)

        process = await asyncio.create_subprocess_shell(
            shell_command,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE)
        # if success process.returncode = 0
        # if error process.returncode = 1
        stdout, stderr = await process.communicate()

        instance: Domain = self.store.get_cache(domain)

        instance.on_success = stdout.decode().replace("\n", " ").strip()
        instance.on_error = stderr.decode().replace("\n", " ").strip()

        logger.debug(
            f"process.communicate: {process.returncode} {instance.serialize()}"
        )
        if process.returncode == 0:
            instance.status = SUCCESS
        else:
            instance.status = FAILED
        self.store.set_cache(domain, instance, instance.cache_time_out)
示例#29
0
  def setUp(self):
    super().setUp()

    self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
    FakeBrowserSource.gr_source = FakeGrSource()
    self.actor['fbs_id'] = '222yyy'
    self.source = FakeBrowserSource.new(actor=self.actor).put()
    self.auth = f'token=towkin&key={self.source.urlsafe().decode()}'
    self.other_source = FakeBrowserSource(id='333zzz', domains=['foo.com']).put()

    for a in self.activities:
      a['object']['author'] = self.actor

    self.activities_no_replies = copy.deepcopy(self.activities)
    for a in self.activities_no_replies:
      del a['object']['replies']
      del a['object']['tags']
示例#30
0
    def domain_acquire(self, request):
        user = self.authenticate(request)

        validator = Validator(request)
        user_domain = validator.new_user_domain()
        device_mac_address = validator.device_mac_address()
        device_name = validator.string('device_name', required=True)
        device_title = validator.string('device_title', required=True)
        check_validator(validator)

        with self.create_storage() as storage:
            domain = storage.get_domain_by_name(user_domain)
            if domain and domain.user_id != user.id:
                raise servicesexceptions.parameter_error(
                    'user_domain', 'User domain name is already in use')

            update_token = util.create_token()
            if not domain:
                domain = Domain(user_domain,
                                device_mac_address,
                                device_name,
                                device_title,
                                update_token=update_token)
                domain.user = user
                storage.add(domain)
            else:
                domain.update_token = update_token
                domain.device_mac_address = device_mac_address
                domain.device_name = device_name
                domain.device_title = device_title

            return domain
示例#31
0
    def setUp(self):
        super(InstagramTest, self).setUp()
        self.handler.messages = []
        self.inst = Instagram.new(
            self.handler,
            actor={
                'objectType': 'person',
                'id': 'tag:instagram.com,2013:420973239',
                'username': '******',
                'displayName': 'Ryan Barrett',
                'url': 'https://snarfed.org/',
                'image': {
                    'url': 'http://pic.ture/url'
                },
                # ...
            })

        self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
示例#32
0
def get_domain_or_404(name, allow_none=False):
    if not name:
        webapp2.abort(404)

    domain = Domain.query(Domain.name == name).get()
    if not domain and not allow_none:
        webapp2.abort(404)

    return domain
示例#33
0
    def list_redirects_csv(self):

        self.response.headers['Content-Type'] = 'text/csv'

        writer = csv.writer(self.response, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)

        domains = Domain.query(Domain.redirect_enabled == True)
        for domain in domains.iter():
            writer.writerow([domain.name, domain.redirect_url])
示例#34
0
    def test_get(self):
        with db_session:
            a = Author.get(author_id='ded2d8cd-73c6-41c0-bcf7-db2983805231')
            logger.debug(a.organization)
            self.assertEqual(a.name, 'Abraham Silberschatz')

            d = Domain.get(domain_id='1.1')
            logger.debug(d.name)
            self.assertEqual(d.name, '算法与理论')
示例#35
0
def get_or_create_domain(domain_string):
    """
    check in the instance of the specific domain is saved . 
    If yes return an instance of domain, if not save an instance and return it 
    
    Params
        domain_string - the domain name (ex. facebook.com): string
    
    Return
        an instance of domain object : Domain 
    
    """
    domain = db.Query(Domain).filter('name = ', domain_string).get()
    if not domain:
        domain = Domain()
        domain.name = domain_string
        db.put(domain)

    return domain 
示例#36
0
 def add_or_update_domain():
     domain = Domain.get_or_insert(
         util.domain_from_link(
             util.replace_test_domains_with_localhost(
                 auth_entity.key.id())))
     domain.auth = auth_entity.key
     if state not in domain.tokens:
         domain.tokens.append(state)
     domain.put()
     flash(f'Authorized you for {domain.key.id()}.')
示例#37
0
    def test_creating_new_domains(self):
        domain = Domain()
        domain.id = 42
        domain.name = "Test VM"
        domain.state = domain.RUNNING
        domain.max_memory = 2*2024*2024
        domain.memory = domain.max_memory
        domain.vcpu = 1
        domain.save()

        all_domains_in_database = Domain.objects.all()
        self.assertEquals(len(all_domains_in_database), 1)
        only_domain_in_database = all_domains_in_database[0]
        self.assertEquals(only_domain_in_database, domain)

        self.assertEquals(only_domain_in_database.name, "Test VM")
示例#38
0
    def create_new_user(self, request):
        validator = Validator(request)
        email = validator.email()
        password = validator.password()
        user_domain = validator.new_user_domain(error_if_missing=False)
        errors = validator.errors

        if errors:
            message = ", ".join(errors)
            raise servicesexceptions.bad_request(message)

        user = None
        action = None
        with self.create_storage() as storage:
            by_email = storage.get_user_by_email(email)
            if by_email and by_email.email == email:
                raise servicesexceptions.conflict('Email is already registered')

            if user_domain:
                by_domain = storage.get_domain_by_name(user_domain)
                if by_domain and by_domain.user_domain == user_domain:
                    raise servicesexceptions.conflict('User domain name is already in use')

            update_token = util.create_token()

            user = User(email, util.hash(password), not self.activate_by_email)

            if user_domain:
                domain = Domain(user_domain, None, update_token)
                domain.user = user
                user.domains.append(domain)
                storage.add(domain)

            if self.activate_by_email:
                action = user.enable_action(ActionType.ACTIVATE)

            storage.add(user)

        if self.activate_by_email:
            self.mail.send_activate(self.main_domain, user.email, action.token)

        return user
示例#39
0
    def domain_acquire(self, request):
        user = self.authenticate(request)

        validator = Validator(request)
        user_domain = validator.new_user_domain()

        with self.create_storage() as storage:
            domain = storage.get_domain_by_name(user_domain)
            if domain and domain.user_id != user.id:
                raise servicesexceptions.conflict('User domain name is already in use')

            update_token = util.create_token()
            if not domain:
                domain = Domain(user_domain, None, update_token)
                domain.user = user
                storage.add(domain)
            else:
                domain.update_token = update_token

            return domain
示例#40
0
    def dispatch_request(self):
        token = request.values['token']

        domains = [d.key.id() for d in Domain.query(Domain.tokens == token)]
        if not domains:
            indieauth_start = util.host_url(f'/indieauth/start?token={token}')
            self.error(
                f'Not connected to Bridgy. <a href="{indieauth_start}" target="_blank"">Connect now!</a>',
                404)

        return jsonify(domains)
示例#41
0
    def domain_acquire(self, request):
        user = self.authenticate(request)

        validator = Validator(request)
        user_domain = validator.new_user_domain()
        device_mac_address = validator.device_mac_address()
        device_name = validator.string('device_name', required=True)
        device_title = validator.string('device_title', required=True)
        check_validator(validator)

        with self.create_storage() as storage:
            domain = storage.get_domain_by_name(user_domain)
            if domain and domain.user_id != user.id:
                raise servicesexceptions.parameter_error('user_domain', 'User domain name is already in use')

            update_token = util.create_token()
            if not domain:
                domain = Domain(user_domain, device_mac_address, device_name, device_title, update_token=update_token)
                domain.user = user
                storage.add(domain)
            else:
                domain.update_token = update_token
                domain.device_mac_address = device_mac_address
                domain.device_name = device_name
                domain.device_title = device_title

            return domain
示例#42
0
    def _getFields(self, xml):
        if xml.nodeName != "table":
            raise ValueError("Is not a table")

        fields = []
        for field in xml.getElementsByTagName("field"):
            tmp = Field()
            domain = Domain()
            for attributeName, attributeValue in field.attributes.items():
                if attributeName.lower() == "name":
                    tmp.name = attributeValue
                elif attributeName.lower() == "rname":
                    tmp.rname = attributeValue
                elif attributeName == "domain":
                    tmp.domain = attributeValue
                elif attributeName.lower() == "domain.name":
                    domain.name = attributeValue
                elif attributeName.lower() == "domain.char_length":
                    domain.char_length = attributeValue
                elif attributeName.lower() == "domain.precision":
                    domain.precision = attributeValue
                elif attributeName.lower() == "domain.scale":
                    domain.scale = attributeValue
                elif attributeName.lower() == "domain.type":
                    domain.type = attributeValue
                elif attributeName.lower() == "description":
                    tmp.description = attributeValue
                elif attributeName.lower() == "props":
                    for prop in attributeValue.split(", "):
                        if prop == "input":
                            tmp.input = True
                        elif prop == "edit":
                            tmp.edit = True
                        elif prop == "show_in_grid":
                            tmp.show_in_grid = True
                        elif prop == "show_in_details":
                            tmp.show_in_details = True
                        elif prop == "is_mean":
                            tmp.is_mean = True
                        elif prop == "autocalculated":
                            tmp.autocalculated = True
                        elif prop == "required":
                            tmp.required = True
                        else:
                            raise ValueError(
                                "Invalid format of props string: {}".format(
                                    attributeValue))
                else:
                    raise ValueError(
                        "Incorrect attribute name \"{}\" in tag \"{}\" ".
                        format(attributeName, field.nodeName))
            if tmp.domain is None:
                tmp.domain = domain
            fields.append(tmp)
        return fields
示例#43
0
  def finish(self, auth_entity, state=None):
    if not auth_entity:
      return

    assert state
    domain = Domain.get_or_insert(util.domain_from_link(auth_entity.key.id()))
    domain.auth = auth_entity.key
    if state not in domain.tokens:
      domain.tokens.append(state)
    domain.put()

    self.messages.add(f'Authorized you for {domain.key.id()}.')
    self.redirect('/')
示例#44
0
def get_domain_or_404(name, allow_none=False):
  """Gets a domain entity for the given name.

  Aborts with 404 if entity doesn't exist and if that is not allowed.
  """
  if not name:
    webapp2.abort(404)

  domain = Domain.query(Domain.name==name).get()
  if not domain and not allow_none:
    webapp2.abort(404)

  return domain
示例#45
0
 def get(self):
     if self.request.headers.get('X-AppEngine-Cron') != "true":
         self.error(403)
         return
     total = 0
     for domain in Domain.query():
         mem_count = memcache.get(get_cache_key(domain))
         if mem_count:
             total = total + mem_count
         else:
             total = total + domain.clickcount
     ct = models.ClickcountDate()
     ct.clickcount = total
     ct.put()
示例#46
0
def main():
    unvisited_links = session.query(Link).\
                        filter(Link.visited_at == None).all()

    if len(unvisited_links) == 0:
        print("Nothing to visit right now.")

    for link in unvisited_links:
        try:
            r = requests.get(link)
            soup = BeautifulSoup(r.text, 'html.parser')

            for site_url in set([o.get('href') for o in soup.find_all('a')]):
                if site_url is None:
                    continue

                url = site_url

                if not is_url(site_url):
                    url = urljoin(link.get_domain(), site_url)

                print('Found: {}'.format(url))

                l = session.query(Link).\
                        filter(Link.url == url).first()

                if l is not None:
                    continue

                l = Link(url=url)
                domain = l.get_domain()

                domain_in_db = session.query(Domain).\
                                        filter(Domain.url == domain).\
                                        first()

                if domain_in_db in None:
                    print("Found new domain: {}".format(domain))
                    domain_in_db = Domain(url=domain)
                    save(domain_in_db)

                l.domain = domain_in_db
                save(l)

        except:
            print('Something went wrong.')
        finally:
            link.visited_at = datetime.now()
            save(link)
示例#47
0
文件: main.py 项目: geudrik/darkstar
def add_new_domain(domain):
    """ Add a new domain to be perpetually resolved
    """

    # Check to see if this domain is already being tracked
    exists = Domain.search().filter('term', domain=domain).execute()
    if exists:
        raise DomainExists(
            f"Supplied domain {domain} is already being tracked by Darkstar")

    # This is a new domain, lets try to resolve it for the first time
    try:
        hostname, aliastlist, ipaddrlist = gethostbyname_ex(domain)
    except gaierror as e:
        current_app.logger.exception(e)
        raise e
示例#48
0
    def process_item(self, item, spider):
        if 'URLPipeline' not in getattr(spider, "pipelines"):
            return item

        if self.is_subdomain(item['url']):
            print 'subdomain: ', item['url']
            return item
        try:
            dmain = Domain(url_domain=item['url'],
                        start_url=item['source'],
                        status_code=item['status'],
                        domain_ext=item['domain_ext'])
            db.session.add(dmain)
            db.session.commit()
        except IntegrityError:
            db.session.rollback()
        return item
示例#49
0
 def post(self):
     reqs = request.get_json(force=True)
     if not reqs:
         raise JsonRequiredError()
     try:
         # check if domain pair existed
         u = Domain.query.filter_by(url=reqs['url'],
                                    port=reqs['port']).first()
         if not (u is None):
             raise UserExistedError()
         reqs['user_id'] = current_identity.id
         current_identity.num_domains += 1
         reqs['id'] = current_identity.num_domains
         u = Domain(**reqs)
         db.session.add(u)
         db.session.commit()
         return {}, 201, {'Location': '/domains/%d' % u.relative_id}
     except KeyError:
         raise JsonInvalidError()
示例#50
0
 def get(self):
     if self.request.headers.get('X-AppEngine-Cron') != "true":
         self.error(403)
         return
     for domain in Domain.query():
         # TODO: Maybe maintain a list of domains which do have a memcache counter?
         k = get_cache_key(domain)
         new_count = memcache.get(k)
         if not new_count:
             logging.info("No clickcount in memcache for: %s", domain.name)
         elif new_count < domain.clickcount:
             logging.error("New clickcount for %s would be %d, is %d", domain.name, new_count, domain.clickcount)
             memcache.delete(k)
         elif new_count == domain.clickcount:
             logging.info("clickcount for %s unchanged: %d", domain.name, new_count)
         else:
             old_count = domain.clickcount
             domain.clickcount = new_count
             domain.put()
             logging.info("clickcount for %s: %d -> %d", domain.name, old_count, new_count)
示例#51
0
    def post(self, domain_name):
        domain = get_domain_or_404(domain_name, allow_none=True)
        if not domain:
            domain = Domain(name=domain_name, clickcount=0)
        data = self.request.body.strip()
        config = json.decode(data) if len(data) > 0 else {}
        config['__locales'] = []
        if 'strings' in config:
            for locale in config['strings']:
                config['__locales'].append(locale)
        domain.content = json.encode(config)

        # Redirects
        domain.redirect_enabled = False
        domain.redirect_url = None
        if 'redirect_url' in config:
            domain.redirect_enabled = True
            domain.redirect_url = config['redirect_url']

        domain.put()
        self.response.headers['Content-Type'] = 'text/plain'
        self.response.set_status(204)
示例#52
0
文件: schd.py 项目: yidun55/crawler
 def get_domains(self):
     """return the all domains as a generator, from start to finish, inclusive.
     """
     return Domain.filter(self.rd)
示例#53
0
    def post(self):
        params = self.request.params
        if not 'domain' in params:
            self.abort(404)

        cache_key_domain = 'domain:d:' + params['domain']

        domain = memcache.get(cache_key_domain)
        if domain == 0:
            self.abort(404)
            return
        if not domain:
            domain = Domain.query(Domain.name == params['domain']).get()
            if not domain:
                # store for 60 seconds
                memcache.set(cache_key_domain, 0, 60)
                self.abort(404)
                return
            else:
                memcache.set(cache_key_domain, domain, 60)

        # count clicks from outside the iframe that have a previous (pt) and current (ct) visit timestamp
        # previous timestamp may be empty if the client has no record of this user having visited the domain before)
        # current timestamp may not be empty
        # for simplicity timestamps are expressed as integers (unix timestamps) in the users timezone
        try:
            if 'from' in params and 'pt' in params and 'ct' in params:
                if params['from'] == 'outside':
                    current_visit = int(params['ct']) if params['ct'] != "" else 0
                    if current_visit < 1397340000000:
                        raise Error("Invalid value for 'ct': '%s'" % params['ct'])

                    previous_visit = int(params['pt']) if params['pt'] != "" else current_visit - COUNT_THRESHOLD
                    if previous_visit < 1397340000000:
                        raise Error("Invalid value for 'pt': '%s'" % params['pt'])

                    if current_visit < previous_visit:
                        raise Error("Value '%s' for 'ct' must be greater or equal than value '%s' for 'pt'" % (params['ct'], params['pt']))

                    if current_visit - previous_visit >= COUNT_THRESHOLD:
                        if memcache.incr(jobs.get_cache_key(domain)) is None:
                            jobs.init_clicks(domain)
                        if memcache.incr("clicks_total") is None:
                            jobs.init_clicks_total()
        except Error as e:
            self.response.headers['Content-Type'] = 'application/api-problem+json'
            self.response.set_status(403)
            self.response.write(
                json.encode(
                    {
                        "problemType": "https://github.com/dothiv/clickcounter-backend/wiki/problem-invalid-request",
                        "title": "%s" % e.message
                    }
                )
            )
            return

        # explicit request to have content-type application/json
        self.response.headers['Content-Type'] = 'application/json'
        # allow origin
        if 'Origin' in self.request.headers:
            self.response.headers['Access-Control-Allow-Origin'] = self.request.headers['Origin']
        config = createDomainConfig(domain, getClientLocales(self.request.headers.get('accept-language')))
        self.response.write(json.encode(config))
示例#54
0
 def delete(self, domain_name):
     domain = Domain.query(Domain.name == domain_name).get()
     if domain:
         domain.key.delete()
     self.response.headers['Content-Type'] = 'text/plain'
     self.response.set_status(204)
示例#55
0
文件: sync.py 项目: yidun55/crawler
#!/usr/bin/env python
# coding:utf-8
import redis
from torndb import Connection

from models import Domain, Flow, Rule
import settings

rd = redis.Redis(settings.REDIS_HOST, settings.REDIS_PORT)
db = Connection("127.0.0.1", "spider", user="******", password="******")

domains = db.query("select * from domain")

for d in domains:
    domain = Domain(rd, d["domain"])
    domain.update(d)
    if "scheduled" not in domain:
        domain["scheduled"] = 0

    rule_datas = db.query("select * from rule where domain = %s", d["domain"])
    for rule_data in rule_datas:
        rule = Rule(rd, d["domain"])
        rule.update(rule_data)

    flows = db.query("select * from flow where domain = %s", d["domain"])
    for f in flows:
        flow = Flow(rd, f["flow"])
        flow.update(f)
        if "scheduled" not in flow:
            flow["scheduled"] = 0
示例#56
0
文件: app.py 项目: yidun55/crawler
 def get(self):
     self.render('schedule.html', domains=Domain.filter(self.rdb))
示例#57
0
  def test_job_should_emit_status_for_crawled_urls(self):
    first_domain = Domain("http://example.com")
    first_domain.crawled = True
    job = Job([first_domain])

    self.assertEqual(job.get_status()["completed"], 1)