def test_domains(self):
protein_data = [
('NM_000184', 'HBG2'),
('NM_000517', 'HBA2'),
('NM_024694', 'ADGB')
]
proteins: Dict[str, Protein] = {} # gene_name -> protein
for refseq, gene_name in protein_data:
g = Gene(name=gene_name)
proteins[gene_name] = Protein(gene=g, refseq=refseq)
sub_domain = InterproDomain(
accession='IPR012292',
description='Globin_dom',
occurrences=[
Domain(start=3, end=142, protein=proteins['HBA2']),
Domain(start=784, end=869, protein=proteins['ADGB'])
]
)
domain = InterproDomain(
accession='IPR009050',
description='Globin-like',
children=[sub_domain],
occurrences=[
Domain(start=3, end=147, protein=proteins['HBG2']),
Domain(start=1, end=142, protein=proteins['HBA2'])
]
)
db.session.add_all([domain, sub_domain, *proteins.values()])
with NamedTemporaryFile('w+') as f:
gmt_from_domains(path=f.name, include_sub_types=True)
accepted_lines = [
'IPR009050\tGlobin-like\t' + '\t'.join(genes) + '\n'
for genes in permutations(['HBA2', 'HBG2', 'ADGB'])
] + [
'IPR012292\tGlobin_dom\t' + '\t'.join(genes) + '\n'
for genes in permutations(['HBA2', 'ADGB'])
]
gmt_lines = f.readlines()
assert all(line in accepted_lines for line in gmt_lines)
assert len(gmt_lines) == 2
def setUp(self):
super().setUp()
self.actor['numeric_id'] = '212038'
self.source = Facebook.new(actor=self.actor)
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
self.auth = f'token=towkin&key={self.source.key.urlsafe().decode()}'
self.mox.StubOutWithMock(gr_facebook, 'now_fn')
def addomain():
form = FormDomain()
if request.method == 'GET':
return render_template('addomain.html', form=form)
if request.method == 'POST':
if form.validate_on_submit():
form = FormDomain(request.form)
mycompany = form.company.data
mydomain = form.domain.data
dominio = Domain.query.filter_by(name=mydomain).first()
if dominio:
flash('Domínio já existe', 'notification is-danger')
else:
query = Domain(company=mycompany, name=mydomain)
db.session.add(query)
flash("Domínio cadastrado", 'notification is-success')
return redirect(url_for('addomain'))
else:
flash('Fala ao cadastrar o domínio', 'notification is-danger')
return render_template('addomain.html', form=form)
def domain_acquire(self, request):
user = self.authenticate(request)
validator = Validator(request)
user_domain = validator.new_user_domain()
device_mac_address = validator.device_mac_address()
device_name = validator.string('device_name', required=True)
device_title = validator.string('device_title', required=True)
check_validator(validator)
with self.create_storage() as storage:
domain = storage.get_domain_by_name(user_domain)
if domain and domain.user_id != user.id:
raise servicesexceptions.parameter_error(
'user_domain', 'User domain name is already in use')
update_token = util.create_token()
if not domain:
domain = Domain(user_domain,
device_mac_address,
device_name,
device_title,
update_token=update_token)
domain.user = user
storage.add(domain)
else:
domain.update_token = update_token
domain.device_mac_address = device_mac_address
domain.device_name = device_name
domain.device_title = device_title
return domain
def create_domain(self, domain):
domain_query_runner = self.db.query(Domain).filter(
Domain.domain == domain)
if domain_query_runner.count() == 0:
domain_entity = Domain(domain=domain)
self.db.add(domain_entity)
self.db.commit()
def upsert_author(info):
with db_session:
a = Author.get(author_id=info['author_id'])
if a:
logger.debug('author has existed, author_id=%s', info['author_id'])
return
# logger.debug(info)
author = Author(author_id=info['author_id'])
author.name = info['name'] if info['name'] else ''
author.image_url = info['image_url'] if info['image_url'] else ''
author.organization = info['organization'] if info[
'organization'] else ''
author.home_page = info['home_page'] if info['home_page'] else ''
author.paper_count = info['paper_count']
author.citied_count = info['cited_count']
for d in info['domains']:
if d['id'] == '':
continue
ds = Domain.get(domain_id=d['id'])
if ds:
author.domains.add(ds)
else:
domain = Domain(domain_id=d['id'])
domain.name = d['name']
author.domains.add(domain)
def add_domain(permission):
body = request.get_json()
try:
domain = Domain(body['domain_name'])
domain.insert()
return jsonify({'success': True, 'domain_id': domain.id})
except:
abort(422)
def get_or_create_domain(self, domain: str) -> Tuple[Domain, bool]:
# we need create redis cache
try:
return self.store.get_cache(domain), False
except ValueError as exc:
instance = Domain(domain)
self.store.set_cache(domain, instance, instance.cache_time_out)
return instance, True
def run(self):
# Fixtures
if Domain.objects.count() == 0:
domain = Domain(scheme='http', netloc='sametmax.com')
url = Url(path='/')
domain.urls.append(url)
domain.save()
for domain in Domain.objects:
self._find_links(domain)
def _getFields(self, xml):
if xml.nodeName != "table":
raise ValueError("Is not a table")
fields = []
for field in xml.getElementsByTagName("field"):
tmp = Field()
domain = Domain()
for attributeName, attributeValue in field.attributes.items():
if attributeName.lower() == "name":
tmp.name = attributeValue
elif attributeName.lower() == "rname":
tmp.rname = attributeValue
elif attributeName == "domain":
tmp.domain = attributeValue
elif attributeName.lower() == "domain.name":
domain.name = attributeValue
elif attributeName.lower() == "domain.char_length":
domain.char_length = attributeValue
elif attributeName.lower() == "domain.precision":
domain.precision = attributeValue
elif attributeName.lower() == "domain.scale":
domain.scale = attributeValue
elif attributeName.lower() == "domain.type":
domain.type = attributeValue
elif attributeName.lower() == "description":
tmp.description = attributeValue
elif attributeName.lower() == "props":
for prop in attributeValue.split(", "):
if prop == "input":
tmp.input = True
elif prop == "edit":
tmp.edit = True
elif prop == "show_in_grid":
tmp.show_in_grid = True
elif prop == "show_in_details":
tmp.show_in_details = True
elif prop == "is_mean":
tmp.is_mean = True
elif prop == "autocalculated":
tmp.autocalculated = True
elif prop == "required":
tmp.required = True
else:
raise ValueError(
"Invalid format of props string: {}".format(
attributeValue))
else:
raise ValueError(
"Incorrect attribute name \"{}\" in tag \"{}\" ".
format(attributeName, field.nodeName))
if tmp.domain is None:
tmp.domain = domain
fields.append(tmp)
return fields
def post(self, domain_name):
domain = get_domain_or_404(domain_name, allow_none=True)
if not domain:
domain = Domain(name=domain_name, clickcount=0, money=0., status=0.)
# example content:
# "firstvisit":"center","secondvisit":"center","heading":"Vielen Dank!","subheading":"Dein Klick auf domain.hiv hat soeben einen Gegenwert von 1 ct ausgelöst.","claim":"Wir sind Teil der Bewegung","about":"Über dotHIV","vote":"Vote","activated":"Bisher aktiviert:","currency":"€","corresponding":"entspricht","clicks":"Klicks"
domain.content = self.request.body
domain.put()
self.response.headers['Content-Type'] = 'text/plain'
self.response.set_status(204)
def index(request):
form = DomainForm()
if request.method == "POST":
form = DomainForm(data=request.POST)
if form.is_valid():
domain = form.cleaned_data["domain"]
d = Domain(user=request.user, domain=domain)
d.save()
scan.delay(d.id)
messages.add_message(
request, messages.INFO, 'Domain başarılı bir şekilde alındı. '
'Durum takibini Taramalar sayfasından yapabilirsiniz')
return render(request, "index.html", locals())
def main():
unvisited_links = session.query(Link).\
filter(Link.visited_at == None).all()
if len(unvisited_links) == 0:
print("Nothing to visit right now.")
for link in unvisited_links:
try:
r = requests.get(link)
soup = BeautifulSoup(r.text, 'html.parser')
for site_url in set([o.get('href') for o in soup.find_all('a')]):
if site_url is None:
continue
url = site_url
if not is_url(site_url):
url = urljoin(link.get_domain(), site_url)
print('Found: {}'.format(url))
l = session.query(Link).\
filter(Link.url == url).first()
if l is not None:
continue
l = Link(url=url)
domain = l.get_domain()
domain_in_db = session.query(Domain).\
filter(Domain.url == domain).\
first()
if domain_in_db in None:
print("Found new domain: {}".format(domain))
domain_in_db = Domain(url=domain)
save(domain_in_db)
l.domain = domain_in_db
save(l)
except:
print('Something went wrong.')
finally:
link.visited_at = datetime.now()
save(link)
def _getDomains(self):
domains = []
for domain in self.xml.getElementsByTagName("domain"):
tmp = Domain()
for attributeName, attributeValue in domain.attributes.items():
if attributeName.lower() == "name":
tmp.name = attributeValue
elif attributeName.lower() == "description":
tmp.description = attributeValue
elif attributeName.lower() == "type":
tmp.type = attributeValue
tmp.type_id = postgres_utils.get_type_in_postgres(tmp)
elif attributeName.lower() == "align":
tmp.align = attributeValue
elif attributeName.lower() == "width":
tmp.width = attributeValue
elif attributeName.lower() == "precision":
tmp.precision = attributeValue
elif attributeName.lower() == "props":
for prop in attributeValue.split(", "):
if prop == "show_null":
tmp.show_null = True
elif prop == "summable":
tmp.summable = True
elif prop == "case_sensitive":
tmp.case_sensitive = True
elif prop == "show_lead_nulls":
tmp.show_lead_nulls = True
elif prop == "thousands_separator":
tmp.thousands_separator = True
else:
raise ValueError(
"Invalid format of props string: {}".format(
attributeValue))
elif attributeName.lower() == "char_length":
tmp.char_length = attributeValue
elif attributeName.lower() == "length":
tmp.length = attributeValue
elif attributeName.lower() == "scale":
tmp.scale = attributeValue
else:
raise ValueError(
"Incorrect attribute name \"{}\" in tag \"{}\" ".
format(attributeName, domain.nodeName))
domains.append(tmp)
return domains
def test_callback_new_domain(self):
self.expect_indieauth_check()
self.expect_site_fetch()
self.mox.ReplayAll()
resp = self.callback()
self.assertEqual('http://localhost/#!Authorized you for snarfed.org.',
urllib.parse.unquote_plus(resp.headers['Location']))
self.assert_entities_equal([
Domain(id='snarfed.org',
tokens=['towkin'],
auth=self.auth_entity.key),
],
Domain.query().fetch(),
ignore=('created', 'updated'))
def _getDbDomains(self, cur):
domains = []
db_domains = cur.execute("SELECT * from dbd$domains").fetchall()
domain_metadata = self._getMetadata(cur, "dbd$domains")
for domain in db_domains:
domain_dictionary = dict(zip(domain_metadata, list(domain)))
tmp = Domain(domain_dictionary)
data_type = cur.execute(
"SELECT * FROM dbd$data_types WHERE id = :type_id", {
"type_id": tmp.data_type_id
}).fetchall()
tmp.type = data_type[0][1]
domains.append(tmp)
return domains
def setUp(self):
super().setUp()
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
FakeBrowserSource.gr_source = FakeGrSource()
self.actor['fbs_id'] = '222yyy'
self.source = FakeBrowserSource.new(actor=self.actor).put()
self.auth = f'token=towkin&key={self.source.urlsafe().decode()}'
self.other_source = FakeBrowserSource(id='333zzz', domains=['foo.com']).put()
for a in self.activities:
a['object']['author'] = self.actor
self.activities_no_replies = copy.deepcopy(self.activities)
for a in self.activities_no_replies:
del a['object']['replies']
del a['object']['tags']
def process_item(self, item, spider):
if 'URLPipeline' not in getattr(spider, "pipelines"):
return item
if self.is_subdomain(item['url']):
print 'subdomain: ', item['url']
return item
try:
dmain = Domain(url_domain=item['url'],
start_url=item['source'],
status_code=item['status'],
domain_ext=item['domain_ext'])
db.session.add(dmain)
db.session.commit()
except IntegrityError:
db.session.rollback()
return item
def _save(self, results):
for href in results:
uri = urlparse(href)
try:
domain = Domain.objects.get(netloc=uri.netloc)
except DoesNotExist as e:
domain = Domain(scheme=uri.scheme, netloc=uri.netloc)
path = uri.path
if not path:
path = '/'
url = Url(path=path)
domain.urls.append(url)
domain.save()
def inspect(base_domain):
domain = Domain(base_domain)
domain.http = Endpoint("http", "root", base_domain)
domain.httpwww = Endpoint("http", "www", base_domain)
domain.https = Endpoint("https", "root", base_domain)
domain.httpswww = Endpoint("https", "www", base_domain)
# Analyze HTTP endpoint responsiveness and behavior.
basic_check(domain.http)
basic_check(domain.httpwww)
basic_check(domain.https)
basic_check(domain.httpswww)
# Analyze HSTS header, if present, on each HTTPS endpoint.
hsts_check(domain.https)
hsts_check(domain.httpswww)
return result_for(domain)
def setUp(self):
super(InstagramTest, self).setUp()
self.handler.messages = []
self.inst = Instagram.new(
self.handler,
actor={
'objectType': 'person',
'id': 'tag:instagram.com,2013:420973239',
'username': '******',
'displayName': 'Ryan Barrett',
'url': 'https://snarfed.org/',
'image': {
'url': 'http://pic.ture/url'
},
# ...
})
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
def post(self):
reqs = request.get_json(force=True)
if not reqs:
raise JsonRequiredError()
try:
# check if domain pair existed
u = Domain.query.filter_by(url=reqs['url'],
port=reqs['port']).first()
if not (u is None):
raise UserExistedError()
reqs['user_id'] = current_identity.id
current_identity.num_domains += 1
reqs['id'] = current_identity.num_domains
u = Domain(**reqs)
db.session.add(u)
db.session.commit()
return {}, 201, {'Location': '/domains/%d' % u.relative_id}
except KeyError:
raise JsonInvalidError()
import sys
import getopt
from sqlalchemy.orm import sessionmaker
from models import Title, db_connect, create_table, Domain, Subdomain
from sqlalchemy.orm.exc import NoResultFound
engine = db_connect()
create_table(engine)
Session = sessionmaker(bind=engine)
session = Session()
domains = ['google.com', 'yahoo.com', 'bing.com']
subdomains = [('images.google.com', 1), ('maps.google.com', 1), ('mail.google.com', 1),
('view.yahoo.com', 2), ('tw.news.yahoo.com', 2), ('stores.yahoo.com', 2)]
for domain in domains:
d = Domain(domain_name=domain)
session.add(d)
session.commit()
for subdomain in subdomains:
sd = Subdomain(subdomain_name=subdomain[0], domain_id=subdomain[1])
session.add(sd)
session.commit()
def parser(line):
nonlocal skipped, wrong_length, not_matching_chrom
try:
protein = proteins[line[6]] # by refseq
except KeyError:
skipped += 1
return
# If there is no data about the domains, skip this record
if len(line) == 7:
return
try:
assert len(line) == 12
except AssertionError:
print(line, len(line))
# does start is lower than end?
assert int(line[11]) < int(line[10])
accession = line[7]
# according to:
# http://www.ncbi.nlm.nih.gov/pmc/articles/PMC29841/#__sec2title
assert accession.startswith('IPR')
start, end = int(line[11]), int(line[10])
# TODO: the assertion fails for some domains: what to do?
# assert end <= protein.length
if end > protein.length:
wrong_length += 1
if line[3] != protein.gene.chrom:
skipped += 1
not_matching_chrom.append(line)
return
if accession not in interpro_domains:
interpro = InterproDomain(
accession=line[7], # Interpro Accession
short_description=line[8], # Interpro Short Description
description=line[9], # Interpro Description
)
interpro_domains[accession] = interpro
interpro = interpro_domains[accession]
similar_domains = [
# select similar domain occurrences with criteria being:
domain for domain in protein.domains
# - the same interpro id
if domain.interpro == interpro and
# - at least 75% of common coverage for shorter occurrence of domain
((min(domain.end, end) - max(domain.start, start)) /
min(len(domain), end - start) > 0.75)
]
if similar_domains:
try:
assert len(similar_domains) == 1
except AssertionError:
print(similar_domains)
domain = similar_domains[0]
domain.start = min(domain.start, start)
domain.end = max(domain.end, end)
else:
domain = Domain(interpro=interpro,
protein=protein,
start=start,
end=end)
new_domains.append(domain)
try:
opts, args = getopt.getopt(argv, "a:r:b:l:R:g:h", ["add=", "remove=", "list=", "bulk=", "REM=", "get=", "help"])
except:
print(usage)
sys.exit()
engine = db_connect()
create_table(engine)
Session = sessionmaker(bind=engine)
session = Session()
for opt, arg in opts:
if opt in ['-a', '--add']:
if not arg.startswith('http'):
arg = 'http://' + arg
query = Domain(name=arg)
session.add(query)
session.commit()
print('%s added' % arg)
if opt in ['-g', '--get']:
with open(arg, 'w') as f:
writer = csv.writer(f)
writer.writerow(('domain_name', 'subdomain_name'))
query = session.query(Domain.domain_name, Subdomain.subdomain_name)
query = query.join(Subdomain)
for q in query:
writer.writerow(q)
query = session.query(Domain.domain_name)
for q in query:
def save_content(url, title, content, evn):
domain, uri = extract_url(url)
d = Domain(domain, evn)
d.update_content(uri, title, content)
def setUp(self):
super(InstagramTest, self).setUp()
self.source = Instagram.new(self.handler, actor=self.actor)
self.domain = Domain(id='snarfed.org', tokens=['towkin']).put()
self.auth = f'token=towkin&key={self.source.key.urlsafe().decode()}'
def test_domain_python_to_json_float(self):
to_json = Domain(name='').python_to_json_float
self.assertEqual(to_json(0), '0.0')
self.assertEqual(to_json(0.), '0.0')
self.assertEqual(to_json(4.50000), '4.5')
self.assertEqual(to_json(1e-07), '0.0000001')
