예제 #1
0
파일: handlers.py 프로젝트: 0x3a/crits
 def __init__(self):
     self.rules = {}
     etlds = TLD.objects()
     for etld in etlds:
         tld = etld.tld.split('.')[-1]
         self.rules.setdefault(tld, [])
         self.rules[tld].append(re.compile(self.regexpize(etld.tld)))
예제 #2
0
 def __init__(self):
     self.rules = {}
     etlds = TLD.objects()
     for etld in etlds:
         tld = etld.tld.split('.')[-1]
         self.rules.setdefault(tld, [])
         self.rules[tld].append(re.compile(self.regexpize(etld.tld)))
예제 #3
0
def populate_tlds(drop):
    """
    Populate default set of TLDs into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    if not drop:
        print "Drop protection does not apply to effective TLDs"
    TLD.drop_collection()
    f = os.path.join(settings.SITE_ROOT, '..', 'extras', 'effective_tld_names.dat')
    count = 0
    for line in open(f, 'r').readlines():
        line = line.strip()
        if line and not line.startswith('//'):
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
            count += 1
    print "Effective TLDs: added %s TLDs!" % count
def populate_tlds(drop):
    """
    Populate default set of TLDs into the system.

    :param drop: Drop the existing collection before trying to populate.
    :type: boolean
    """

    if not drop:
        print "Drop protection does not apply to effective TLDs"
    TLD.drop_collection()
    f = os.path.join(settings.SITE_ROOT, '..', 'extras', 'effective_tld_names.dat')
    count = 0
    for line in open(f, 'r').readlines():
        line = line.strip()
        if line and not line.startswith('//'):
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
            count += 1
    print "Effective TLDs: added %s TLDs!" % count
예제 #5
0
def update_tlds(data=None):
    """
    Update the TLD list in the database.

    :param data: The TLD data.
    :type data: file handle.
    :returns: dict with key "success" (boolean)
    """

    if not data:
        return {'success': False}
    line = data.readline()
    while line:
        line = line.rstrip()
        if line and not line.startswith('//'):
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
        line = data.readline()

    # Update the package local tld_parser with the new domain info
    tld_parser = etld()

    return {'success': True}
예제 #6
0
파일: handlers.py 프로젝트: 0x3a/crits
def update_tlds(data=None):
    """
    Update the TLD list in the database.

    :param data: The TLD data.
    :type data: file handle.
    :returns: dict with key "success" (boolean)
    """

    if not data:
        return {'success': False}
    line = data.readline()
    while line:
        line = line.rstrip()
        if line and not line.startswith('//'):
            TLD.objects(tld=line).update_one(set__tld=line, upsert=True)
        line = data.readline()

    # Update the package local tld_parser with the new domain info
    tld_parser = etld()

    return {'success': True}
예제 #7
0
def extract_domains(data):
    pattern = r'[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?[\.[a-zA-Z]{2,}'
    domains = [each for each in re.findall(pattern, data) if len(each) > 0]
    final_domains = []
    for item in domains:
        if len(item) > 1 and item.find('.') != -1:
            try:
                tld = item.split(".")[-1]
                check = TLD.objects(tld=tld).first()
                if check:
                    final_domains.append(item)
            except:
                pass
    return final_domains
예제 #8
0
def extract_emails(data):
    pattern = r'[a-zA-Z0-9-\.\+]+@.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?[\.[a-zA-Z]{2,}'
    emails = [each for each in re.findall(pattern, data) if len(each) > 0]
    final_emails = []
    for item in emails:
        if len(item) > 1 and item.find('.') != -1:
            try:
                tld = item.split(".")[-1]
                check = TLD.objects(tld=tld).first()
                if check:
                    final_emails.append(item)
            except:
                pass
    return final_emails
예제 #9
0
def extract_emails(data):
    pattern = r'[a-zA-Z0-9-\.\+]+@.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?[\.][a-zA-Z]{2,}'
    emails = [each for each in re.findall(pattern, data) if len(each) > 0]
    final_emails = []
    unique_emails = []
    for item in emails:
        if len(item) > 1 and item.find('.') != -1:
            try:
                tld = item.split(".")[-1]
                check = TLD.objects(tld=tld).first()
                if check:
                    if item not in unique_emails:
                        unique_emails.append(item)
            except:
                pass
    unique_emails.sort()
    return unique_emails