def URLPermutations(url): url_class = URL(url) urls = [] for url_variant in url_class.url_permutations(url): urls.append(url_variant) #print url_variant return urls
def main(): key = sys.argv[1] db = "../gsb_v4.db" platforms = ["WINDOWS"] sbl = SafeBrowsingList(key, db_path=db, platforms=platforms) #sbl.update_hash_prefix_cache() print(sbl.storage.get_threat_lists()) url = sys.argv[2] u = URL(url) print(u.url) print(u.canonical) for i in u.url_permutations(u.canonical): print(i) print(u.digest(i)) url_hashes = u.hashes print(url_hashes) full_hashes = list(url_hashes) print(full_hashes) cues = [to_hex(fh[0:4]) for fh in full_hashes] print(cues) print(sbl.storage.lookup_hash_prefix(cues)) bl = sbl.lookup_url(url) print(bl)
def test_permutations(self): for k, v in self.url_permutations.items(): p = list(URL.url_permutations(k)) self.assertEqual(p, v)
def lookup_db(): from gglsbl import SafeBrowsingList from gglsbl.protocol import URL from gglsbl.utils import to_hex res = {} rdict = { 'status':'', 'message':'', } key = '' db = '../gsb_v4.db' platforms = ['WINDOWS'] if os.path.isfile(config): cp = ConfigParser() cp.read(config) if 'api' in cp: if 'key' in cp['api']: key = cp['api']['key'] if 'database' in cp: if 'localdb' in cp['database']: db = cp['database']['localdb'] if not key: logging.error('API key not found.') rdict['status'] = 500 rdict['message'] = 'Internal Server Error' url = '' update = False if request.method == 'GET': url = request.args.get('url') update = request.args.get('update') if not url: rdict['status'] = 400 rdict['message'] = "The parameter 'url' is missing" if not rdict['status']: sbl = SafeBrowsingList(key, db_path=db, platforms=platforms) logging.debug(sbl.storage.get_threat_lists()) #if update: # sbl.update_hash_prefix_cache() u = URL(url) #res['url'] = { res = { 'query': u.url, 'canonical': u.canonical, 'permutations': [], } for i in u.url_permutations(u.canonical): p = { 'pattern': i, 'sha256': to_hex(u.digest(i)) } #res['url']['permutations'].append(p) res['permutations'].append(p) url_hashes = u.hashes full_hashes = list(url_hashes) cues = [to_hex(fh[0:4]) for fh in full_hashes] #res['cues'] = cues res['results'] = [] matched = sbl.storage.lookup_hash_prefix(cues) for m in matched: prefix = to_hex(m[1]) for p in res['permutations']: if re.match(prefix, p['sha256']): result = { 'pattern': p['pattern'], #'prefix': to_hex(m[1]), 'prefix': prefix, 'matched': str(m[0]), } res['results'].append(result) #bl = sbl.lookup_url(url) #res['matched'] = bl logging.info(res) res = jsonify(res) if not res: if not rdict["status"]: rdict["status"] = 400 rdict["message"] = "Invalid request." res = jsonify(rdict) res.status_code = rdict["status"] return res
def test_permutations(self): for k,v in self.url_permutations.items(): p = list(URL.url_permutations(k)) self.assertEqual(p, v)