def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id = md5,
domain = v.domain,
datetime = tzone.now(),
source = self.ID,
referrer = 'https://www.dshield.org/feeds/suspiciousdomains_Low.txt',
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name, len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
df = df.fillna('')
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id=md5,
url="http://" + v.url[:248],
ip=v.ip,
description=v.Type,
datetime=tzone.now(),
source=self.ID,
referrer='https://cybercrime-tracker.net/index.php',
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name,
len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
if not isinstance(df.iloc[0,0], str):
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id = md5,
url = v.url[:255],
datetime = datetime.strptime(v.submission_time[:-6], '%Y-%m-%dT%H:%M:%S').replace(tzinfo=timezone.utc),
description = v.target,
referrer = v.phish_detail_url,
source = self.ID,
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.warning("AccessDenied")
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name, len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id=md5,
ip=v.ip,
datetime=datetime.strptime(
v.datetime,
'%Y-%m-%d %H:%M').replace(tzinfo=timezone.utc),
description=v.description,
referrer=v.reference,
source=self.ID,
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name,
len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id = md5,
ip = v.ip,
url = 'http://' + v.domain[:248],
datetime = datetime.strptime(v.datetime, '%Y/%m/%d_%H:%M').replace(tzinfo=timezone.utc),
description = v.description,
source = self.ID,
referrer = 'https://www.malwaredomainlist.com/mdl.php?search=' + v.ip,
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name, len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id = md5,
ip = v.ip,
datetime = tzone.now(),
source = self.ID,
referrer = 'http://cinsscore.com/list/ci-badguys.txt',
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name, len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
report = self.getBlacklist()
#print(json.dumps(blacklist))
queries = []
if 'data' in report:
for data in report['data']:
line = str(self.ID) + ","
line += str(data['ipAddress'])
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id=md5,
ip=data['ipAddress'],
datetime=tzone.now(),
description="totalReports: " +
str(data['totalReports']) +
", abuseConfidenceScore: " +
str(data['abuseConfidenceScore']),
source=self.ID,
referrer='https://www.abuseipdb.com/check/' +
data['ipAddress'],
)
except Exception as e:
print(e)
logger.error(e)
queries.append(query)
else:
logger.error("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name,
len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
try:
res = requests.get(self.URL)
except Exception as e:
logger.error(e)
if not res.text == '':
open(self.DataFilePath, 'w').write(res.text)
xml_data = open(self.DataFilePath).read()
root = ET.XML(xml_data)
all_records = []
for child in root[0]:
if child.tag == 'item':
record = {}
for subchild in child:
record[subchild.tag] = subchild.text
all_records.append(record)
df = pd.DataFrame(all_records)
queries = []
if not df.empty:
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id=md5,
domain=v.description.split(',')[0].replace(
'Host: ', ''),
ip=v.description.split(',')[1].replace(
' IP address: ', ''),
datetime=datetime.strptime(
v.title.split('(')[1].split(')')[0],
'%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc),
countrycode=v.description.split(',')[7].replace(
' country: ', ''),
description=','.join(v.description.split(',')[2:7]),
referrer=v.link,
source=self.ID,
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name,
len(queries))
return queries
def parse(self):
logger.info("start parsing: %s", self.name)
df = self.makeDataframe()
queries = []
if not df.empty:
df = df.fillna('')
for i, v in df.iterrows():
line = str(self.ID) + ","
line += str(v.values)
md5 = hashlib.md5(line.encode('utf-8')).hexdigest()
try:
query = blacklist(
id=md5,
ip=v.ip.split('|')[0],
domain=v.host,
url=v.url[:255],
datetime=datetime.strptime(
v.firstseen,
'%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone.utc),
source=self.ID,
description=v.threat + ', ' + v.malware + ', ' +
v.registrar + ', ' + str(v.asn),
countrycode=v.country.split('|')[0],
referrer='https://ransomwaretracker.abuse.ch/host/' +
v.host + '/',
)
except Exception as e:
logger.error("%s: %s", e, line)
queries.append(query)
else:
logger.info("no update")
logger.info("done parsing: %s, %s queries were parsed", self.name,
len(queries))
return queries