def CollectORGPEOPLE(self, USERNAME):
try:
utilities.pi('\n{}Attempting to collect all of {}\'s github people'.format(INFO, USERNAME))
d = regexops.GrabNextPageOfOrgPeople("https://github.com/orgs/{}/people".format(USERNAME))
p = [] # pages visited
p.append("https://github.com/orgs/{}/people".format(USERNAME))
f = [] # list of followers, the container
#return p
while True:
if d is None: # If there is only ONE page of followers
for s in p:
d = regexops.GrabNamesOfOrgPeople(s)
for i in d:
f.append(i)
return f
# If there is more than one page of followers.
for o in p:
time.sleep(1)
if o is None:
p.remove(p[-1]) # Makes sure Nonetype is removed
for z in p:
n = regexops.GrabNamesOfOrgPeople(z)
for so in n:
f.append(so)
return f
o = regexops.GrabNextPageOfOrgPeople(o)
p.append(o)
#print p | for debugging
#return p | for debugging
except TypeError:
pass
def CollectFollowing(self, USERNAME):
try:
#print ""
#FollowingNumber = regexops.GetNumberOfWhoUserIsFollowing(USERNAME)
utilities.pi('\n{}Attempting to collect all users that {} is following on github'.format(INFO ,USERNAME))
d = regexops.GrabNextPageOfFollowing("https://github.com/{}/following".format(USERNAME))
p = [] # pages visited
p.append("https://github.com/{}/following".format(USERNAME))
f = [] # list of followers, the container
#return p
while True:
if d is None: # If there is only ONE page of followers
for s in p:
d = regexops.GetNamesOfFollowing(s)
for i in d:
f.append(i)
return f
# If there is more than one page of followers.
for o in p:
time.sleep(1)
if o is None:
p.remove(p[-1]) # Makes sure Nonetype is removed
for z in p:
n = regexops.GetNamesOfFollowing(z)
for so in n:
f.append(so)
return f
o = regexops.GrabNextPageOfFollowing(o)
p.append(o)
#print p | for debugging
#return p | for debugging
except TypeError:
pass
def code(self, query, listonly=False):
if query == 'q':
utilities.pi('{}Exiting Treasure ...'.format(notifications.INFO))
exit(0)
elif query == '' or query == ' ':
utilities.pi(' {} Empty code searches are not allowed.'.format(notifications.ERROR))
exit(0)
utilities.pi("\n {}Hunting for {}".format(notifications.INFO, query))
page = 1
breakpoint = 1
while breakpoint == 1:
params = dict(
q = query,
type = "Code",
p = page
)
r = utilities.GetHTTPostRequest(searchfor.SEARCH, params)
if searchfor.is_last_page(r.content):
utilities.pi('** No more results for {} **'.format(query))
breakpoint = 2 # Break the while loop.
break
for u in searchfor.extract(r.content):
ru = searchfor.raw_url(u)
if listonly:
print ru
else:
fn = searchfor.make_fname(u)
ret = utilities.GetHTTPRequest(ru)
if ret.status_code == 200:
codedumppath = '{}.code.dump'.format(query.replace(' ', '_'))
# Print output
utilities.pi(" {}".format(notifications.FOUND) + ru.split('/')[-1] + " " + "from {}".format(ru.split('/')[3]))
with open(codedumppath, "a+") as file:
cb = utilities.sbc(codedumppath, ret.text.encode('utf-8'))
if cb is None:
# Quick Fix.
# If there are any unicode errors, just pass them for now.
try:
filecontent = ret.content.decode('utf-8').encode('utf-8')
except Exception:
pass
file.write(filecontent + '\n')
file.close()
elif cb is True:
pass
else:
pass
page += 1
def GetWhenUserJoined(self, USERNAME):
link2profile = github.domain + USERNAME
response = utilities.GetHTTPRequest(link2profile)
try:
if response.status_code == 404:
utilities.pi("{}User not found!".format(ERROR))
return None
exit(0)
jdc = re.findall(r'<time class="join-date".*?>(.*)</time>', response.text)
jd = jdc[0]
return jd
except IndexError:
return None
def WriteUsersReposToFile(self, Username, Repos):
utilities.pi(INFO + "Collected {} of {}'s Repositories".format(len(Repos), Username))
time.sleep(3)
utilities.pi(INFO + "Wrote {} of {}'s Repositories to disk\n".format(len(Repos), Username))
idwe = 'https://github.com'
Username = Username.lower()
repospath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/repositories/{}.repositories'.format(Username)
with open(repospath, 'a+') as file:
for line in Repos:
line = line.lower()
bc = utilities.sbc(repospath, line)
if bc is None:
#print line | for debugging rounds.
reponame = line.split('/')[2]
file.write(reponame + " - " + idwe + line + "\n")
def CollectRepositories(self, USERNAME):
orgdetection = utilities.GetHTTPRequest('https://github.com/orgs/{}/people'.format(USERNAME))
if orgdetection.status_code == 404:
pass
elif orgdetection.status_code == 200:
utilities.pi("{}This query is for USERS only! try this insead: './es.py -g org atom repos'".format(ERROR))
exit(0)
try:
#print ""
#FollowersNumber = regexops.GetNumberOfUserFollowers(USERNAME)
utilities.pi('\n{}Attempting to collect all of {}\'s github repositories'.format(INFO, USERNAME))
d = regexops.GrabNextPageOfRepos("https://github.com/{}/repositories".format(USERNAME))
p = [] # pages visited
p.append("https://github.com/{}/repositories".format(USERNAME))
f = [] # list of followers, the container
#return p
while True:
if d is None: # If there is only ONE page of followers
for s in p:
d = regexops.GetRepoNames(s)
for i in d:
f.append(i)
return f
# If there is more than one page of followers.
for o in p:
time.sleep(1)
if o is None:
p.remove(p[-1]) # Makes sure Nonetype is removed
for z in p:
n = regexops.GetRepoNames(z)
for so in n:
f.append(so)
return f
o = regexops.GrabNextPageOfRepos(o)
p.append(o)
#print p | for debugging
#return p | for debugging
except TypeError:
pass
def search(self):
utilities.pi("""
#######################################################
# #
# """+colors.Y+"""Treasure"""+colors.N+""" - """+colors.G+"""HUNT"""+colors.N+""" """+colors.B+"""for"""+colors.N+""" """+colors.W+"""CODE"""+colors.N+""" #
# Type q or ^C to exit """+colors.Y+"""Treasure"""+colors.N+""" #
# #
# """+colors.R+"""Help ?"""+colors.N+""": #
# """+colors.C+"""1. https://help.github.com/articles/searching-code/"""+colors.N+""" #
# #
#######################################################""")
breakpoint = 0 # Break point for the while loop.
while breakpoint == 0:
try:
choice = utilities.select(" search|code: ")
searchfor.code(choice)
except KeyboardInterrupt:
breakpoint == 1
utilities.pi("\n{}Exiting Treasure ...".format(notifications.INFO))
break
def WriteUserToDatabaseSILENTLY(self, Username):
Username = Username.lower()
userdatabase = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/users/user.database'
DNC = utilities.sbc(userdatabase, Username) # Database name check
if DNC is True:
pass
IDN = regexops.GetUserID(Username)
if IDN is None:
utilities.pi("\n{}User doesn't exist.\n".format(ERROR, ))
exit(0)
databasepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/users/user.database'
with open(databasepath, 'a+') as file:
bc = utilities.sbc(databasepath, Username)
if bc is None:
#print line | for debugging rounds.
file.write(Username + " - " + str(IDN) + " - " + github.domain + Username + "\n")
if bc is True:
pass
def CollectFollowers(self, USERNAME):
try:
#print ""
FollowersNumber = regexops.GetNumberOfUserFollowers(USERNAME)
utilities.pi('\n{}Attempting to collect {} of {}\'s github followers'.format(INFO, FollowersNumber ,USERNAME))
d = regexops.GrabNextPageOfFollowers("https://github.com/{}/followers".format(USERNAME))
p = [] # pages visited
p.append("https://github.com/{}/followers".format(USERNAME))
f = [] # list of followers, the container
#return p
while True:
if d is None: # If there is only ONE page of followers
for s in p:
d = regexops.GetNamesOfFollowers(s)
for i in d:
f.append(i)
return f
# If there is more than one page of followers.
for o in p:
time.sleep(1)
if o is None:
p.remove(p[-1]) # Makes sure Nonetype is removed
for z in p:
n = regexops.GetNamesOfFollowers(z)
for so in n:
f.append(so)
return f
o = regexops.GrabNextPageOfFollowers(o)
p.append(o)
#print p | for debugging
#return p | for debugging
except TypeError:
#utilities.pi(INFO + 'User Doesn\'t exist.')
#exit(0)
pass
def QueryUserStarredRepoDatabase(self, username, reponame):
username = username.lower()
reposdatabasepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/repositories/starred/{}.starred.repos'.format(username)
utilities.pi(" ")
with open(reposdatabasepath, 'r') as file:
# Database item check
ic = utilities.sbc(reposdatabasepath, reponame) # Database item/name check
for item in file:
itemlink = item.split(' - ')[1].replace("\n", '') # Link to github profile
item = item.split(' - ')[0] # github username
if reponame in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in {}'s starred repositories database".format(FAIL, reponame, username))
break
utilities.pi(" ")
def WriteUserToDatabaseVERBALLY(self, Username):
Username = Username.lower()
IDN = regexops.GetUserID(Username)
if IDN is None:
utilities.pi("\n{}User doesn't exist.\n".format(ERROR, ))
exit(0)
databasepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/users/user.database'
with open(databasepath, 'a+') as file:
bc = utilities.sbc(databasepath, Username)
if bc is None:
utilities.pi('\n{}Added {} to the user database.\n'.format(INFO, Username))
#print line | for debugging rounds.
file.write(Username + " - " + str(IDN) + " - " + github.domain + Username + "\n")
if bc is True:
utilities.pi('\n{}{} has already been added to the user database.\n'.format(INFO, Username))
def WriteWhoUserIsFollowingToFile(self, Username, Following):
utilities.pi(INFO + "Collected {} users that {} is following".format(len(Following), Username))
time.sleep(3)
utilities.pi(INFO + "Writing {} to disk".format(len(Following), Username))
Username = Username.lower() # Override any UPPERCASE strings
followspath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/following/{}.following'.format(Username)
with open(followspath, 'a+') as file:
for line in Following:
line = line.lower()
bc = utilities.sbc(followspath, line)
if bc is None:
#print line | for debugging rounds.
IDN = regexops.GetUserID(line)
if IDN is None:
IDN = "No Profile Traceroute."
file.write(line + " - " + IDN + " - " + github.domain + line + "\n")
utilities.pi(INFO + "Wrote {} to disk\n".format(len(Following), Username))
def WriteOrgPeopleToFile(self, Username, people):
utilities.pi(INFO + "Collected {} of {}'s people".format(len(people), Username))
time.sleep(3)
utilities.pi(INFO + "Writing {} of {}'s people to disk".format(len(people), Username))
Username = Username.lower()
peoplespath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/organizations/people/{}s.people'.format(Username)
with open(peoplespath, 'a+') as file:
for line in people:
line = line.lower()
bc = utilities.sbc(peoplespath, line)
if bc is None:
#print line | for debugging rounds.
IDN = regexops.GetUserID(line)
if IDN is None:
IDN = "No Profile Traceroute."
file.write(line + " - " + IDN + " - " + github.domain + line + "\n")
utilities.pi(INFO + "Wrote {} of {}'s people to disk.\n".format(len(people), Username))
def WriteUsersFollowersToFile(self, Username, Followers):
utilities.pi(INFO + "Collected {} of {}'s followers".format(len(Followers), Username))
time.sleep(3)
utilities.pi(INFO + "Writing {} of {}'s followers to disk".format(len(Followers), Username))
Username = Username.lower()
#os.chdir('..')
followerspath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/followers/{}.followers'.format(Username)
with open(followerspath, 'a+') as file:
for line in Followers:
line = line.lower()
bc = utilities.sbc(followerspath, line)
if bc is None:
#print line | for debugging rounds.
IDN = regexops.GetUserID(line)
if IDN is None:
IDN = "No Profile Traceroute."
file.write(line + " - " + IDN + " - " + github.domain + line + "\n")
utilities.pi(INFO + "Wrote {} of {}'s followers to disk.\n".format(len(Followers), Username))
def BlockchainIdentifiers(self, file):
BCIDSExtract = extract.BlockchainIdentifiers(file)
if len(BCIDSExtract) is 0:
utilities.pi("{}No Blockchain Identifiers in {}".format(notifications.FAIL, file))
if len(BCIDSExtract) > 0:
FoundBCIDS = [] # Re-filter, so you get exactly what you're looking for.
for instance in BCIDSExtract:
BCIDSRegex = re.compile(r'[0-9a-f]{5,8}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{4}\-[0-9a-f]{5,13}')
BCIDSList = BCIDSRegex.search(instance)
BCIDSs = BCIDSList.group()
FoundBCIDS.append(BCIDSs)
UOD = {}
for item in FoundBCIDS:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" Blockchain Identifiers ")
utilities.pi("--------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Blockchain Identifier from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Blockchain Identifiers from {}\n".format(str(count), file))
def FacebookAccessTokens(self, file):
FATExtract = extract.FacebookAccessTokens(file)
if len(FATExtract) is 0:
utilities.pi("{}No Facebook Access Tokens in {}".format(notifications.FAIL, file))
if len(FATExtract) > 0:
FoundFATs = [] # Re-filter, so you get exactly what you're looking for.
for instance in FATExtract:
FATRegex = re.compile(r'(access_token\=[0-9]{15}\|(.*){27})')
FATList = FATRegex.search(instance)
FATs = FATList.group()
FoundFATs.append(FATs)
UOD = {}
for item in FoundFATs:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" Facebook Access Tokens ")
utilities.pi("--------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Facebook Access Token from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Facebook Access Tokens from {}\n".format(str(count), file))
def InstagramAccessToken(self, file):
IATExtract = extract.InstagramAccessTokens(file) # Collected emails
if len(IATExtract) is 0:
utilities.pi("{}No Instagram access tokens in {}".format(notifications.FAIL, file))
if len(IATExtract) > 0: # legit file, containing at least 1 email address.
FoundIATs = [] # Re-filter, so you get exactly what you're looking for.
for instance in IATExtract:
IATRegex = re.compile(r'[0-9]{7,10}\.[0-9a-f]{5,8}\.[0-9a-f]{32}')
IATContainer = IATRegex.search(instance)
IATs = IATContainer.group()
FoundIATs.append(IATs)
UOD = {}
for item in FoundIATs:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------------------------")
utilities.pi(" EXTRACTED Instagram access tokens ")
utilities.pi("--------------------------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} instagram access token from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} instagram access tokens from {}\n".format(str(count), file))
def BTCAddresses(self, file):
BTCWAExtract = extract.BitcoinWalletAddress(file)
if len(BTCWAExtract) is 0:
utilities.pi("{}No Bitcoin addresses in {}".format(notifications.FAIL, file))
if len(BTCWAExtract) > 0: # legit file, containing at least 1 link.
FoundWallets = [] # Re-filter, so you get exactly what you're looking for.
for instance in BTCWAExtract:
BTCWalletRegex = re.compile(r'(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{26,30}(?![a-km-zA-HJ-NP-Z0-9])|(?<![a-km-zA-HJ-NP-Z0-9])[13][a-km-zA-HJ-NP-Z0-9]{33,35}(?![a-km-zA-HJ-NP-Z0-9])')
wallet = BTCWalletRegex.findall(instance)
for address in wallet: FoundWallets.append(address)
UOD = {}
for item in FoundWallets:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED BTC Addresses ")
utilities.pi("--------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Bitcoin address from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Bitcoin addresses from {}\n".format(str(count), file))
def QueryUserFollowingDatabase(self, username, followingname):
username = username.lower()
followingdatabasepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/following/{}.following'.format(username)
unidbool = followingname.isdigit() # smart case detection
utilities.pi(" ")
if unidbool is True:
with open(followingdatabasepath, 'r') as file:
# Database item check
ic = utilities.sbc(followingdatabasepath, followingname) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace("\n", '')
itemid = item.split(' - ')[0].replace("\n", '') # Link to github profile
item = item.split(' - ')[1] # github username
if followingname in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in {}'s following database".format(FAIL, followingname, username))
break
elif unidbool is False:
with open(followingdatabasepath, 'r') as file:
# Database item check
ic = utilities.sbc(followingdatabasepath, followingname) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace("\n", '')
itemid = item.split(' - ')[1].replace("\n", '') # Link to github profile
item = item.split(' - ')[0] # github username
if followingname in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in {}'s following database".format(FAIL, followingname, username))
break
utilities.pi(" ")
def QueryUserDatabase(self, UNID):
userdatabase = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/users/user.database'
unidbool = UNID.isdigit() # smart case detection
efb = os.stat(userdatabase).st_size == 0
if efb is True:
utilities.pi("{} User database is currently empty!".format(FAIL))
exit(0)
utilities.pi(" ")
if unidbool is False:
with open(userdatabase, 'r') as file:
# Database item check
ic = utilities.sbc(userdatabase, UNID) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace("\n", '')
itemid = item.split(' - ')[1].replace("\n", '') # Link to github profile
item = item.split(' - ')[0] # github username
if UNID in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in database".format(FAIL, UNID))
break
elif unidbool is True:
with open(userdatabase, 'r') as file:
# Database item check
ic = utilities.sbc(userdatabase, UNID) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace("\n", '')
itemid = item.split(' - ')[0].replace("\n", '') # Link to github profile
item = item.split(' - ')[1] # github username
if UNID in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in database".format(FAIL, UNID))
break
utilities.pi(" ")
def PhoneNumbers(self, file):
PNExtract = extract.PhoneNumbers(file)
if len(PNExtract) is 0:
utilities.pi("{}No Phone numbers in {}".format(notifications.FAIL, file))
if len(PNExtract) > 0 and len(PNExtract[0]) < 15: # Try not to grab any CCNs
FoundPhoneNumbers = [] # Re-filter, so you get exactly what you're looking for.
for instance in PNExtract:
PNRegex = re.compile(r'(\d{3})\D*(\d{3})\D*(\d{4})\D*(\d*)$')
PNC = PNRegex.search(instance)
PN = PNC.group()
FoundPhoneNumbers.append(PN)
UOD = {}
for item in FoundPhoneNumbers:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED Phone Numbers ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
if output.isdigit() is False and ":" not in output and "@" not in output:
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "{} Extracted Phone Number(s) from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "{} Extracted Phone Number(s) from {}\n".format(str(count), file))
if len(PNExtract) is 15:
utilities.pi("{}No Phone numbers in {}".format(notifications.FAIL, file))
def QueryOrgPeopleDatabase(self, username, person):
username = username.lower()
orgpeoplepath = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) + '/core/database/organizations/people/{}s.people'.format(username)
utilities.pi(" ")
puib = person.isdigit()
if puib is False:
with open(orgpeoplepath, 'r') as file:
# Database item check
ic = utilities.sbc(orgpeoplepath, person) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace('\n', '')
itemid = item.split(' - ')[1].replace("\n", '') # Link to github profile
item = item.split(' - ')[0] # github username
if person in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in {}'s people database".format(FAIL, person, username))
break
utilities.pi(" ")
if puib is True:
with open(orgpeoplepath, 'r') as file:
# Database item check
ic = utilities.sbc(orgpeoplepath, person) # Database item/name check
for item in file:
itemlink = item.split(' - ')[2].replace('\n', '')
itemid = item.split(' - ')[0].replace("\n", '') # Link to github profile
item = item.split(' - ')[1] # github username
if person in item and ic is True:
utilities.pi("{}{}".format(FOUND, item + " - " + itemid + " - " + itemlink))
break
if ic is None:
utilities.pi("{}{} Not found in {}'s people database".format(FAIL, person, username))
break
utilities.pi(" ")
def MACAddresses(self, file):
MACExtract = extract.MACs(file)
if len(MACExtract) is 0:
utilities.pi("{}No MAC addresses in {}".format(notifications.FAIL, file))
if len(MACExtract) > 0: # legit file, containing at least 1 MAC, (: or - deliminated) address.
FoundMACS = [] # Re-filter, so you get exactly what you're looking for.
for instance in MACExtract:
macsrch = re.compile(r'([0-9A-Fa-f]{2}\-[0-9A-Fa-f]{2}\-[0-9A-Fa-f]{2}\-[0-9A-Fa-f]{2}\-[0-9A-Fa-f]{2}\-[0-9A-Fa-f]{2})')
macsrch1 = re.compile(r'([0-9A-Fa-f]{2}\:[0-9A-Fa-f]{2}\:[0-9A-Fa-f]{2}\:[0-9A-Fa-f]{2}\:[0-9A-Fa-f]{2}\:[0-9A-Fa-f]{2})')
cdm = macsrch.findall(instance)
hdm = macsrch1.findall(instance)
for mach in hdm: FoundMACS.append(mach)
for macc in cdm: FoundMACS.append(macc)
UOD = {}
for item in FoundMACS:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED MACs ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "{} Extracted MAC address from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "{} Extracted MAC addresses from {}\n".format(str(count), file))
def HyperLinks(self, file):
LinkExtract = extract.HyperLinks(file)
if len(LinkExtract) is 0:
utilities.pi("{}No Links in {}".format(notifications.FAIL, file))
if len(LinkExtract) > 0: # legit file, containing at least 1 link.
FoundLinks = [] # Re-filter, so you get exactly what you're looking for.
for instance in LinkExtract:
LinkExtractRegex = re.compile(r'^((https|ftp|http|data|dav|cid|chrome|apt|cvs|bitcoin|dns|imap|irc|ldap|mailto|magnet|proxy|res|rsync|rtmp|rtsp|shttp|sftp|skype|ssh|snmp|snews|svn|telnet|tel|tftp|udp)://|(www|ftp)\.)[a-z0-9-]+(\.[a-z0-9-]+)+([/?].*)?$')
LinkList = LinkExtractRegex.search(instance)
Links = LinkList.group()
FoundLinks.append(Links)
UOD = {}
for item in FoundLinks:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED links ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} link from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} links from {}\n".format(str(count), file))
def HashTypes(self, file):
HashExtract = extract.HashTypes(file)
if len(HashExtract) is 0:
utilities.pi("{}No Hashes in {}".format(notifications.FAIL, file))
if len(HashExtract) > 0: # If you actually grab something then continue
FoundHashes = [] # Re-filter, so you get exactly what you're looking for.
for instance in HashExtract:
# Stand-alone regex's for finding hash values.
md5regex = re.compile(r'[a-fA-F0-9]{32}')
sha1regex = re.compile(r'[[a-fA-F0-9]{40}')
sha256regex = re.compile(r'[a-fA-F0-9]{64}')
sha384regex = re.compile(r'[a-fA-F0-9]{96}')
sha512regex = re.compile(r'[a-fA-F0-9]{128}')
# Find hash value of given regex's
md5list = md5regex.findall(instance)
sha1list = sha1regex.findall(instance)
sha256list = sha256regex.findall(instance)
sha384list = sha384regex.findall(instance)
sha512list = sha512regex.findall(instance)
# Add hash values to un-filtered list for filtering.
for md5 in md5list: FoundHashes.append(md5)
for sha1 in sha1list: FoundHashes.append(sha1)
for sha256 in sha256list: FoundHashes.append(sha256)
for sha384 in sha384list: FoundHashes.append(sha384)
for sha512 in sha512list: FoundHashes.append(sha512)
UOD = {} # Filter out any duplicates
for item in FoundHashes:
UOD[item] = 1 # No duplicates at all !
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" Extracted Hash Values ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Hash found from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Hash(es) found from {}\n".format(str(count), file))
def IPv6Addresses(self, file):
IPv6Extract = extract.IPv6Addresses(file)
if len(IPv6Extract) is 0:
utilities.pi("{}No IPv6 addresses in {}".format(notifications.FAIL, file))
if len(IPv6Extract) > 0: # legit file, containing at least 1 ipv6 number.
FoundIPV6s = [] # Re-filter, so you get exactly what you're looking for.
for instance in IPv6Extract:
IPv6Regex = re.compile(r'^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$')
IPv6List = IPv6Regex.search(instance)
IPv6s = IPv6List.group()
FoundIPV6s.append(IPv6s)
UOD = {}
for item in FoundIPV6s:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED IPv6s ")
utilities.pi("--------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} IPv6 address from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} IPv6 addresses from {}\n".format(str(count), file))
def IPv4Addresses(self, file):
IPv4Extract = extract.IPv4Addresses(file) # Collected ipv4s
if len(IPv4Extract) is 0:
utilities.pi("{}No IPv4 addresses in {}".format(notifications.FAIL, file))
if len(IPv4Extract) > 0: # legit file, containing at least 1 ipv4 address.
FoundIPv4s = [] # Re-filter, so you get exactly what you're looking for.
for instance in IPv4Extract:
IPv4Regex = re.compile(r'([0-9]+)(?:\.[0-9]+){3}')
IPv4Container = IPv4Regex.search(instance)
IPv4s = IPv4Container.group()
FoundIPv4s.append(IPv4s)
UOD = {}
for item in FoundIPv4s:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED IPV4s ")
utilities.pi("--------------------------")
count = 0
for output in keys:
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} IPv4 address from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} IPv4 addresses from {}\n".format(str(count), file))
def CreditCardNumbers(self, file):
CCNExtract = extract.CreditCardNumbers(file)
if len(CCNExtract) is 0:
utilities.pi("{}No Creditcard numbers in {}".format(notifications.FAIL, file))
if len(CCNExtract) > 0: # legit file, containing at least 1 CCN numbers.
FoundCCNs = [] # Re-filter, so you get exactly what you're looking for.
for instance in CCNExtract:
CCNRegex = re.compile(r'^(?:(4[0-9]{12}(?:[0-9]{3})?)|(5[1-5][0-9]{14})|(6(?:011|5[0-9]{2})[0-9]{12})|(3[47][0-9]{13})|(3(?:0[0-5]|[68][0-9])[0-9]{11})|((?:2131|1800|35[0-9]{3})[0-9]{11}))$')
CCNLIST = CCNRegex.search(instance)
CCN = CCNLIST.group()
FoundCCNs.append(CCN)
UOD = {}
for item in FoundCCNs:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED CCNs ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Creditcard number from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Creditscard numbers from {}\n".format(str(count), file))
def SocialSecurityNumbers(self, file):
SSNExtract = extract.SSNs(file)
if len(SSNExtract) is 0:
utilities.pi("{}No Social securtiy numbers in {}".format(notifications.FAIL, file))
if len(SSNExtract) > 0: # legit file, containing at least 1 SSN, ( - deliminated) number.
FoundSSNs = [] # Re-filter, so you get exactly what you're looking for.
for instance in SSNExtract:
SSN1Regex = re.compile(r'^(?!000|666)[0-8][0-9]{2}(?!00)[0-9]{2}(?!0000)[0-9]{4}$')
SSN2Regex = re.compile(r'^(?!000|666)[0-8][0-9]{2}-(?!00)[0-9]{2}-(?!0000)[0-9]{4}$')
SSN1LIST = SSN1Regex.findall(instance) # no deliminator
SSN2LIST = SSN2Regex.findall(instance) # - deliminator
for SSNV1 in SSN1LIST: FoundSSNs.append(SSNV1)
for SSNV2 in SSN2LIST: FoundSSNs.append(SSNV2)
UOD = {}
for item in FoundSSNs:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED SSNs ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Social security number from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Social security numbers from {}\n".format(str(count), file))
def Emails(self, file):
EmailExtract = extract.Emails(file) # Collected emails
if len(EmailExtract) is 0:
utilities.pi("{}No Emails in {}".format(notifications.FAIL, file))
if len(EmailExtract) > 0: # legit file, containing at least 1 email address.
FoundEmails = [] # Re-filter, so you get exactly what you're looking for.
for instance in EmailExtract:
EmailRegex = re.compile(r'[\w\-][\w\-\.]+@[\w\-][\w\-\.]+[a-zA-Z]{1,4}')
EmailContainer = EmailRegex.search(instance)
Emails = EmailContainer.group()
FoundEmails.append(Emails)
UOD = {}
for item in FoundEmails:
UOD[item] = 1
keys = UOD.keys()
utilities.pi("--------------------------")
utilities.pi(" EXTRACTED Emails ")
utilities.pi("--------------------------")
count = 0
for output in keys:
database.ExportFile(output)
count += 1
utilities.pi(notifications.INFO + output)
if count is 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Email Address from {}\n".format(str(count), file))
elif count > 1:
utilities.pi("\n" + notifications.STATUS + "Extracted {} Email Addresses from {}\n".format(str(count), file))