def main():
parser = argparse.ArgumentParser(
description='Downloads samples from VT Intelligence.')
parser.add_argument('InputJSON',
help='The VT Intelligence notifications JSON.')
parser.add_argument('OutputDirectory',
help='The output directory for the samples.')
parser.add_argument("-a",
"--apikey",
help="Your VT Intelligence API key."
"",
required=True)
args = parser.parse_args()
try:
os.stat(args.OutputDirectory)
except:
os.mkdir(args.OutputDirectory)
inputjson = json.loads(open(args.InputJSON).read())
api = IntelApi(args.apikey)
for notification in inputjson['notifications']:
print("Downloading {0}".format(notification['sha256']))
api.get_file(notification['sha256'], args.OutputDirectory)
print("\tDownloaded {0}".format(notification['sha256']))
def get_intel_vt(self):
block = self.VTAPI
config_dict = self.my_config.get(block, None)
if config_dict is None:
raise Exception("Missing %s config" % block)
if self.INTEL in config_dict and \
config_dict.get(self.INTEL, False):
apikey = config_dict.get(self.API_KEY)
return IntelApi(apikey)
raise Exception("Unable to instantiate IntelApi for VT")
def main():
config = configparser.ConfigParser()
if os.path.isfile(os.path.expanduser('~/.pyti')):
config.read(os.path.expanduser('~/.pyti'))
intelApi = IntelApi(config['VTi']['apikey'])
privApi = PrivateApi(config['VTi']['apikey'])
allNotes = get_all_notifications(intelApi)
keyFunc = lambda f: f['ruleset_name']
dedup = list()
dedupHash = list()
for s in allNotes:
if s['sha1'] not in dedupHash:
dedup.append(s)
dedupHash.append(s['sha1'])
mwr = mwrepo.mwrepo(config['MalwareRepo']['basePath'])
toDownload = [s for s in dedup if not mwr.sha1exists(s['sha1'])]
printNoteSummary('To Download',
groupby(sorted(toDownload, key=keyFunc), key=keyFunc))
print('Downloading {} files...'.format(len(toDownload)))
ind = 0
errors = list()
for s in toDownload:
print('... ({}/{}): {} '.format(ind + 1, len(toDownload), s['sha1']),
end='')
ss = privApi.get_file(s['sha1'])
if type(ss) == dict:
print('= Error: {}'.format(ss['error']), end='')
errors.append(s['sha1'])
else:
mwr.savefile(ss)
print()
ind = ind + 1
idsToRemove = list(
set([s['id'] for s in allNotes if s['sha1'] not in errors]))
delete_intel_notifications(intelApi, idsToRemove)
def main():
parser = argparse.ArgumentParser(
description='Downloads samples from VT Intelligence based upon a query.')
parser.add_argument('OutputDirectory',
help='The output directory for the samples.')
parser.add_argument('Query',
help='The valid VTI query.')
parser.add_argument("-a", "--apikey",
help="Your VT Intelligence API key."
"", required=True)
parser.add_argument("-n", "--number_of_samples",
help="The number of files to download."
"",
type=int, default=50, required=True)
args = parser.parse_args()
try:
os.stat(args.OutputDirectory)
except:
os.makedirs(args.OutputDirectory)
intel_api = IntelApi(args.apikey)
public_api = PublicApi(args.apikey)
# private_api = PrivateApi(args.apikey)
downloads = 0
nextpage = None
df = pd.DataFrame()
while downloads <= args.number_of_samples:
try:
results = None
while results is None:
nextpage, results = intel_api.get_hashes_from_search(args.Query, nextpage)
if results.status_code != 200:
print("\tError downloading hashes, retrying...")
time.sleep(60)
results = None
else:
results = results.json()
print("Downloading hashes for samples...")
for hash in results['hashes']:
if downloads < args.number_of_samples:
filename = os.path.join(args.OutputDirectory,
hash.upper())
try:
os.stat(args.OutputDirectory)
except:
os.makedirs(args.OutputDirectory)
if not os.path.isfile(filename):
print("Downloading {0}".format(hash))
downloaded = False
while downloaded is False:
try:
response = intel_api.get_file(hash, args.OutputDirectory)
except KeyboardInterrupt:
if os.path.isfile(filename):
os.remove(filename)
raise
print("\t\tDownloaded {0}".format(hash))
print("\t\tVerifying hash...")
expected_hash = hash.upper()
dl_hash = sha256_file(filename).upper()
if expected_hash != dl_hash:
print("\t**** DOWNLOAD ERROR! SHA256 Does not match!")
print("\t\tExpected SHA256: {0}".format(expected_hash))
print("\t\tCalculated SHA256: {0}".format(dl_hash))
print("\t\tHave you exceeded your quota?")
else:
print("\t\t\tHash verified!")
downloads += 1
print("\t\tDownloaded {0:,} samples...".format(downloads))
downloaded = True
file_report = None
while file_report is None:
print("\t\tDownloading file report...")
file_report = public_api.get_file_report(hash)
if 'error' in file_report:
print("\t\t\t\tError, retrying...")
time.sleep(60)
file_report = None
ds = pd.Series(file_report['results'])
ds.name = hash.upper()
df = df.append(ds)
else:
break
if nextpage is None or downloads >= args.number_of_samples:
break
except KeyboardInterrupt:
print("Caught CTRL-C!")
break
now = datetime.datetime.now()
now_str = "{0}_{1:02}_{2:02}_{3:02}_{4:02}_{5:02}_{6}".format(now.year,
now.month,
now.day,
now.hour,
now.minute,
now.second,
now.microsecond)
print("Writing metadata CSV...")
df.to_csv(os.path.join(args.OutputDirectory, "vti_metadata_{0}.csv".format(now_str)))
print("Downloaded {0:,} Total Samples".format(downloads))
def main():
parser = argparse.ArgumentParser(
description='Downloads samples from VT Intelligence.')
parser.add_argument('OutputDirectory',
help='The output directory for the samples.')
parser.add_argument("-a",
"--apikey",
help="Your VT Intelligence API key."
"",
required=True)
args = parser.parse_args()
try:
os.stat(args.OutputDirectory)
except:
os.mkdir(args.OutputDirectory)
api = IntelApi(args.apikey)
downloads = 0
nextpage = None
while True:
samplestodelete = []
nextpage, results = api.get_intel_notifications_feed(nextpage)
results = results.json()
print("Downloading {0} Samples...".format(len(
results['notifications'])))
for notification in results['notifications']:
downloads += 1
subdir = os.path.join(args.OutputDirectory,
notification['ruleset_name'])
filename = os.path.join(subdir, notification['sha256'])
try:
os.stat(subdir)
except:
os.mkdir(subdir)
print("Downloading {0}".format(notification['sha256']))
response = api.get_file(notification['sha256'], subdir)
print("\tDownloaded {0}".format(notification['sha256']))
expected_hash = notification['sha256'].upper()
dl_hash = sha256_file(filename).upper()
if expected_hash != dl_hash:
print("**** DOWNLOAD ERROR! SHA256 Does not match!")
print("\tExpected SHA256: {0}".format(expected_hash))
print("\tCalculated SHA256: {0}".format(dl_hash))
print("\tWill not delete this sample from the feed.")
print("\tHave you exceeded your quota?")
else:
samplestodelete.append(notification['id'])
if len(samplestodelete) > 0:
api.delete_intel_notifications(samplestodelete)
print("Deleted {0} Samples From Feed".format(len(samplestodelete)))
if nextpage is None:
break
print("Downloaded {0} Total Samples".format(downloads))
def main():
parser = argparse.ArgumentParser(
description='Downloads samples from VT Intelligence.',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('OutputDirectory',
help='The output directory for the samples.')
parser.add_argument("-a", "--apikey",
help="Your VT Intelligence API key."
"", required=True)
parser.add_argument("-p", "--positives",
help="Detections must have at least this many positives."
"",
type=int, default=25)
parser.add_argument("-n", "--number_of_samples",
help="The number of files to download. "
"Set to zero for all downloads.",
type=int, default=0)
parser.add_argument("-d", "--delete_downloaded",
help="Delete downloaded samples and metadata from feed."
"", action='store_true')
parser.add_argument("-dn", "--delete_non_matches",
help="Delete samples that do not match from feed."
"", action='store_true')
parser.add_argument("-dd", "--dont_download_sample",
help="Enable to just get metadata, without downloading samples."
"", action='store_true')
args = parser.parse_args()
try:
os.stat(args.OutputDirectory)
except:
os.makedirs(args.OutputDirectory)
intel_api = IntelApi(args.apikey)
downloads = 0
nextpage = None
df = pd.DataFrame()
rows_to_add = []
while True:
try:
results = None
while results is None:
results = intel_api.get_intel_notifications_feed(nextpage)
nextpage = results['results']['next']
results = results['results']
if 'error' in results:
print("\tError downloading hashes, retrying...")
time.sleep(60)
results = None
print("Downloading hashes for samples...")
for notification in results['notifications']:
if int(notification['positives']) >= args.positives:
subdir = os.path.join(args.OutputDirectory,
notification['ruleset_name'],
notification['subject'])
filename = os.path.join(subdir, notification['sha256'])
if not os.path.isfile(filename):
# Make the directory
print("\tDownloading {0}".format(notification['sha256']))
if not args.dont_download_sample:
try:
os.stat(subdir)
except:
os.makedirs(subdir)
downloaded = False
while downloaded is False:
try:
response = intel_api.get_file(notification['sha256'], subdir)
except KeyboardInterrupt:
if os.path.isfile(filename):
os.remove(filename)
raise
print("\t\tDownloaded {0}".format(notification['sha256']))
print("\t\tVerifying hash...")
expected_hash = notification['sha256'].upper()
dl_hash = sha256_file(filename).upper()
if expected_hash != dl_hash:
print("\t**** DOWNLOAD ERROR! SHA256 Does not match!")
print("\t\tExpected SHA256: {0}".format(expected_hash))
print("\t\tCalculated SHA256: {0}".format(dl_hash))
print("\t\tWill not delete this sample from the feed.")
print("\t\tHave you exceeded your quota?")
else:
print("\t\t\tHash verified!")
downloaded = True
if args.delete_downloaded:
print("\t\tDeleting downloaded sample from feed...")
del_response = intel_api.delete_intel_notifications([notification['id']])
else:
print("\t\tSkipping sample download, downloading metadata...")
if args.delete_downloaded:
print("\t\tDeleting downloaded sample from feed...")
del_response = intel_api.delete_intel_notifications([notification['id']])
downloads += 1
print("\t\tDownloaded {0:,} samples...".format(downloads))
else:
print("\tDeleting duplicate sample from feed...")
if args.delete_downloaded:
del_response = intel_api.delete_intel_notifications([notification['id']])
ds = pd.Series(notification)
ds.name = notification['sha256']
ds_scans = pd.Series(notification['scans'])
ds_scans.name = notification['sha256']
ds = ds.append(ds_scans)
rows_to_add.append(ds)
else:
if args.delete_non_matches:
# Delete the notification if it does not match
del_response = intel_api.delete_intel_notifications([notification['id']])
if args.number_of_samples > 0 and downloads >= args.number_of_samples:
break
if nextpage is None or (args.number_of_samples > 0 and
downloads >= args.number_of_samples):
break
except KeyboardInterrupt:
print("Caught CTRL-C!")
break
print("Assembling HDF...")
df = df.append(rows_to_add)
now = datetime.datetime.now()
now_str = "{0}_{1:02}_{2:02}_{3:02}_{4:02}_{5:02}_{6}".format(now.year,
now.month,
now.day,
now.hour,
now.minute,
now.second,
now.microsecond)
print("Writing metadata HDF...")
df.to_hdf(os.path.join(args.OutputDirectory, "vti_metadata_{0}.hdf".format(now_str)), 'data')
print("Downloaded {0:,} Total Samples".format(downloads))