def __init__(self, filename, config=None):
if not is_file(filename):
return None
self.config = objdict({})
# initialize default config opts
# these could be overridden by user config
self.config.verbose = False
self.config.enablefilevisualization = False
self.config.enablebytefreqhistogram = False
self.config.enableentropycompressionstats = False
# override default config opts
if config:
for key, value in config.iteritems():
self.config[key] = value
# initialize config opts that cannot be overridden
self.config.filename = filename
self.report = objdict({})
self.report.filename = filename
self.report.filebasename = None
self.report.filedirname = None
self.report.filemimetype = None
self.report.filemagic = None
self.report.filesize = None
self.report.fileminsize = None
self.report.filecompressionratio = None
self.report.fileentropy = None
self.report.fileentropycategory = None
self.report.subfiles = None
self.report.hashes = objdict({})
self.report.visual = objdict({})
def __init__(self, filename):
self.config = None
self.report = None
self.lookuplist = None
if internet() and is_file(filename):
self.config = objdict({})
self.config.filename = filename
self.report = objdict({})
self.lookuplist = ["shadowserver", "teamcymru", "metascan", "virustotal"]
def __init__(self, filename):
if is_file(filename):
self.config = objdict({})
self.config.filename = filename
self.config.md5 = file_hashes(self.config.filename, 'md5')
self.config.sha1 = file_hashes(self.config.filename, 'sha1')
self.api = TeamCymruApi()
self.report = objdict({})
else:
return None
def __init__(self, filename):
if is_file(filename):
self.config = objdict({})
self.config.filename = filename
self.config.md5 = file_hashes(self.config.filename, 'md5')
self.config.ssdeep = file_hashes(self.config.filename, 'ssdeep')
self.api = ShadowServerApi()
self.report = objdict({})
else:
return None
def __init__(self, filename):
if is_file(filename):
self.config = objdict({})
self.config.filename = filename
self.config.apikey = None
self.config.url = objdict({})
self.config.url.hashreport = "https://hashlookup.metascan-online.com/v2/hash/%s" % (file_hashes(self.config.filename, "sha256"))
self.config.params = { "apikey": self.config.apikey, "file_metadata": 1 }
self.config.data = urllib.urlencode(self.config.params)
self.report = None
else:
return None
def __init__(self, filename):
if is_file(filename):
self.config = objdict({})
self.config.filename = filename
# Privileges: public key, Request rate: 4 requests/minute, Daily quota: 5760 requests/day, Monthly quota: 178560 requests/month
self.config.apikey = "2cfed5c8ea3e69b1f68a00a083de7f3cdf4de1ea14a317bc5cd3a332493469da"
self.config.apikey = "9ca790fe3dde490e8fbb5190aa2b2b2ab2406f31e174eb51c37f74a8f88ef1a6"
self.config.url = objdict({})
self.config.url.filereport = "https://www.virustotal.com/vtapi/v2/file/report"
self.config.params = { "resource": file_hashes(self.config.filename, "sha256"), "apikey": self.config.apikey }
self.config.data = urllib.urlencode(self.config.params)
self.report = None
else:
return None
except Exception as e:
print('Failed to delete %s. Reason: %s' % (file_path, e))
if __name__ == "__main__":
# Reading command line args
model_num = sys.argv[1]
output_folder = sys.argv[2]
# Creating model, loading checkpoint and creating output folder
checkpoint_path = '/shared/abhinav.goyal/s2t/rnnt_data/ckpt-' + str(
model_num)
model, _ = create_model('rnnt',
objdict({
'batch_size': 2,
'decoding': False
}),
build=True)
checkpoint.load_checkpoint(checkpoint_path, model)
create_folder(output_folder)
clean_folder(output_folder)
# Extracting weights
prednet_wt_names = {'embedding': [], 'lstm_10': [], 'lstm_11': []}
jointnet_wt_names = {'dense': [], 'dense_1': []}
for weight in model.weights:
name = weight.name
for prednet_wt_name in prednet_wt_names:
if name.startswith(prednet_wt_name):
fname = os.path.join(output_folder,
'pred_net_' + name.replace('/', '_'))
def lookup_file(self):
req = urllib2.Request(self.config.url.filereport, self.config.data)
response = urllib2.urlopen(req)
if response:
self.report = objdict({})
self.report.filereport = json.loads(response.read())
def analyze(self):
self.report.filebasename = file_basename(self.config.filename)
self.report.filedirname = file_dirname(self.config.filename)
self.report.filemimetype = file_mimetype(self.config.filename)
magicresult = file_magic(self.config.filename)
self.report.filemagic = "%s (%s)" % (magicresult["match"]["longname"], magicresult["match"]["shortname"]) if magicresult["match"] else None
self.report.hashes.crc32 = file_hashes(self.config.filename, 'crc32')
self.report.hashes.md5 = file_hashes(self.config.filename, 'md5')
self.report.hashes.sha1 = file_hashes(self.config.filename, 'sha1')
self.report.hashes.sha256 = file_hashes(self.config.filename, 'sha256')
self.report.hashes.sha512 = file_hashes(self.config.filename, 'sha512')
self.report.hashes.ssdeep = file_hashes(self.config.filename, 'ssdeep')
with nostdout():
self.report.subfiles = file_subfiles(self.config.filename)
# this might take some time to finish
# based on the filesize, runtime might increase
# will be autodisabled based on statsfilesizelimit config option
if self.config.enableentropycompressionstats:
stats = objdict(file_entropy_compression_stats(self.config.filename))
self.report.filesize = stats.filesizeinbytes
self.report.fileminsize = float(stats.minfilesize)
self.report.filecompressionratio = float(stats.compressionratio)
self.report.fileentropy = float(stats.entropy)
self.report.fileentropycategory = stats.entropycategory
# this might take some time to finish
# based on the filesize, runtime might increase
# should be autodisabled based on (statsfilesizelimit) config option
if self.config.enablefilevisualization:
self.report.visual.pngrgb = file_to_pngimage(self.config.filename)
self.report.visual.pnggray = file_to_pngimage(self.config.filename, enable_colors=False)
rh = identicon(self.report.hashes.sha256)
self.report.visual.identicon = rh.identicon if rh.success else None
config = Config()
config.x_title = 'Bytes'
config.y_title = 'Frequency'
config.x_scale = .25
config.y_scale = .25
config.width = 900
config.height = 300
config.title_font_size = 9
config.tooltip_font_size = 0
config.tooltip_border_radius = 0
config.no_data_text = ""
config.show_legend = False
config.show_only_major_dots = True
config.human_readable = False
config.show_y_labels = False
config.fill = True
config.style = CleanStyle
bar_chart = pygal.Bar(config)
# if enableentropycompressionstats config option is disabled, stats won't be generated above
# as such we need to explicitly generate, on-demand
if not stats:
stats = objdict(file_entropy_compression_stats(self.config.filename))
bar_chart.add('', stats.bytefreqlist)
self.report.visual.bytefreqhistogram = bar_chart.render(is_unicode=False)
# pygal inserts a copyright symbol in rendered chart output
# need to explicitly clean it before returning
pygalregex = re.compile(r"\xc2\xa9")
self.report.visual.bytefreqhistogram = pygalregex.sub("", self.report.visual.bytefreqhistogram)
else:
self.report.visual.pngrgb = None
self.report.visual.pnggray = None
self.report.visual.identicon = None
self.report.visual.bytefreqhistogram = None
# done with analysis, normalize report and return
self.report = dict_normalize(self.report)