def _walk_dir(self, dir, path_out):
try:
for root, dirs, files in os.walk(dir):
for file in files:
file.replace('$', '\$')
if Utils.is_zip(root, file) or \
Utils.is_rar(root, file) or \
Utils.is_tar(root, file):
self._extract_file(root, file)
Utils.remove_file(root, file)
else:
if Utils.is_apk(root, file):
Utils.rename_file(root, path_out, file)
else:
Utils.remove_file(root, file)
except OSError, e:
logging.error("Error walking dir '%s': %s" % (dir, e))
raise OSError
def _download_phishtank(self, path_in, user_agent, url_feed):
logging.debug("Downloading PhishTank feeds...")
try:
request = urllib2.Request(url_feed)
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', user_agent)]
handler = opener.open(request)
data = handler.read()
with open(os.path.join(path_in, 'phishtank.gz'), "wb") as file_gz:
file_gz.write(data)
file_gz = gzip.GzipFile(os.path.join(path_in, 'phishtank.gz'), 'rb')
data = file_gz.read()
file_gz.close()
Utils.remove_file(self._cfg['dir_in'], 'phishtank.gz')
file_txt = file(os.path.join(path_in, 'phishtank.txt'), 'wb')
file_txt.write(data)
file_txt.close()
except Exception, e:
logging.error("Error downloading PhishTank feed '%s': %s" % (url_feed, e))
raise Exception
def _download_cleanmx(self, path_in, user_agent, url_feed):
logging.debug("Downloading CleanMX feeds...")
try:
request = urllib2.Request(url_feed)
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', user_agent)]
handler = opener.open(request)
data = handler.read()
with open(os.path.join(path_in, 'cleanmx.xml'), "wb") as file_xml:
file_xml.write(data)
file_xml = open(os.path.join(path_in, 'cleanmx.xml'), 'r')
data = file_xml.read()
file_xml.close()
Utils.remove_file(self._cfg['dir_in'], 'cleanmx.xml')
file_txt = file(os.path.join(path_in, 'cleanmx.txt'), 'w')
file_txt.write(json.dumps(xmltodict.parse(data)))
file_txt.close()
except Exception, e:
logging.error("Error downloading CleanMX feed '%s': %s" % (url_feed, e))
raise Exception
