def run(self, url):
# URLs not being processing
if not self.__check_urlBlackList(url):
try:
objfile = ObjFile(url)
# Zeitmessung Start
self.task.update({"started_on": datetime.datetime.now()})
# Download the file
objfile = self.__process_url(url)
# Permittet Type and file must be processed
if objfile.file.is_permittedType() and self.isFileToProcess(
objfile):
# PreProcessing e.g. unpacking
objfile = self.__run_preProcessing(objfile)
# Run processing-plugins
results = self.__run_processing(objfile)
if results:
log.debug(results)
# Run Yara on results
yaraHits = self.__runYara(results)
if yaraHits:
results.update(yaraHits)
# Run report-plugins
self.__run_reporting(results, objfile)
elif not objfile.file.is_permittedType():
log.warn(
"url %s does not provide any allowed file type (%s)" %
(url, objfile.file.get_type()))
except urllib2.HTTPError as e:
log.warn("Unable to perform HTTP request (http code=%s)" % e)
except urllib2.URLError as e:
log.warn("Unable to establish connection: %s" % e)
except IOError as e:
log.warn("Unable to establish connection: %s" % e)
except Exception, e:
import traceback
log.warn(traceback.print_exc())
log.warn("Thread(" + self.processName +
") - %s - Error parsing %s" % (e, url))
finally:
url_dl = urllib2.urlopen(request, timeout=30).read()
except urllib2.HTTPError as e:
raise e
except urllib2.URLError as e:
raise e
except Exception, e:
raise IOError("Thread(" + self.processName +
") - %s - Error parsing %s" % (e, url))
finally:
# Removing SOCKS Tor Proxy
socket.socket = originalSocket
log.info("Download from URL %s" % url)
try:
objfile = ObjFile(url)
objfile.set_file_from_stream(url_dl)
except Exception, e:
raise Exception("Thread(" + self.processName +
") - %s - Error create ObjFile %s" % (e, url))
return objfile
def _setScore(self, score):
self.totalScore = self.totalScore + score
self.numberScores = self.numberScores + 1
def _getScoring(self):
if self.numberScores != 0: # division by zero would be a run-time error
average = float(self.totalScore) / self.numberScores
average = round(average, 1)
