def get_command(command):
"""Return a file system binary command"""
def find_sbin_command(command, exception):
"""Checks if a command is hosted under one of the sbin directories"""
search_paths = ['/usr/local/sbin', '/usr/sbin', '/sbin']
for fullpath in ["%s/%s" % (x, command) for x in search_paths]:
if os.path.exists(fullpath) and os.access(fullpath, os.X_OK):
return sh.Command(fullpath)
raise exception
try:
return sh.__getattr__(command) # pylint: disable=no-member
except sh.CommandNotFound as e:
return find_sbin_command(command, e)
def get_command(command):
"""Return a file system binary command"""
def find_sbin_command(command, exception):
"""Checks if a command is hosted under one of the sbin directories"""
search_paths = ['/usr/local/sbin', '/usr/sbin', '/sbin']
for fullpath in ["%s/%s" % (x, command) for x in search_paths]:
if os.path.exists(fullpath) and os.access(fullpath, os.X_OK):
return sh.Command(fullpath)
raise exception
try:
return sh.__getattr__(command) # pylint: disable=no-member
except sh.CommandNotFound as e:
return find_sbin_command(command, e)
def process_output(line):
print(line)
def bake_echo(name):
def echo(*kwargs):
print("| {} {}".format(name, " ".join(kwargs)))
return "(running {} {})".format(name, " ".join(kwargs))
return echo
for imp in to_import:
name = imp.rpartition("/")[-1]
globals()[name] = sh.__getattr__(imp) if not DEBUG else bake_echo(name)
def lst_to_ctes(*lst):
for cte in lst:
globals()[cte.upper()] = cte
return lst
###############################################################################################
CUDA_CONF = ["CUDA_INC=/usr/local/cuda-5.5.22/include/", "CUDA_LIB=/usr/local/cuda-5.5.22/lib64"]
CONFIGURE_OPT = {
OPENCL: ["--with-opencl"],
CUDA: ["--with-cuda=cuda5"] + CUDA_CONF,
def httrack_download_website(url, path, PATH_HTTRACK=None):
'''url is resolved by requests module (http redirection is not shown to user
path - must exists, and good if it's empty
'''
path = str(path)
#TODO: catch errors
domain = requests.head(url, timeout=5).url
cmd = [
# === GENERAL
'--path', path, # where to place the files
# === LIMITS
'--depth=1', # depth level
'--ext-depth=1', # depth level for external sites
'-m10485760,2097152', # max size for non html 10MB, html 2MB
'--max-time=70',
'--disable-security-limits',
'--max-rate=5000000', # in bytes/sec = 5MB/s
'--connection-per-second=20', # maximum number of connections/seconds
# === FLOW
'--sockets=40', # multiple connections
# === LINKS
#'--extended-parsing', # read links not only in clear html (in JS) - this is by default
'--near', # get non html files
#'--test', # test all URLs (even forbidden ones)
# === BUILD
# 0 - original structure
# 1 - html in web, images in images
# 99 - random names in web
# 1099 - random names (no web dir)
#'--structure=0', # structure: 0-original, 1+ ??
#'-or', '-N "%h%p/%n%q.%t' # own structure?
'--structure=99',
'--keep-links=4', # keep original links
# it seams --replace-external does not much
#'--replace-external', # replace external links with errors, may be good, TEST WHETHER THIS NOT REMOVE THIS LINKS
'--include-query-string', # TEST: where this add this?
'--generate-errors=0', # TEST: I think errors we can read from log
# === SPIDER
'--robots=0', # not follow robots.txt
'--keep-alive',
# === BROWSER ID
'--user-agent=', # empty is ok, sometimes when passen unknown, servers serve mobile site
'--referer=http://webcheck.me',
'--footer=', # do not add anything to files
# === LOG, INDEX, CACHE - probably we do not need logs (only new.txt in hts-cache
#'--extra-log',
#'--debug-log', # TEST what is there
#'--file-log', # log in files?
#'--single-log', # one log
'-I0', # '--index=0' - this does not work, # do not make an index
# === EXPERT
#'--debug-headers', # TEST: do we need this? - not very useful
# === GURU
#'--debug-xfrstats', # generate ops log every minute - not very useful
#'--debug-ratestats', # generate rate stats - not very useful
domain, # site to scan
'+*', # which files to get
]
if not PATH_HTTRACK:
httrack = sh.httrack
else:
httrack = sh.__getattr__(PATH_HTTRACK)
httrack(*cmd)
#! /usr/bin/env python
# -*- encoding: utf-8 -*-
import os
from hashlib import sha1
import sh
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from django.template import Template, Context
from scanner.plugins.plugin import PluginMixin
from scanner.models import STATUS, RESULT_STATUS, RESULT_GROUP
yui_compressor = sh.__getattr__('yui-compressor')
class PluginOptiYUI(PluginMixin):
name = unicode(_('OptiYUI'))
wait_for_download = True
OPTIMIZED_CSS_DIR_NAME = getattr(
settings, 'WEBSCANNER_OPTIYUI_OPTIMIZED_CSS_DIR_NAME', 'optimized_css')
OPTIMIZED_JS_DIR_NAME = getattr(
settings, 'WEBSCANNER_OPTIYUI_OPTIMIZED_JS_DIR_NAME', 'optimized_js')
def run(self, command):
from scanner.models import Results
if not command.test.check_performance:
return
def process_output(line):
print(line)
def bake_echo(name):
def echo(*kwargs):
print("| {} {}".format(name, " ".join(kwargs)))
return "(running {} {})".format(name, " ".join(kwargs))
return echo
for imp in to_import:
name = imp.rpartition("/")[-1]
globals()[name] = sh.__getattr__(imp) if not DEBUG else bake_echo(name)
def lst_to_ctes(*lst):
for cte in lst:
globals()[cte.upper()] = cte
return lst
###############################################################################################
CUDA_CONF = [
"CUDA_INC=/usr/local/cuda-5.5.22/include/",
"CUDA_LIB=/usr/local/cuda-5.5.22/lib64"
]
