def dump_db(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Dumping database")
dbdump_extra_option = '--pgpass' if 'postgresql' in get_database_engine() else ''
venv_run('python src/manage.py dbdump --destination=data/backup %s' % dbdump_extra_option)
colors.green("Done.")
def gulp(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Starting gulp build")
run("gulp clean")
run("gulp build --production")
colors.green("Done.")
def update_python_tools(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Updating Python tools")
venv_run('easy_install --upgrade pip')
venv_run('pip install --no-input --exists-action=i --use-wheel --upgrade setuptools wheel')
colors.green("Done.")
def restart(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Restarting application group")
run('supervisorctl restart {0}:*'.format(env.project_name))
status()
colors.green("Done.")
def test_install():
with settings(warn_only=True):
local("pip uninstall ares_util --yes")
green("Uninstall OK.")
local("pip install --use-wheel --no-index --find-links dist ares_util")
local("pip uninstall ares_util --yes")
green("Install test OK.")
def check_urls(*args, **kwargs):
logging.basicConfig(level=logging.DEBUG)
for url in env.urls_to_check:
print("Checking `{0}`".format(url))
r = requests.get(url, verify=env.urls_to_check_verify_ssl_certificate)
if r.status_code != 200: abort("HTTP status for `{0}` is `{1}`.".format(url, r.status_code))
colors.green("Done.")
def clean():
try:
shutil.rmtree('django_fab_deployer.egg-info')
shutil.rmtree('build')
shutil.rmtree('dist')
shutil.rmtree('__pycache__')
except WindowsError as e:
red(e.message)
green("Cleaned")
def build():
# pip install docutils
local("python setup.py check --verbose --strict --restructuredtext")
local("python setup.py build")
local("python setup.py sdist")
local("python setup.py bdist_wheel")
local("python setup.py bdist_wininst")
green("Build OK.")
def clean():
try:
shutil.rmtree('ares_util.egg-info')
shutil.rmtree('build')
shutil.rmtree('dist')
shutil.rmtree('__pycache__')
except WindowsError as e:
red(e.message)
green("Cleaned")
def build():
# pip install docutils
local("python setup.py check --verbose --strict --restructuredtext")
local("python setup.py build")
local("python setup.py sdist")
local("python setup.py bdist_wheel")
# local("python setup.py bdist_wininst")
green("Build OK.")
def backup(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Creating backup")
run("mkdir -p data/deployment_backup")
now_time = strftime("%Y-%m-%d_%H.%M.%S", gmtime())
venv_run(
"python src/manage.py dumpdata --format json --all --indent=3 --output data/deployment_backup/%s-dump.json" % now_time)
colors.green("Done.")
def npm(upgrade=False, *args, **kwargs):
with cd(env.deploy_path):
colors.blue("Installing node_modules")
run("npm prune")
run("npm install --no-color --link --no-optional --only=dev --rebuild-bundle=false")
if upgrade:
run("npm update --no-color --link --no-optional --only=dev --rebuild-bundle=false")
colors.green("Done.")
def clean(upgrade=False, *args, **kwargs):
with cd(env.deploy_path):
colors.blue("Cleaning Django project")
venv_run('python src/manage.py clearsessions')
venv_run('python src/manage.py clear_cache')
venv_run('python src/manage.py clean_pyc --optimize --path=src/')
venv_run('python src/manage.py compile_pyc --path=src/')
colors.green("Done.")
def test_install():
with settings(warn_only=True):
local("pip uninstall django_fab_deployer --yes")
green("Uninstall OK.")
local(
"pip install --use-wheel --no-index --find-links dist django_fab_deployer"
)
local("pip uninstall django_fab_deployer --yes")
green("Install test OK.")
def graceful_restart(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Restarting Gunicorn with HUP signal")
run('supervisorctl pid {0}:{0}_gunicorn | xargs kill -s HUP'.format(env.project_name))
if env.celery_enabled:
colors.blue("Restarting Celery with HUP signal")
run('supervisorctl pid {0}:{0}_celeryd | xargs kill -s HUP'.format(env.project_name))
run('supervisorctl pid {0}:{0}_celerybeat | xargs kill -s HUP'.format(env.project_name))
colors.green("Done.")
def check(*args, **kwargs):
colors.blue("Checking local project")
with settings(warn_only=True):
local("git status --porcelain")
local("python src/manage.py check --deploy")
with settings(warn_only=True):
local("python src/manage.py validate_templates")
local("python src/manage.py test --noinput")
colors.green("Done.")
def get_dumps(delete=False, *args, **kwargs):
delete = fab_arg_to_bool(delete)
with cd(env.deploy_path):
colors.blue("Rsyncing local backups with remote")
rsync_project(local_dir='data/',
remote_dir='data/backup',
exclude=['.git*', 'cache*', 'filer_*'],
delete=delete,
upload=False)
colors.green("Done.")
def getCookie(loginUrl):
colors.green('您想要进行cookie录制吗(y/N)')
while True:
finish = raw_input('')
i = finish.lower().lower()
if i == 'n' or i == '\n' or i == '\r' or i=='':
return {}
else:
break
th = threading.Thread(target=start_proxy)
th.start()
if loginUrl == '':
return ''
# recording cookie
try:
webbrowser.open_new(loginUrl)
except Exception as err:
print ''
colors.yellow( '如果您已经完成了cookie录制，请在控制台中输入 Y')
while True:
finish = raw_input('')
if finish.lower() == 'y':
break
else:
finish = ''
# read cookie from cookieFile
#print config.conf
#cookieFileName = config.config.conf['CookieFileName']
# cookieFileName = '/root/cntzapfile.txt'
#print 'Read cookie from file: ',cookieFileName
cookieFileName =config.conf['CookieFileName']
cookieTmp = open(cookieFileName,'r').read()
cookie = {}
try:
if len(cookieTmp.strip()) != 0:
for line in cookieTmp.split(';'):
name,value = line.strip().split('=',1)
cookie[name] = value
print 'cookie is:',cookie
except Exception as err:
pass
print "cookie:"
print cookie
return cookie
def rebuild_staticfiles(*args, **kwargs):
if not confirm('Are you sure you want to rebuild all staticfiles?', default=False):
abort('Deployment cancelled')
with cd(env.deploy_path):
colors.blue("Rebuilding staticfiles")
run("rm -r data/static")
venv_run('python src/manage.py collectstatic --noinput')
run('bower install --config.interactive=false')
gulp()
venv_run('python src/manage.py compress')
colors.green("Done.")
def status(*args, **kwargs):
with cd(env.deploy_path):
colors.blue("Retrieving status")
run('supervisorctl status | grep "{0}"'.format(env.project_name))
watched_services = [
'nginx',
'supervisor',
]
db_engine = get_database_engine()
if 'postgresql' in db_engine:
watched_services.append('postgresql')
elif 'mysql' in db_engine:
watched_services.append('mysql')
for service in watched_services:
run('service {} status'.format(service))
colors.green("Done.")
def test():
local(
"nosetests --with-coverage --cover-package=django_fab_deployer --cover-tests --cover-erase --with-doctest"
)
green("Test OK.")
def publish():
if confirm(u'Really publish?', default=False):
local('twine upload dist/*')
green("Published.")
def publish_insecure():
if confirm(u'Really publish?', default=False):
local('python setup.py sdist upload -r pypi')
local('python setup.py bdist_wheel upload -r pypi')
green("Published.")
def update_package_tools():
local("easy_install -U pip")
local("pip install setuptools --upgrade")
local("pip install wheel --upgrade")
green("Updated.")
def install_requirements():
local("pip install -r requirements.txt --upgrade --use-wheel")
green("Installed || Updated.")
def test():
local("nosetests --with-coverage --cover-package=ares_util --cover-tests --cover-erase --with-doctest")
green("Test OK.")
def pytest():
local("py.test ares_util --doctest-modules --cov=ares_util -v --showlocals")
green("Test OK.")
def update_package_tools():
local("easy_install -U pip")
local("pip install setuptools --upgrade")
local("pip install wheel --upgrade")
green("Updated.")
def install_requirements():
local("pip install -r requirements.txt --upgrade --use-wheel")
green("Installed || Updated.")
def publish():
if confirm(u'Really publish?', default=False):
local('python setup.py sdist upload -r pypi')
local('python setup.py bdist_wheel upload -r pypi')
green("Published.")
def test():
local("nosetests --with-coverage --cover-package=dataviso_sequencer --cover-tests --cover-erase --with-doctest")
green("Tests OK.")
def start(baseUrl,seedUrl):
# clean reffer in reffer.txt
f = open("reffer.txt","w")
f.close()
#seed = Request(base='http://192.168.42.131/dvwa/index.php',url='http://192.168.42.131/dvwa/index.php',method='get')
seed = request.Request(base=baseUrl,url=seedUrl,timeout=config.conf['connTimeout'],query={},method='get')
#seed = request.Request(base='http://192.168.42.132/dvwa/',url='http://192.168.42.132/dvwa/',query={},method='get')
colors.blue( '种子URL： %s\n'%seed._url)
logfileName = create_logfile(seed._url)
cookie = getCookie(seed._url)
# begin crawler
tup = urlparse.urlparse(seed._url)
netloc = tup.netloc # seed url
count = 0
q = Queue.Queue()
bf = bloomFilter.BloomFilter(0.001,100000)
# readreffer from reffer.txt
'''
reffer = readReffer()
reqSet = []
reqSet.append(seed)
reqSet.extend(reffer)
for i in reqSet:
q.put(i)
bf.insert(i._url)
'''
q.put(seed)
bf.insert(seed._url)
nums = config.conf['MaxThread']
pool = ThreadPool(nums)
begin = time.time()
while(not q.empty()):
req = q.get()
req._cookies = cookie
reqs = crawler.crawl(req,tree)
if req._query != {} and is_tree_full(req._url,tree):
#if req._query != {}:
count += 1
print 'URL: ',req._BFUrl,' ', req._source
pool.add_task(startCheck,req,logfileName)
for x in reqs:
if not bf.exist(x._BFUrl):
bf.insert(x._BFUrl)
q.put(x)
pool.destroy()
end = time.time()
f = open(logfileName,'r')
colors.blue('\n扫描结果：\n\n')
x = f.read()
colors.green(x)
colors.blue('\n扫描结果已保存在 "%s"\n\n'%(os.getcwd()+'/'+logfileName)+' 中')
cost = end - begin
print "耗时：%f秒"%cost
print "进行测试的URL数量：",count
f.close()
f = open(logfileName,'a')
f.write(advice())
f.close()
os.system('ps -ef | grep -v grep | grep proxy.py | awk \'{print $2}\'|xargs kill -9')
'''