Exemplo n.º 1
0
def dump_db(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Dumping database")

        dbdump_extra_option = '--pgpass' if 'postgresql' in get_database_engine() else ''
        venv_run('python src/manage.py dbdump --destination=data/backup %s' % dbdump_extra_option)

    colors.green("Done.")
Exemplo n.º 2
0
def gulp(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Starting gulp build")

        run("gulp clean")
        run("gulp build --production")

    colors.green("Done.")
Exemplo n.º 3
0
def update_python_tools(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Updating Python tools")

        venv_run('easy_install --upgrade pip')
        venv_run('pip install --no-input --exists-action=i --use-wheel --upgrade setuptools wheel')

    colors.green("Done.")
Exemplo n.º 4
0
def restart(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Restarting application group")
        run('supervisorctl restart {0}:*'.format(env.project_name))

    status()

    colors.green("Done.")
Exemplo n.º 5
0
def test_install():
    with settings(warn_only=True):
        local("pip uninstall ares_util --yes")
        green("Uninstall OK.")

    local("pip install --use-wheel --no-index --find-links dist ares_util")
    local("pip uninstall ares_util --yes")

    green("Install test OK.")
Exemplo n.º 6
0
def check_urls(*args, **kwargs):
    logging.basicConfig(level=logging.DEBUG)

    for url in env.urls_to_check:
        print("Checking `{0}`".format(url))
        r = requests.get(url, verify=env.urls_to_check_verify_ssl_certificate)
        if r.status_code != 200: abort("HTTP status for `{0}` is `{1}`.".format(url, r.status_code))

    colors.green("Done.")
Exemplo n.º 7
0
def clean():
    try:
        shutil.rmtree('django_fab_deployer.egg-info')
        shutil.rmtree('build')
        shutil.rmtree('dist')
        shutil.rmtree('__pycache__')
    except WindowsError as e:
        red(e.message)

    green("Cleaned")
Exemplo n.º 8
0
def build():
    # pip install docutils
    local("python setup.py check --verbose --strict --restructuredtext")

    local("python setup.py build")
    local("python setup.py sdist")
    local("python setup.py bdist_wheel")
    local("python setup.py bdist_wininst")

    green("Build OK.")
Exemplo n.º 9
0
def clean():
    try:
        shutil.rmtree('ares_util.egg-info')
        shutil.rmtree('build')
        shutil.rmtree('dist')
        shutil.rmtree('__pycache__')
    except WindowsError as e:
        red(e.message)

    green("Cleaned")
Exemplo n.º 10
0
def build():
    # pip install docutils
    local("python setup.py check --verbose --strict --restructuredtext")

    local("python setup.py build")
    local("python setup.py sdist")
    local("python setup.py bdist_wheel")
    # local("python setup.py bdist_wininst")

    green("Build OK.")
Exemplo n.º 11
0
def backup(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Creating backup")

        run("mkdir -p data/deployment_backup")

        now_time = strftime("%Y-%m-%d_%H.%M.%S", gmtime())
        venv_run(
            "python src/manage.py dumpdata --format json --all --indent=3 --output data/deployment_backup/%s-dump.json" % now_time)

    colors.green("Done.")
Exemplo n.º 12
0
def npm(upgrade=False, *args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Installing node_modules")

        run("npm prune")
        run("npm install --no-color --link --no-optional --only=dev --rebuild-bundle=false")

        if upgrade:
            run("npm update --no-color --link --no-optional --only=dev --rebuild-bundle=false")

    colors.green("Done.")
Exemplo n.º 13
0
def clean(upgrade=False, *args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Cleaning Django project")

        venv_run('python src/manage.py clearsessions')
        venv_run('python src/manage.py clear_cache')
        venv_run('python src/manage.py clean_pyc --optimize --path=src/')

        venv_run('python src/manage.py compile_pyc --path=src/')

    colors.green("Done.")
Exemplo n.º 14
0
def test_install():
    with settings(warn_only=True):
        local("pip uninstall django_fab_deployer --yes")
        green("Uninstall OK.")

    local(
        "pip install --use-wheel --no-index --find-links dist django_fab_deployer"
    )
    local("pip uninstall django_fab_deployer --yes")

    green("Install test OK.")
Exemplo n.º 15
0
def graceful_restart(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Restarting Gunicorn with HUP signal")
        run('supervisorctl pid {0}:{0}_gunicorn | xargs kill -s HUP'.format(env.project_name))

        if env.celery_enabled:
            colors.blue("Restarting Celery with HUP signal")

            run('supervisorctl pid {0}:{0}_celeryd | xargs kill -s HUP'.format(env.project_name))
            run('supervisorctl pid {0}:{0}_celerybeat | xargs kill -s HUP'.format(env.project_name))

    colors.green("Done.")
Exemplo n.º 16
0
def check(*args, **kwargs):
    colors.blue("Checking local project")

    with settings(warn_only=True):
        local("git status --porcelain")

    local("python src/manage.py check --deploy")
    with settings(warn_only=True):
        local("python src/manage.py validate_templates")
    local("python src/manage.py test --noinput")

    colors.green("Done.")
Exemplo n.º 17
0
def get_dumps(delete=False, *args, **kwargs):
    delete = fab_arg_to_bool(delete)

    with cd(env.deploy_path):
        colors.blue("Rsyncing local backups with remote")

        rsync_project(local_dir='data/',
                      remote_dir='data/backup',
                      exclude=['.git*', 'cache*', 'filer_*'],
                      delete=delete,
                      upload=False)

    colors.green("Done.")
Exemplo n.º 18
0
def getCookie(loginUrl):
    colors.green('您想要进行cookie录制吗(y/N)')


    while True: 
        finish = raw_input('')
        i = finish.lower().lower() 
        if i == 'n' or i == '\n' or i == '\r' or i=='':
            return  {}
        else:
            break
    th = threading.Thread(target=start_proxy)
    th.start()
    if loginUrl == '':
        return ''
    # recording cookie
    try:
        webbrowser.open_new(loginUrl)
    except Exception as err:
        print ''

    colors.yellow( '如果您已经完成了cookie录制,请在控制台中输入 Y')
    while True: 
        finish = raw_input('')
        if finish.lower() == 'y':
            break
        else:
            finish = ''
    # read cookie from cookieFile
    #print config.conf
    #cookieFileName = config.config.conf['CookieFileName']
    # cookieFileName = '/root/cntzapfile.txt'
    #print 'Read cookie from file: ',cookieFileName
    cookieFileName  =config.conf['CookieFileName']
    cookieTmp = open(cookieFileName,'r').read()
    cookie = {}
    try:
        if len(cookieTmp.strip()) != 0:
            for line in cookieTmp.split(';'):
                name,value = line.strip().split('=',1)
                cookie[name] = value
        print 'cookie is:',cookie
    except Exception as err:
        pass
    print  "cookie:"
    print cookie
    return cookie
Exemplo n.º 19
0
def rebuild_staticfiles(*args, **kwargs):
    if not confirm('Are you sure you want to rebuild all staticfiles?', default=False):
        abort('Deployment cancelled')

    with cd(env.deploy_path):
        colors.blue("Rebuilding staticfiles")

        run("rm -r data/static")

        venv_run('python src/manage.py collectstatic --noinput')
        run('bower install --config.interactive=false')

        gulp()

        venv_run('python src/manage.py compress')

    colors.green("Done.")
Exemplo n.º 20
0
def status(*args, **kwargs):
    with cd(env.deploy_path):
        colors.blue("Retrieving status")

        run('supervisorctl status | grep "{0}"'.format(env.project_name))

        watched_services = [
            'nginx',
            'supervisor',
        ]

        db_engine = get_database_engine()

        if 'postgresql' in db_engine:
            watched_services.append('postgresql')
        elif 'mysql' in db_engine:
            watched_services.append('mysql')

        for service in watched_services:
            run('service {} status'.format(service))

    colors.green("Done.")
Exemplo n.º 21
0
def test():
    local(
        "nosetests --with-coverage --cover-package=django_fab_deployer --cover-tests --cover-erase --with-doctest"
    )

    green("Test OK.")
Exemplo n.º 22
0
def publish():
    if confirm(u'Really publish?', default=False):
        local('twine upload dist/*')

        green("Published.")
Exemplo n.º 23
0
def publish_insecure():
    if confirm(u'Really publish?', default=False):
        local('python setup.py sdist upload -r pypi')
        local('python setup.py bdist_wheel upload -r pypi')

        green("Published.")
Exemplo n.º 24
0
def update_package_tools():
    local("easy_install -U pip")
    local("pip install setuptools --upgrade")
    local("pip install wheel --upgrade")

    green("Updated.")
Exemplo n.º 25
0
def install_requirements():
    local("pip install -r requirements.txt --upgrade --use-wheel")

    green("Installed || Updated.")
Exemplo n.º 26
0
def test():
    local("nosetests --with-coverage --cover-package=ares_util --cover-tests --cover-erase --with-doctest")

    green("Test OK.")
Exemplo n.º 27
0
def pytest():
    local("py.test ares_util --doctest-modules --cov=ares_util -v --showlocals")

    green("Test OK.")
Exemplo n.º 28
0
def update_package_tools():
    local("easy_install -U pip")
    local("pip install setuptools --upgrade")
    local("pip install wheel --upgrade")

    green("Updated.")
Exemplo n.º 29
0
def install_requirements():
    local("pip install -r requirements.txt --upgrade --use-wheel")

    green("Installed || Updated.")
Exemplo n.º 30
0
def publish():
    if confirm(u'Really publish?', default=False):
        local('python setup.py sdist upload -r pypi')
        local('python setup.py bdist_wheel upload -r pypi')

        green("Published.")
Exemplo n.º 31
0
def test():
    local("nosetests --with-coverage --cover-package=dataviso_sequencer --cover-tests --cover-erase --with-doctest")

    green("Tests OK.")
Exemplo n.º 32
0
def start(baseUrl,seedUrl):
    # clean reffer in reffer.txt
    f = open("reffer.txt","w")
    f.close()

    #seed = Request(base='http://192.168.42.131/dvwa/index.php',url='http://192.168.42.131/dvwa/index.php',method='get')
    seed = request.Request(base=baseUrl,url=seedUrl,timeout=config.conf['connTimeout'],query={},method='get')
    #seed = request.Request(base='http://192.168.42.132/dvwa/',url='http://192.168.42.132/dvwa/',query={},method='get')
    colors.blue( '种子URL: %s\n'%seed._url)
    logfileName = create_logfile(seed._url)
    cookie = getCookie(seed._url)
    
    # begin crawler
    tup = urlparse.urlparse(seed._url)
    netloc = tup.netloc # seed url 
    count = 0
    q = Queue.Queue()
    bf = bloomFilter.BloomFilter(0.001,100000)
    # readreffer from reffer.txt
    '''
    reffer = readReffer()
    reqSet = []
    reqSet.append(seed)
    reqSet.extend(reffer)
    for i in reqSet:
        q.put(i)
        bf.insert(i._url)
    '''
    q.put(seed)
    bf.insert(seed._url)

    nums = config.conf['MaxThread']
    pool = ThreadPool(nums)
    begin = time.time()
    while(not q.empty()):
        req = q.get()
        req._cookies = cookie
        reqs = crawler.crawl(req,tree)

        if req._query != {} and is_tree_full(req._url,tree):
        #if req._query != {}:
            count += 1 
            print 'URL: ',req._BFUrl,'  ', req._source
            pool.add_task(startCheck,req,logfileName)
        

        for x in reqs:
            if not bf.exist(x._BFUrl):
                bf.insert(x._BFUrl)
                q.put(x)


    pool.destroy()
    end = time.time()
    
    f = open(logfileName,'r')
    colors.blue('\n扫描结果:\n\n')
    x  = f.read()
    colors.green(x)
    colors.blue('\n扫描结果已保存在 "%s"\n\n'%(os.getcwd()+'/'+logfileName)+' 中')
    cost = end - begin 
    print "耗时:%f秒"%cost
    print "进行测试的URL数量:",count
    f.close()
    f = open(logfileName,'a')
    f.write(advice())
    f.close()
    os.system('ps -ef | grep -v grep | grep proxy.py | awk \'{print $2}\'|xargs kill -9')
    '''