def thread_db(obj):
		import socket
		import time
		global db_instance
		timeout = 60
		socket.setdefaulttimeout(timeout)
		last_rows = []
		db_instance = obj()
		ServerPool.get_instance()
		shell.log_shadowsocks_version()
		import resource
		logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d'  % resource.getrlimit(resource.RLIMIT_NOFILE))
		try:
			while True:
				load_config()
				db_instance.load_cfg()
				try:
					db_instance.push_db_all_user()
					rows = db_instance.pull_db_all_user()
					if rows:
						db_instance.pull_ok = True
					db_instance.del_server_out_of_bound_safe(last_rows, rows)
					last_rows = rows
				except Exception as e:
					trace = traceback.format_exc()
					logging.error(trace)
					#logging.warn('db thread except:%s' % e)
				if db_instance.event.wait(get_config().UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive():
					break
		except KeyboardInterrupt as e:
			pass
		db_instance.del_servers()
		ServerPool.get_instance().stop()
		db_instance = None
	def thread_db(obj):
		import socket
		import time
		global db_instance
		timeout = 60
		socket.setdefaulttimeout(timeout)
		last_rows = []
		db_instance = obj()
		ServerPool.get_instance()
		shell.log_shadowsocks_version()

		try:
			import resource
			logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d'  % resource.getrlimit(resource.RLIMIT_NOFILE))
		except:
			pass

		try:
			while True:
				load_config()
				db_instance.load_cfg()
				try:
					db_instance.push_db_all_user()
					rows = db_instance.pull_db_all_user()
					if rows:
						db_instance.pull_ok = True
						config = shell.get_config(False)
						for port in config['additional_ports']:
							val = config['additional_ports'][port]
							val['port'] = int(port)
							val['enable'] = 1
							val['transfer_enable'] = 1024 ** 7
							val['u'] = 0
							val['d'] = 0
							if "password" in val:
								val["passwd"] = val["password"]
							rows.append(val)
					db_instance.del_server_out_of_bound_safe(last_rows, rows)
					last_rows = rows
				except Exception as e:
					trace = traceback.format_exc()
					logging.error(trace)
					#logging.warn('db thread except:%s' % e)
				if db_instance.event.wait(get_config().UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive():
					break
		except KeyboardInterrupt as e:
			pass
		db_instance.del_servers()
		ServerPool.get_instance().stop()
		db_instance = None
Exemple #3
0
def get_instance():
    """Return an instance of Client."""
    config = load_config('client')
    user_agents = config['user-agents']
    proxies = config['proxies']

    if len(user_agents) > 0:
        user_agent = user_agents[
            random.randint(0, len(user_agents) - 1)
        ]
    else:
        user_agent = DEFAULT_UA

    if len(proxies) > 0:
        proxy = proxies[
            random.randint(0, len(proxies) - 1)
        ]
    else:
        proxy = None

    try:
        instance = _instances[user_agent]
    except KeyError:
        instance = Client(user_agent, proxy)
        _instances[user_agent] = instance

    return instance
Exemple #4
0
# -*- coding: utf-8 -*-
"""HTTP client for Parker."""

import random
import requests
from configloader import load_config

DEFAULT_UA = "Parker v0.1.0"
_PERMITTED_STATUS_CODES = [200]
_instances = dict()
_config = load_config('client')


def get_proxy():
    """Return a random proxy from proxy config."""
    proxies = _config['proxies']

    return proxies[
        random.randint(0, len(proxies) - 1)
    ] if len(proxies) > 0 else None


def get_instance():
    """Return an instance of Client."""
    global _instances
    user_agents = _config['user-agents']

    user_agent = user_agents[
        random.randint(0, len(user_agents) - 1)
    ] if len(user_agents) > 0 else DEFAULT_UA
Exemple #5
0
# -*- coding: utf-8 -*-
"""RedisSet object for Parker."""

import redis
from configloader import load_config

_instances = dict()
_redis_config = load_config("redis")
_redis = redis.StrictRedis(**_redis_config)
_seconds_in_a_day = 60 * 60 * 24
_seconds_in_five_days = 5 * _seconds_in_a_day


def get_instance(key, expire=None):
    """Return an instance of RedisSet."""
    global _instances
    try:
        instance = _instances[key]
    except KeyError:
        instance = RedisSet(
            key,
            _redis,
            expire=expire
        )
        _instances[key] = instance

    return instance


class RedisSet(object):
Exemple #6
0
# -*- coding: utf-8 -*-
"""File functions for Parker."""

import json
import os
from configloader import load_config
from stringops import generate_chunks

_config = load_config("parker")

VAR_DIR = _config.get(
    "storage-directory",
    os.path.join(
        os.environ.get('PROJECT'),
        'var'
    )
)
LOG_DIR = os.path.join(VAR_DIR, 'log')
DATA_DIR = os.path.join(VAR_DIR, 'data')
IMG_DIR = os.path.join(VAR_DIR, 'img')


def create_dirs(path):
    """Create all directories in @path."""
    try:
        os.makedirs(path)
    except OSError, error:
        if error.errno != 17:
            raise

Exemple #7
0
# -*- coding: utf-8 -*-
"""RQ queues for Parker."""

from rq import Queue
from redis import StrictRedis
from configloader import load_config

_redis_config = load_config('redis')

crawl_q = Queue(
    'crawl',
    connection=StrictRedis(
        **_redis_config
    )
)
consume_q = Queue(
    'consume',
    connection=StrictRedis(
        **_redis_config
    )
)