def thread_db(obj):
import socket
import time
global db_instance
timeout = 60
socket.setdefaulttimeout(timeout)
last_rows = []
db_instance = obj()
ServerPool.get_instance()
shell.log_shadowsocks_version()
import resource
logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE))
try:
while True:
load_config()
db_instance.load_cfg()
try:
db_instance.push_db_all_user()
rows = db_instance.pull_db_all_user()
if rows:
db_instance.pull_ok = True
db_instance.del_server_out_of_bound_safe(last_rows, rows)
last_rows = rows
except Exception as e:
trace = traceback.format_exc()
logging.error(trace)
#logging.warn('db thread except:%s' % e)
if db_instance.event.wait(get_config().UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive():
break
except KeyboardInterrupt as e:
pass
db_instance.del_servers()
ServerPool.get_instance().stop()
db_instance = None
def thread_db(obj):
import socket
import time
global db_instance
timeout = 60
socket.setdefaulttimeout(timeout)
last_rows = []
db_instance = obj()
ServerPool.get_instance()
shell.log_shadowsocks_version()
try:
import resource
logging.info('current process RLIMIT_NOFILE resource: soft %d hard %d' % resource.getrlimit(resource.RLIMIT_NOFILE))
except:
pass
try:
while True:
load_config()
db_instance.load_cfg()
try:
db_instance.push_db_all_user()
rows = db_instance.pull_db_all_user()
if rows:
db_instance.pull_ok = True
config = shell.get_config(False)
for port in config['additional_ports']:
val = config['additional_ports'][port]
val['port'] = int(port)
val['enable'] = 1
val['transfer_enable'] = 1024 ** 7
val['u'] = 0
val['d'] = 0
if "password" in val:
val["passwd"] = val["password"]
rows.append(val)
db_instance.del_server_out_of_bound_safe(last_rows, rows)
last_rows = rows
except Exception as e:
trace = traceback.format_exc()
logging.error(trace)
#logging.warn('db thread except:%s' % e)
if db_instance.event.wait(get_config().UPDATE_TIME) or not ServerPool.get_instance().thread.is_alive():
break
except KeyboardInterrupt as e:
pass
db_instance.del_servers()
ServerPool.get_instance().stop()
db_instance = None
def get_instance():
"""Return an instance of Client."""
config = load_config('client')
user_agents = config['user-agents']
proxies = config['proxies']
if len(user_agents) > 0:
user_agent = user_agents[
random.randint(0, len(user_agents) - 1)
]
else:
user_agent = DEFAULT_UA
if len(proxies) > 0:
proxy = proxies[
random.randint(0, len(proxies) - 1)
]
else:
proxy = None
try:
instance = _instances[user_agent]
except KeyError:
instance = Client(user_agent, proxy)
_instances[user_agent] = instance
return instance
# -*- coding: utf-8 -*-
"""HTTP client for Parker."""
import random
import requests
from configloader import load_config
DEFAULT_UA = "Parker v0.1.0"
_PERMITTED_STATUS_CODES = [200]
_instances = dict()
_config = load_config('client')
def get_proxy():
"""Return a random proxy from proxy config."""
proxies = _config['proxies']
return proxies[
random.randint(0, len(proxies) - 1)
] if len(proxies) > 0 else None
def get_instance():
"""Return an instance of Client."""
global _instances
user_agents = _config['user-agents']
user_agent = user_agents[
random.randint(0, len(user_agents) - 1)
] if len(user_agents) > 0 else DEFAULT_UA
# -*- coding: utf-8 -*-
"""RedisSet object for Parker."""
import redis
from configloader import load_config
_instances = dict()
_redis_config = load_config("redis")
_redis = redis.StrictRedis(**_redis_config)
_seconds_in_a_day = 60 * 60 * 24
_seconds_in_five_days = 5 * _seconds_in_a_day
def get_instance(key, expire=None):
"""Return an instance of RedisSet."""
global _instances
try:
instance = _instances[key]
except KeyError:
instance = RedisSet(
key,
_redis,
expire=expire
)
_instances[key] = instance
return instance
class RedisSet(object):
# -*- coding: utf-8 -*-
"""File functions for Parker."""
import json
import os
from configloader import load_config
from stringops import generate_chunks
_config = load_config("parker")
VAR_DIR = _config.get(
"storage-directory",
os.path.join(
os.environ.get('PROJECT'),
'var'
)
)
LOG_DIR = os.path.join(VAR_DIR, 'log')
DATA_DIR = os.path.join(VAR_DIR, 'data')
IMG_DIR = os.path.join(VAR_DIR, 'img')
def create_dirs(path):
"""Create all directories in @path."""
try:
os.makedirs(path)
except OSError, error:
if error.errno != 17:
raise
# -*- coding: utf-8 -*-
"""RQ queues for Parker."""
from rq import Queue
from redis import StrictRedis
from configloader import load_config
_redis_config = load_config('redis')
crawl_q = Queue(
'crawl',
connection=StrictRedis(
**_redis_config
)
)
consume_q = Queue(
'consume',
connection=StrictRedis(
**_redis_config
)
)
