def main():
global cfg, logger
cfg = load_config()
log_cfg = cfg['log']
log_file = log_cfg['file']
log_level = log_cfg['level']
logger = mylogger()
logger.init_logger('task-purger', log_level, log_file,
SysLogHandler.LOG_LOCAL1) # For debug
logger.info('Starting...................')
dbpc_cfg = cfg['dbpc']
DBPC_HOST = dbpc_cfg['host']
DBPC_PORT = int(dbpc_cfg['port'])
DBPC_SERVICE = dbpc_cfg['service']
DBPC_COMPONENT = dbpc_cfg['component']
DBPC_INTERVAL = int(dbpc_cfg['interval'])
t_dbpc = dbpc.dbpc(DBPC_HOST, DBPC_PORT,
DBPC_SERVICE, DBPC_COMPONENT,
DBPC_INTERVAL)
t_dbpc.start()
try:
loop()
except KeyboardInterrupt:
pass
logger.info('Stopping...................')
print 'THE END.'
def main():
global cfg, logger, execute_sql
cfg = load_config()
log_cfg = cfg['log']
log_file = log_cfg['file']
log_level = log_cfg['level']
logger = mylogger()
logger.init_logger('task-purger', log_level, log_file,
SysLogHandler.LOG_LOCAL1) # For debug
logger.info('Starting...................')
execute_sql = timeit(execute_sql, 'SQL', logger)
# signal.signal(signal.SIGQUIT, stop_all)
# signal.signal(signal.SIGTERM, stop_all)
dbpc_cfg = cfg['dbpc']
DBPC_HOST = dbpc_cfg['host']
DBPC_PORT = int(dbpc_cfg['port'])
DBPC_SERVICE = dbpc_cfg['service']
DBPC_COMPONENT = dbpc_cfg['component']
DBPC_INTERVAL = int(dbpc_cfg['interval'])
t_dbpc = dbpc.dbpc(DBPC_HOST, DBPC_PORT,
DBPC_SERVICE, DBPC_COMPONENT,
DBPC_INTERVAL)
t_dbpc.start()
try:
loop()
except KeyboardInterrupt:
pass
logger.info('Stopping...................')
print 'THE END.'
def main():
global cfg, logger, flag_queue
# global execute_sql, insert_vddb, _do_insert
cfg = load_config()
logger = mylogger()
p_manager = Manager()
flag_queue = p_manager.Queue()
log_cfg = cfg['log']
log_file = log_cfg['file']
log_level = log_cfg['level']
logger.init_logger('tmp-result-generator', log_level, log_file,
SysLogHandler.LOG_LOCAL1) # For debug
signal.signal(signal.SIGQUIT, stop_all)
signal.signal(signal.SIGTERM, stop_all)
logger.info('==================[Starting....]===================')
db_type = cfg['db-type']
db_info = cfg['db'][db_type]
pool_size_get = cfg['pool-size-get']
pool_size_insert = cfg['pool-size-insert']
dbpc_cfg = cfg['dbpc']
DBPC_HOST = dbpc_cfg['host']
DBPC_PORT = int(dbpc_cfg['port'])
DBPC_SERVICE = dbpc_cfg['service']
DBPC_COMPONENT = dbpc_cfg['component']
DBPC_INTERVAL = int(dbpc_cfg['interval'])
t_dbpc = dbpc(DBPC_HOST, DBPC_PORT,
DBPC_SERVICE, DBPC_COMPONENT,
DBPC_INTERVAL)
t_dbpc.start()
pool_get = Pool(pool_size_get)
for i in range(pool_size_get):
pool_get.apply_async(loop_get, args=(flag_queue, db_info))
pool_get.close()
manager = Manager()
res_queue = manager.Queue(pool_size_insert)
task_queue = manager.Queue(pool_size_insert)
pool_insert = Pool(pool_size_insert)
for i in range(pool_size_insert):
pool_insert.apply_async(do_insert, args=(flag_queue, task_queue, res_queue, db_info))
pool_insert.close()
try:
loop_insert(flag_queue, task_queue, res_queue, db_info)
except KeyboardInterrupt:
pass
stop_all(None, None)
print '<THE END>'
logger.info('======================[THE END]==================')
def main():
global cfg, logger, flag_queue
# global execute_sql, insert_vddb, _do_insert
cfg = load_config()
logger = mylogger()
p_manager = Manager()
flag_queue = p_manager.Queue()
log_cfg = cfg['log']
log_file = log_cfg['file']
log_level = log_cfg['level']
logger.init_logger('tmp-result-generator', log_level, log_file,
SysLogHandler.LOG_LOCAL1) # For debug
signal.signal(signal.SIGQUIT, stop_all)
signal.signal(signal.SIGTERM, stop_all)
logger.info('==================[Starting....]===================')
db_type = cfg['db-type']
db_info = cfg['db'][db_type]
pool_size_get = cfg['pool-size-get']
pool_size_insert = cfg['pool-size-insert']
dbpc_cfg = cfg['dbpc']
DBPC_HOST = dbpc_cfg['host']
DBPC_PORT = int(dbpc_cfg['port'])
DBPC_SERVICE = dbpc_cfg['service']
DBPC_COMPONENT = dbpc_cfg['component']
DBPC_INTERVAL = int(dbpc_cfg['interval'])
t_dbpc = dbpc(DBPC_HOST, DBPC_PORT, DBPC_SERVICE, DBPC_COMPONENT,
DBPC_INTERVAL)
t_dbpc.start()
pool_get = Pool(pool_size_get)
for i in range(pool_size_get):
pool_get.apply_async(loop_get, args=(flag_queue, db_info))
pool_get.close()
manager = Manager()
res_queue = manager.Queue(pool_size_insert)
task_queue = manager.Queue(pool_size_insert)
pool_insert = Pool(pool_size_insert)
for i in range(pool_size_insert):
pool_insert.apply_async(do_insert,
args=(flag_queue, task_queue, res_queue,
db_info))
pool_insert.close()
try:
loop_insert(flag_queue, task_queue, res_queue, db_info)
except KeyboardInterrupt:
pass
stop_all(None, None)
print '<THE END>'
logger.info('======================[THE END]==================')
BROKER_JSONRPC_METHOD = query_broker['jsonrpc_method']
BROKER_EXCHANGE = query_broker['exchange']
BROKER_ROUTING_KEY = query_broker['routing_key']
BROKER_PUB_TIMEOUT = query_broker['publish_timeout']
del query_broker
dbpc_config = config['dbpc']
DBPC_HOST = dbpc_config['host']
DBPC_PORT = dbpc_config['port']
DBPC_SERVICE = dbpc_config['service']
DBPC_INTERVAL = dbpc_config['interval']
del dbpc_config
del config
logger = mylogger()
logger.init_logger('gateway-uploader', LOG_LEVEL, LOG_FILE, SysLogHandler.LOG_LOCAL1) # For debug
qb = QueryBroker(logger, BROKER_ROUTING_KEY, BROKER_EXCHANGE,
BROKER_PUB_TIMEOUT, BROKER_MQ_URL)
DBPC_COMPONENT = 'gateway-uploader'
if __name__ == '__main__':
t_dbpc = dbpc.dbpc(DBPC_HOST, DBPC_PORT,
DBPC_SERVICE, DBPC_COMPONENT,
DBPC_INTERVAL)
t_dbpc.start()
loop_tasks(logger, REDIS_SERVER, REDIS_LIST_NAME,
PICKLE_DIR, PICKLE_EXT, PICKLE_CORRUPT_TIME,
BROKER_ROUTING_KEY, BROKER_EXCHANGE, BROKER_MQ_URL,
from white_deal import *
from kombu import Connection
from logging.handlers import SysLogHandler
import MySQLdb
import statsd
import marisa_trie
from common.mylogger import mylogger
from common import dbpc
from DBUtils.PersistentDB import PersistentDB
from functools import partial
import global_vars as gv
from high2low import *
gv.g_logger= mylogger()
version = '1.0.0.0'
# the major dirs
bin_dir = os.path.dirname(os.path.abspath(__file__))
if os.path.islink(os.getenv('PWD')):
bin_dir = os.getenv('PWD')
run_dir = os.getenv('PWD')
#os.chdir(bin_dir)
def make_db_pool(database, conns):
# custom connecting method, so we can set up connection config immediately
# after connecting to the server
user = database[8:].split(':')[0]
#port = database[8:].split(':')[2].split('/')[0]
urls = (
'/copyrighted', 'Query',
'/identified', 'Query'
)
app = web.application(urls, globals())
app.internalerror = internalerror
# ==============================================================================
# Init things
# ==============================================================================
## Init Logger
logger = mylogger.mylogger()
logger_bill = mylogger.mylogger()
logger_info = mylogger.mylogger()
logger.init_logger('gateway', LOG_LEVEL, LOG_FILE, SysLogHandler.LOG_LOCAL1) # For debug
logger_bill.init_logger('gateway-bill', LOG_LEVEL, LOG_FILE, SysLogHandler.LOG_LOCAL0) # For Bill
logger_info.init_logger('gateway-info', LOG_LEVEL, LOG_FILE, SysLogHandler.LOG_LOCAL2) # For Info
log_normal(logger, {
'action': 'init-logger-ok',
'info': {
'level': LOG_LEVEL,
}
}, LOG_INFO)
IMD_RESULT = wrap_result(DEFAULT_JSONRPC_ID, OVERALL_UNDETECTED_UNCOPYRIGHTED, [])
IMD_RESULT_UNDETECTED = wrap_result(DEFAULT_JSONRPC_ID, OVERALL_ALL_UNDETECTED, [])
result = wrap_result(DEFAULT_JSONRPC_ID, OVERALL_WORKING, [])
web.header('Retry-After', '%d' % RETRY_AFTER)
web.header('X-Processing-Retry', '%d' % int(PROCESSING_RETRY))
return web.OK(result)
urls = ('/copyrighted', 'Query', '/identified', 'Query')
app = web.application(urls, globals())
app.internalerror = internalerror
# ==============================================================================
# Init things
# ==============================================================================
## Init Logger
logger = mylogger.mylogger()
logger_bill = mylogger.mylogger()
logger_info = mylogger.mylogger()
logger.init_logger('gateway', LOG_LEVEL, LOG_FILE,
SysLogHandler.LOG_LOCAL1) # For debug
logger_bill.init_logger('gateway-bill', LOG_LEVEL, LOG_FILE,
SysLogHandler.LOG_LOCAL0) # For Bill
logger_info.init_logger('gateway-info', LOG_LEVEL, LOG_FILE,
SysLogHandler.LOG_LOCAL2) # For Info
log_normal(logger, {
'action': 'init-logger-ok',
'info': {
'level': LOG_LEVEL,
}
}, LOG_INFO)