def client_pool(pool):
    global g_send_count, g_client_num, g_done_client, server_addr, global_lock
    global data
    global g_round
    conn = None
    try:
        for i in xrange(0, g_round):
            conn = pool.get_conn(server_addr)
            if conn == None:
                print "get_conn failed"
                return
            send_all(conn.sock, data)
            _data = recv_all(conn.sock, len(data))
            global_lock.acquire()
            g_send_count += 1
            global_lock.release()
            pool.put_back(conn)
        global_lock.acquire()
        g_done_client += 1
        global_lock.release()
        print "client done", g_done_client
    except Exception, e:
        getLogger("client").exception_ex("client: " + str(e))
        print "client: ", str(e)
        if conn:
            pool.put_back(conn, True)
            print "put_back conn , is_err=True"
        return
Exemple #2
0
 def setUp(self):
     self.server = SSLSocketEngineTest(getLogger("server"))
     self.client = SSLSocketEngineTest(getLogger("client"))
     self.server.start_server(self.server_addr,
                              readable_cb=echo_server_non_block,
                              readable_cb_args=(self.server.engine, ))
     self.client.start_client()
     print "[start]", str(self)
 def _handler(conn):
     try:
         buf = yield conn.read(len(data))
         yield conn.write(buf)
         server.watch_conn(conn)
     except PeerCloseError:
         pass
     except Exception, e:
         getLogger("server").exception(e)
         print e
 def server_handler(conn):
     sock = conn.sock
     try:
         _data = recv_all(sock, len(data))
         send_all(sock, _data)
         server.watch_conn(conn)
     except Exception, e:
         print "server handler", str(e)
         getLogger("server").exception(str(e))
         server.close_conn(conn)
         return False
 def _client():
     _data = None
     try:
         engine = self.client.engine
         print "connecting"
         conn = yield engine.connect_coro(self.server_addr)
         print "connected"
         yield conn.write(self.data)
         _data = yield conn.read(len(self.data))
         conn.close()
     except Exception, e:
         getLogger("client").exception(e)
         self.fail(e)
         return
def echo_server_coro(conn, engine):
#    print "echo"
    try:
        temp, eof = engine.read_avail(conn, 1024)
        if temp:
#            print "serverside write", len(temp)
            yield conn.write(temp)
        elif eof:
#            print "serverside close"
            getLogger("server").info("client close")
            conn.close()
            return
        engine.watch_conn(conn)
    except Exception, e:
        getLogger("server").exception(conn.error)
        conn.close()
def start_echo_server():
    global server_addr
    poll = io_poll.get_poll()
    print str(poll)
    server = CoroSocketEngine(poll, is_blocking=False)
    server.set_logger(getLogger("server"))

    def _on_readable(conn):
        buf, eof = conn.read_avail(4096)
        if buf:
            #print "write", len(buf)
            yield conn.write(buf)
        if eof:
            conn.close()
        else:
            conn.watch()
        return
    server.listen_addr(server_addr, readable_cb=_on_readable)

    def _run(_server):
        while True:
            try:
                _server.poll()
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        return
def start_unblock_server(poll=None):
    global server_addr
    if not poll:
        if 'EPoll' in dir(iopoll):
            poll = iopoll.EPoll(True)
        else:
            poll = iopoll.Poll()
    server = TCPSocketEngine(poll, is_blocking=False, debug=False)
    server.set_logger(getLogger("server"))
#    server.get_time = tc.time
    
    print "starting unblock server with", str(poll)
    def _on_recv(conn):
        #print "on_recv"
#        server.remove_conn(conn)
        server.watch_conn(conn)
        server.write_unblock(conn, conn.get_readbuf(), None, None)
        return
    server.listen_addr(server_addr, server.read_unblock, (len(data), _on_recv, None))

    def _run(_server):
        while True:
            try:
                _server.poll()
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        return
def start_echo_line_server():
    global server_addr
    poll = None
    if 'EPoll' in dir(iopoll):
        poll = iopoll.EPoll(True)
        print "using epoll et mode"
    else:
        poll = iopoll.Poll()
    server = TCPSocketEngine(poll, is_blocking=False, debug=False)
    server.set_logger(getLogger("server"))
#    server.get_time = tc.time

    def _on_recv(conn):
        server.watch_conn(conn)
        server.write_unblock(conn, conn.get_readbuf(), None, None)
        return
    server.listen_addr(server_addr, server.readline_unblock, (MAX_LINE_LEN, _on_recv, None))

    def _run(_server):
        while True:
            try:
                _server.poll()
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        return
def test_client_line():
    poll = None
    if 'EPoll' in dir(iopoll):
        poll = iopoll.EPoll(True)
        print "client using epoll et mode"
    else:
        poll = iopoll.Poll()
    engine = TCPSocketEngine(poll, debug=False)
#    engine.get_time = tc.time
    engine.set_logger(getLogger("client"))
    start_time = time.time()
    def __on_conn_err(e, client_id):
        print client_id, "connect error", str(e)
        os._exit(1)
        return
    def __on_err(conn, client_id, count, *args):
        print client_id, "error", str(conn.error), count
        return
    def __on_recv(conn, client_id, count, data):
        global g_done_client
        if count >= 0 and data:
            buf = conn.get_readbuf()
            if buf != data:
                print "data recv invalid, client:%s, count:%s, data:[%s]" % (client_id, count, buf)
                os._exit(0)
        if count < g_round:
            #print client_id, count
            l = random.randint(1, MAX_LEN -1)
            newdata = random_string(l)
            engine.write_unblock(conn, newdata, __on_send, __on_err, (client_id, count + 1, newdata))
        else:
            engine.close_conn(conn)
            g_done_client += 1
            print "client", client_id, "done"
            if g_done_client == g_client_num:
                print "test client done time: ", time.time() - start_time
                os._exit(0)
        return
    def __on_send( conn, client_id, count, data):
#        print "send", client_id, count, "len", len(data)
        engine.read_unblock(conn, len(data), __on_recv, __on_err, (client_id, count, data))
        return
    def __on_conn(sock, client_id):
#        print "conn", client_id, time.time()
        __on_recv(Connection(sock), client_id, -1, None)
        return
    def _run(engine):
        global g_done_client
        while g_done_client < g_client_num:
            try:
                engine.poll()
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        print g_done_client
        return
Exemple #11
0
def echo_server_non_block (conn, engine):
#    print "echo"
    try:
        temp = conn.sock.recv (1024)
        if temp:
#            print "serverside write", len(temp)
            engine.write_unblock (conn, temp, echo_server_non_block, __on_echo_server_error, (engine,))
        else:
#            print "serverside close"
            getLogger ("server").info ("client close")
            engine.close_conn (conn)
    except (socket.error), e:
        if e[0] == errno.EAGAIN:
#            print "serverside watch"
            engine.watch_conn (conn)
        else:
#            print "serverside close"
            __on_echo_server_error (conn, e)
            engine.close_conn (conn)
def test_client_unblock():
    poll = None
    if 'EPoll' in dir(iopoll):
        poll = iopoll.EPoll(True)
        print "client using epoll et mode"
    else:
        poll = iopoll.Poll()
    engine = SSLSocketEngine(poll, SSL_CERT, debug=True)
#    engine.get_time = tc.time
    engine.set_logger(getLogger("client"))
    start_time = time.time()
    def __on_conn_err(e, client_id):
        print "client", client_id, "connect error", str(e)
        os._exit(1)
        return
    def __on_err(conn, client_id, count):
        print client_id, count, type(conn.error), conn.error
        raise conn.error
        return
    def __on_recv(conn, client_id, count):
#        print count
        global g_done_client
        if count >= 0:
            buf = conn.get_readbuf()
            if buf != data:
                print "data recv invalid, client:", client, "data:", buf
                os._exit(0)
        if count < round:
            print "send", client_id, count + 1, time.time()
            engine.write_unblock(conn, data, __on_send, __on_err, (client_id, count + 1))
        else:
            engine.close_conn(conn)
            g_done_client += 1
            print "client", client_id, "done", time.time()
            if g_done_client == g_client_num:
                print "test client done time: ", time.time() - start_time
        return
    def __on_send( conn, client_id, count):
        engine.read_unblock(conn, len(data), __on_recv, __on_err, (client_id, count))
        return
    def __on_conn(sock, client_id):
        print "ssl conn", client_id, time.time()
        __on_recv(Connection(sock), client_id, -1)
        return
    def _run(engine):
        global g_done_client
        while g_done_client < g_client_num:
            try:
                engine.poll(1)
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        print g_done_client
        return
Exemple #13
0
def echo_server_non_block(conn, engine):
    #    print "echo"
    try:
        temp = conn.sock.recv(1024)
        if temp:
            #            print "serverside write", len(temp)
            engine.write_unblock(conn, temp, echo_server_non_block,
                                 __on_echo_server_error, (engine, ))
        else:
            #            print "serverside close"
            getLogger("server").info("client close")
            engine.close_conn(conn)
    except (socket.error, ssl.SSLError), e:
        if e[0] == errno.EAGAIN or e[0] == ssl.SSL_ERROR_WANT_READ:
            #            print "serverside watch"
            engine.watch_conn(conn)
        else:
            #            print "serverside close"
            __on_echo_server_error(conn, e)
            engine.close_conn(conn)
def start_block_server():
    global server_addr
    server = TCPSocketEngine(iopoll.Poll(), is_blocking=True, debug=False)
    server.set_logger(getLogger("server"))

    def server_handler(conn):
        sock = conn.sock
        try:
            _data = recv_all(sock, len(data))
            send_all(sock, _data)
            server.watch_conn(conn)
        except Exception, e:
            print "server handler", str(e)
            getLogger("server").exception(str(e))
            server.close_conn(conn)
            return False
def start_coro_server(poll=None):
    global server_addr
    poll = iopoll.get_poll()
    server = CoroSocketEngine(poll, is_blocking=False, debug=False)
    server.set_logger(getLogger("server"))
#    server.get_time = tc.time
    print "starting unblock server with", str(poll)
    def _handler(conn):
        try:
            buf = yield conn.read(len(data))
            yield conn.write(buf)
            server.watch_conn(conn)
        except PeerCloseError:
            pass
        except Exception, e:
            getLogger("server").exception(e)
            print e
        return
def start_echo_server():
    global server_addr
    poll = None
    if 'EPoll' in dir(iopoll):
        poll = iopoll.EPoll(True)
        print "using epoll et mode"
    else:
        poll = iopoll.Poll()
    server = TCPSocketEngine(poll, is_blocking=False, debug=False)
    server.set_logger(getLogger("server"))
#    server.get_time = tc.time

    def _on_readable(conn):
        def __on_send(conn):
            #print "write_ok"
            #_on_readable(conn)
            server.watch_conn(conn)
            return
        buf, eof = server.read_avail(conn, 4096)
        if buf:
            #print "write", len(buf)
            server.write_unblock(conn, buf, __on_send, None)
        elif eof:
            server.close_conn(conn)
        else:
            server.watch_conn(conn)
        return
    server.listen_addr(server_addr, readable_cb=_on_readable)

    def _run(_server):
        while True:
            try:
                _server.poll()
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        return
def start_unblock_server_ssl():
    global server_addr
    poll = None
    if 'EPoll' in dir(iopoll):
        poll = iopoll.EPoll(True)
        print "using epoll et mode"
    else:
        poll = iopoll.Poll()
    #server = SSLSocketEngine(poll, SSL_CERT, is_blocking=False, debug=True)
    server = TCPSocketEngine(poll, is_blocking=False, debug=True)
    patch_ssl_engine(server, SSL_CERT)

    server.set_logger(getLogger("server"))
#    server.get_time = tc.time

    def _on_err(conn):
        raise conn.error

    def _on_send(conn):
        #print "on send"
        server.watch_conn(conn)
        return
    def _on_recv(conn):
        #print "on_recv"
        server.remove_conn(conn)
        server.write_unblock(conn, conn.get_readbuf(), _on_send, _on_err)
        return
    server.listen_addr_ssl(server_addr, server.read_unblock, (len(data), _on_recv, None))

    def _run(_server):
        while True:
            try:
                _server.poll(1)
            except Exception, e:
                traceback.print_exc()
                os._exit(1)
        return
            global_lock.acquire()
            g_done_client += 1
            print "client done", g_done_client
            if g_done_client == g_client_num:
                print "time:", time.time() - start_time
                global_lock.release()
                os._exit(0)
            global_lock.release()
        except socket.error, e:
            engine.logger.exception(e)
            print e
            os._exit(1)
        return

    engine = CoroSocketEngine(io_poll.get_poll())
    engine.set_logger(getLogger("client"))
    for i in xrange(g_client_num):
        engine.run_coro(client(engine, i, server_addr, g_round))
    while True:
        engine.poll()


def main():
    Log("client", config=conf)
    Log("server", config=conf)
    server = start_echo_server()
    time.sleep(1)
    test_client()

if __name__ == '__main__':
    main()
Exemple #19
0
web_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../edm_web'))
sys.path.append(web_path)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'edm_web.settings')
if django.VERSION >= (1, 7):
    django.setup()

from django.utils import timezone
from django_redis import get_redis_connection
from django.db import connection
from app.trigger.models import TriggerTask, TriggerTaskOne, TriggerAction
from app.trigger.utils import tools
from lib._signal import init_gevent_signal
from lib.log import getLogger

log = getLogger("TriggerCron", logfile='/home/log/edm_web/trigger_cron.log')
redis = get_redis_connection()

######################################################
# 节假日触发器
def holiday_trig():
    actions = TriggerAction.objects.filter(trigger__status='enable', trigger__type='holiday', status='enable')
    maillist_sql = 'SELECT `address` FROM `mm-pool`.`ml_subscriber_%s` WHERE list_id=%d'
    cursor = connection.cursor()
    for action in actions:
        action_time = datetime.datetime.combine(action.con_holiday_date, datetime.datetime.min.time()) + timezone.timedelta(minutes=action.t_action_time)
        print action.id, action_time
        if action_time.date() == datetime.date.today():
            trigger = action.trigger
            maillist_ids = trigger.get_maillist_ids()
            trigger_task, _ = TriggerTask.objects.get_or_create(trigger=trigger, customer=trigger.customer)
Exemple #20
0
import datetime
import redis
from lib.log import getLogger
log = getLogger()

# redis 连接池
# pool = redis.ConnectionPool(host='111.230.151.179', port=6379, decode_responses=True, db=3, password='******')
# r = redis.Redis(connection_pool=pool)
r = redis.Redis(host='111.230.151.179',
                port=6379,
                decode_responses=True,
                db=3,
                password='******')
pipe = r.pipeline(transaction=True)


def get_no_by_day(t):
    pipe.hsetnx('auto_no', t, 0)
    pipe.hincrby('auto_no', t)
    return pipe.execute()


def del_keys(t):
    keys = r.hkeys('auto_no')
    for key in keys:
        if key == t:
            pass
        else:
            pipe.hdel('auto_no', key)
    pipe.execute()
Exemple #21
0
 def setUp(self):
     self.server = SSLSocketEngineTest(getLogger("server"))
     self.client = SSLSocketEngineTest(getLogger("client"))
     self.client.start_client()
     print "[start]", str(self)
Exemple #22
0
def __on_echo_server_error (conn, *args):
    getLogger ("server").error (conn.error)
Exemple #23
0
    django.setup()

from django.utils import timezone
from django_redis import get_redis_connection
from django.db import InterfaceError, DatabaseError
from app.task.models import SendTask
from app.trigger.models import Trigger, TriggerTask, TriggerTaskOne, TriggerAction
from app.trigger.utils import caches
from app.trigger.utils import tools
from app.trigger.utils import orgemail
from app.trigger.utils.smtp import smtpSendEmail
from lib._signal import init_gevent_signal
from lib.dbreconnect import reconnectDB
from lib.log import getLogger

log = getLogger("Trigger")
redis = get_redis_connection()
signal_stop = False


######################################################
# 任务触发器
def _trigtask(data):
    ctime, task_ident, email, link, agent_info = json.loads(data)[:5]
    # log.info(u"[Trig:Task] task_ident: {}, email: {}, link: {}".format(task_ident, email, link))
    actions = caches.getTriggerActions(task_ident)
    if actions:
        log.info(u"[Trig:Task] task_ident: {}, email: {}, link: {}".format(
            task_ident, email, link))
        task_obj = SendTask.objects.get(send_name=task_ident)
        for action in actions:
Exemple #24
0
 def setUp (self):
     self.server = SocketEngineTest (getLogger ("server"))
     self.client = SocketEngineTest (getLogger ("client"))
     self.client.start_client ()
     print "[start]", str (self)
Exemple #25
0
 def setUp (self):
     self.server = SocketEngineTest (getLogger ("server"))
     self.client = SocketEngineTest (getLogger ("client"))
     self.server.start_server (self.server_addr, readable_cb=echo_server_non_block, readable_cb_args=(self.server.engine, ))
     self.client.start_client ()
     print "[start]", str (self)
Exemple #26
0
"""核注清单Spider"""
import json
import time

from spider.utils.mail import error_2_send_email
from spider.utils.spider_base import BaseCls
from lib.log import getLogger

log = getLogger(__name__)


class NemsSpider(BaseCls):

    def __init__(self, *args, **kwargs):
        super(NemsSpider, self).__init__(*args, **kwargs)
        # self.CookieUrl = r'http://sz.singlewindow.cn/dyck/swProxy/emspubserver/sw/ems/invt/Bws/query?ngBasePath=http%3A%2F%2Fsz.singlewindow.cn%3A80%2Fdyck%2FswProxy%2Femspubserver%2F'
        self.CookieUrl = r'http://sz.singlewindow.cn/dyck/swProxy/deskserver/sw/deskIndex?menu_id=bws'
        self.search_post_url = r'http://sz.singlewindow.cn/dyck/swProxy/sasserver/sw/ems/invt/Bws/list'
        self.detial_post_url = r'http://sz.singlewindow.cn/dyck/swProxy/sasserver/sw/ems/invt/Bws/details/{}'
        # self.check_login()

    def get_web_cookie(self):
        while not self.get_login_cookie():
            log.info('登陆失败,1S后重新登陆..')
            time.sleep(1)
        headers = {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'X-Requested-With': 'XMLHttpRequest',
            'Content-Type': 'application/json',
            'Referer': None,
        }
# URL  = 'https://matalgoapi-dev-ev2-b-track.mc-track-csase-dev-ev2-use0-b.p.azurewebsites.net/getResults'
URL = 'http://40.121.207.138:80/getResults'
POST_DATA_TEMPLATE = {
    "SUPPLIER_ID": "313415",
    "SUPPLIER_NAME": "PokerCentral, LLC",
    "SUPPLIER_STREET_ADDRESS_1": "3960 HOWARD HUGHES PKWY STE 500 ",
    "SUPPLIER_CITY": "LAS VEGAS,NV,US",
    "SUPPLIER_ZIP": "",
    "COUNTRY": "US",
    "CORRELATION_ID": "587693d3-8bd9-42e8-a4e8-608ccfd3d232",
    "SUPPLIER_STATE": "NV"
}

# Get 1000 datas

log = getLogger("Test5000")
setup_log = LoggingSetup(name="Test5000",
                         console_level=INFO,
                         file_level=ERROR,
                         daily_file=False)
setup_log.init_logging()


def get_1000_data():
    headers = [
        'RawOrganizationKey', 'CorrelationID', 'EIN', 'TaxCode',
        'IDFromNetwork', 'IDFromBuyer', 'DunsNumber', 'ComplianceVendorID',
        'CompanyNumber', 'OrganizationName', 'CurrentAlternativeLegalName',
        'Branch', 'RegisteredBusinessName', 'Address1', 'Address2', 'City',
        'State', 'Zip', 'Country', 'CompanyGeoLatLong', 'PhoneNumber',
        'FaxNumber', 'CompanyURL', 'DateOfCompanyRegistration',
Exemple #28
0
def __on_echo_server_error(conn, *args):
    getLogger("server").error(conn.error)