def client_pool(pool):
global g_send_count, g_client_num, g_done_client, server_addr, global_lock
global data
global g_round
conn = None
try:
for i in xrange(0, g_round):
conn = pool.get_conn(server_addr)
if conn == None:
print "get_conn failed"
return
send_all(conn.sock, data)
_data = recv_all(conn.sock, len(data))
global_lock.acquire()
g_send_count += 1
global_lock.release()
pool.put_back(conn)
global_lock.acquire()
g_done_client += 1
global_lock.release()
print "client done", g_done_client
except Exception, e:
getLogger("client").exception_ex("client: " + str(e))
print "client: ", str(e)
if conn:
pool.put_back(conn, True)
print "put_back conn , is_err=True"
return
def setUp(self):
self.server = SSLSocketEngineTest(getLogger("server"))
self.client = SSLSocketEngineTest(getLogger("client"))
self.server.start_server(self.server_addr,
readable_cb=echo_server_non_block,
readable_cb_args=(self.server.engine, ))
self.client.start_client()
print "[start]", str(self)
def _handler(conn):
try:
buf = yield conn.read(len(data))
yield conn.write(buf)
server.watch_conn(conn)
except PeerCloseError:
pass
except Exception, e:
getLogger("server").exception(e)
print e
def server_handler(conn):
sock = conn.sock
try:
_data = recv_all(sock, len(data))
send_all(sock, _data)
server.watch_conn(conn)
except Exception, e:
print "server handler", str(e)
getLogger("server").exception(str(e))
server.close_conn(conn)
return False
def _client():
_data = None
try:
engine = self.client.engine
print "connecting"
conn = yield engine.connect_coro(self.server_addr)
print "connected"
yield conn.write(self.data)
_data = yield conn.read(len(self.data))
conn.close()
except Exception, e:
getLogger("client").exception(e)
self.fail(e)
return
def echo_server_coro(conn, engine):
# print "echo"
try:
temp, eof = engine.read_avail(conn, 1024)
if temp:
# print "serverside write", len(temp)
yield conn.write(temp)
elif eof:
# print "serverside close"
getLogger("server").info("client close")
conn.close()
return
engine.watch_conn(conn)
except Exception, e:
getLogger("server").exception(conn.error)
conn.close()
def start_echo_server():
global server_addr
poll = io_poll.get_poll()
print str(poll)
server = CoroSocketEngine(poll, is_blocking=False)
server.set_logger(getLogger("server"))
def _on_readable(conn):
buf, eof = conn.read_avail(4096)
if buf:
#print "write", len(buf)
yield conn.write(buf)
if eof:
conn.close()
else:
conn.watch()
return
server.listen_addr(server_addr, readable_cb=_on_readable)
def _run(_server):
while True:
try:
_server.poll()
except Exception, e:
traceback.print_exc()
os._exit(1)
return
def start_unblock_server(poll=None):
global server_addr
if not poll:
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
else:
poll = iopoll.Poll()
server = TCPSocketEngine(poll, is_blocking=False, debug=False)
server.set_logger(getLogger("server"))
# server.get_time = tc.time
print "starting unblock server with", str(poll)
def _on_recv(conn):
#print "on_recv"
# server.remove_conn(conn)
server.watch_conn(conn)
server.write_unblock(conn, conn.get_readbuf(), None, None)
return
server.listen_addr(server_addr, server.read_unblock, (len(data), _on_recv, None))
def _run(_server):
while True:
try:
_server.poll()
except Exception, e:
traceback.print_exc()
os._exit(1)
return
def start_echo_line_server():
global server_addr
poll = None
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
print "using epoll et mode"
else:
poll = iopoll.Poll()
server = TCPSocketEngine(poll, is_blocking=False, debug=False)
server.set_logger(getLogger("server"))
# server.get_time = tc.time
def _on_recv(conn):
server.watch_conn(conn)
server.write_unblock(conn, conn.get_readbuf(), None, None)
return
server.listen_addr(server_addr, server.readline_unblock, (MAX_LINE_LEN, _on_recv, None))
def _run(_server):
while True:
try:
_server.poll()
except Exception, e:
traceback.print_exc()
os._exit(1)
return
def test_client_line():
poll = None
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
print "client using epoll et mode"
else:
poll = iopoll.Poll()
engine = TCPSocketEngine(poll, debug=False)
# engine.get_time = tc.time
engine.set_logger(getLogger("client"))
start_time = time.time()
def __on_conn_err(e, client_id):
print client_id, "connect error", str(e)
os._exit(1)
return
def __on_err(conn, client_id, count, *args):
print client_id, "error", str(conn.error), count
return
def __on_recv(conn, client_id, count, data):
global g_done_client
if count >= 0 and data:
buf = conn.get_readbuf()
if buf != data:
print "data recv invalid, client:%s, count:%s, data:[%s]" % (client_id, count, buf)
os._exit(0)
if count < g_round:
#print client_id, count
l = random.randint(1, MAX_LEN -1)
newdata = random_string(l)
engine.write_unblock(conn, newdata, __on_send, __on_err, (client_id, count + 1, newdata))
else:
engine.close_conn(conn)
g_done_client += 1
print "client", client_id, "done"
if g_done_client == g_client_num:
print "test client done time: ", time.time() - start_time
os._exit(0)
return
def __on_send( conn, client_id, count, data):
# print "send", client_id, count, "len", len(data)
engine.read_unblock(conn, len(data), __on_recv, __on_err, (client_id, count, data))
return
def __on_conn(sock, client_id):
# print "conn", client_id, time.time()
__on_recv(Connection(sock), client_id, -1, None)
return
def _run(engine):
global g_done_client
while g_done_client < g_client_num:
try:
engine.poll()
except Exception, e:
traceback.print_exc()
os._exit(1)
print g_done_client
return
def echo_server_non_block (conn, engine):
# print "echo"
try:
temp = conn.sock.recv (1024)
if temp:
# print "serverside write", len(temp)
engine.write_unblock (conn, temp, echo_server_non_block, __on_echo_server_error, (engine,))
else:
# print "serverside close"
getLogger ("server").info ("client close")
engine.close_conn (conn)
except (socket.error), e:
if e[0] == errno.EAGAIN:
# print "serverside watch"
engine.watch_conn (conn)
else:
# print "serverside close"
__on_echo_server_error (conn, e)
engine.close_conn (conn)
def test_client_unblock():
poll = None
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
print "client using epoll et mode"
else:
poll = iopoll.Poll()
engine = SSLSocketEngine(poll, SSL_CERT, debug=True)
# engine.get_time = tc.time
engine.set_logger(getLogger("client"))
start_time = time.time()
def __on_conn_err(e, client_id):
print "client", client_id, "connect error", str(e)
os._exit(1)
return
def __on_err(conn, client_id, count):
print client_id, count, type(conn.error), conn.error
raise conn.error
return
def __on_recv(conn, client_id, count):
# print count
global g_done_client
if count >= 0:
buf = conn.get_readbuf()
if buf != data:
print "data recv invalid, client:", client, "data:", buf
os._exit(0)
if count < round:
print "send", client_id, count + 1, time.time()
engine.write_unblock(conn, data, __on_send, __on_err, (client_id, count + 1))
else:
engine.close_conn(conn)
g_done_client += 1
print "client", client_id, "done", time.time()
if g_done_client == g_client_num:
print "test client done time: ", time.time() - start_time
return
def __on_send( conn, client_id, count):
engine.read_unblock(conn, len(data), __on_recv, __on_err, (client_id, count))
return
def __on_conn(sock, client_id):
print "ssl conn", client_id, time.time()
__on_recv(Connection(sock), client_id, -1)
return
def _run(engine):
global g_done_client
while g_done_client < g_client_num:
try:
engine.poll(1)
except Exception, e:
traceback.print_exc()
os._exit(1)
print g_done_client
return
def echo_server_non_block(conn, engine):
# print "echo"
try:
temp = conn.sock.recv(1024)
if temp:
# print "serverside write", len(temp)
engine.write_unblock(conn, temp, echo_server_non_block,
__on_echo_server_error, (engine, ))
else:
# print "serverside close"
getLogger("server").info("client close")
engine.close_conn(conn)
except (socket.error, ssl.SSLError), e:
if e[0] == errno.EAGAIN or e[0] == ssl.SSL_ERROR_WANT_READ:
# print "serverside watch"
engine.watch_conn(conn)
else:
# print "serverside close"
__on_echo_server_error(conn, e)
engine.close_conn(conn)
def start_block_server():
global server_addr
server = TCPSocketEngine(iopoll.Poll(), is_blocking=True, debug=False)
server.set_logger(getLogger("server"))
def server_handler(conn):
sock = conn.sock
try:
_data = recv_all(sock, len(data))
send_all(sock, _data)
server.watch_conn(conn)
except Exception, e:
print "server handler", str(e)
getLogger("server").exception(str(e))
server.close_conn(conn)
return False
def start_coro_server(poll=None):
global server_addr
poll = iopoll.get_poll()
server = CoroSocketEngine(poll, is_blocking=False, debug=False)
server.set_logger(getLogger("server"))
# server.get_time = tc.time
print "starting unblock server with", str(poll)
def _handler(conn):
try:
buf = yield conn.read(len(data))
yield conn.write(buf)
server.watch_conn(conn)
except PeerCloseError:
pass
except Exception, e:
getLogger("server").exception(e)
print e
return
def start_echo_server():
global server_addr
poll = None
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
print "using epoll et mode"
else:
poll = iopoll.Poll()
server = TCPSocketEngine(poll, is_blocking=False, debug=False)
server.set_logger(getLogger("server"))
# server.get_time = tc.time
def _on_readable(conn):
def __on_send(conn):
#print "write_ok"
#_on_readable(conn)
server.watch_conn(conn)
return
buf, eof = server.read_avail(conn, 4096)
if buf:
#print "write", len(buf)
server.write_unblock(conn, buf, __on_send, None)
elif eof:
server.close_conn(conn)
else:
server.watch_conn(conn)
return
server.listen_addr(server_addr, readable_cb=_on_readable)
def _run(_server):
while True:
try:
_server.poll()
except Exception, e:
traceback.print_exc()
os._exit(1)
return
def start_unblock_server_ssl():
global server_addr
poll = None
if 'EPoll' in dir(iopoll):
poll = iopoll.EPoll(True)
print "using epoll et mode"
else:
poll = iopoll.Poll()
#server = SSLSocketEngine(poll, SSL_CERT, is_blocking=False, debug=True)
server = TCPSocketEngine(poll, is_blocking=False, debug=True)
patch_ssl_engine(server, SSL_CERT)
server.set_logger(getLogger("server"))
# server.get_time = tc.time
def _on_err(conn):
raise conn.error
def _on_send(conn):
#print "on send"
server.watch_conn(conn)
return
def _on_recv(conn):
#print "on_recv"
server.remove_conn(conn)
server.write_unblock(conn, conn.get_readbuf(), _on_send, _on_err)
return
server.listen_addr_ssl(server_addr, server.read_unblock, (len(data), _on_recv, None))
def _run(_server):
while True:
try:
_server.poll(1)
except Exception, e:
traceback.print_exc()
os._exit(1)
return
global_lock.acquire()
g_done_client += 1
print "client done", g_done_client
if g_done_client == g_client_num:
print "time:", time.time() - start_time
global_lock.release()
os._exit(0)
global_lock.release()
except socket.error, e:
engine.logger.exception(e)
print e
os._exit(1)
return
engine = CoroSocketEngine(io_poll.get_poll())
engine.set_logger(getLogger("client"))
for i in xrange(g_client_num):
engine.run_coro(client(engine, i, server_addr, g_round))
while True:
engine.poll()
def main():
Log("client", config=conf)
Log("server", config=conf)
server = start_echo_server()
time.sleep(1)
test_client()
if __name__ == '__main__':
main()
web_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../edm_web'))
sys.path.append(web_path)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", 'edm_web.settings')
if django.VERSION >= (1, 7):
django.setup()
from django.utils import timezone
from django_redis import get_redis_connection
from django.db import connection
from app.trigger.models import TriggerTask, TriggerTaskOne, TriggerAction
from app.trigger.utils import tools
from lib._signal import init_gevent_signal
from lib.log import getLogger
log = getLogger("TriggerCron", logfile='/home/log/edm_web/trigger_cron.log')
redis = get_redis_connection()
######################################################
# 节假日触发器
def holiday_trig():
actions = TriggerAction.objects.filter(trigger__status='enable', trigger__type='holiday', status='enable')
maillist_sql = 'SELECT `address` FROM `mm-pool`.`ml_subscriber_%s` WHERE list_id=%d'
cursor = connection.cursor()
for action in actions:
action_time = datetime.datetime.combine(action.con_holiday_date, datetime.datetime.min.time()) + timezone.timedelta(minutes=action.t_action_time)
print action.id, action_time
if action_time.date() == datetime.date.today():
trigger = action.trigger
maillist_ids = trigger.get_maillist_ids()
trigger_task, _ = TriggerTask.objects.get_or_create(trigger=trigger, customer=trigger.customer)
import datetime
import redis
from lib.log import getLogger
log = getLogger()
# redis 连接池
# pool = redis.ConnectionPool(host='111.230.151.179', port=6379, decode_responses=True, db=3, password='******')
# r = redis.Redis(connection_pool=pool)
r = redis.Redis(host='111.230.151.179',
port=6379,
decode_responses=True,
db=3,
password='******')
pipe = r.pipeline(transaction=True)
def get_no_by_day(t):
pipe.hsetnx('auto_no', t, 0)
pipe.hincrby('auto_no', t)
return pipe.execute()
def del_keys(t):
keys = r.hkeys('auto_no')
for key in keys:
if key == t:
pass
else:
pipe.hdel('auto_no', key)
pipe.execute()
def setUp(self):
self.server = SSLSocketEngineTest(getLogger("server"))
self.client = SSLSocketEngineTest(getLogger("client"))
self.client.start_client()
print "[start]", str(self)
def __on_echo_server_error (conn, *args):
getLogger ("server").error (conn.error)
django.setup()
from django.utils import timezone
from django_redis import get_redis_connection
from django.db import InterfaceError, DatabaseError
from app.task.models import SendTask
from app.trigger.models import Trigger, TriggerTask, TriggerTaskOne, TriggerAction
from app.trigger.utils import caches
from app.trigger.utils import tools
from app.trigger.utils import orgemail
from app.trigger.utils.smtp import smtpSendEmail
from lib._signal import init_gevent_signal
from lib.dbreconnect import reconnectDB
from lib.log import getLogger
log = getLogger("Trigger")
redis = get_redis_connection()
signal_stop = False
######################################################
# 任务触发器
def _trigtask(data):
ctime, task_ident, email, link, agent_info = json.loads(data)[:5]
# log.info(u"[Trig:Task] task_ident: {}, email: {}, link: {}".format(task_ident, email, link))
actions = caches.getTriggerActions(task_ident)
if actions:
log.info(u"[Trig:Task] task_ident: {}, email: {}, link: {}".format(
task_ident, email, link))
task_obj = SendTask.objects.get(send_name=task_ident)
for action in actions:
def setUp (self):
self.server = SocketEngineTest (getLogger ("server"))
self.client = SocketEngineTest (getLogger ("client"))
self.client.start_client ()
print "[start]", str (self)
def setUp (self):
self.server = SocketEngineTest (getLogger ("server"))
self.client = SocketEngineTest (getLogger ("client"))
self.server.start_server (self.server_addr, readable_cb=echo_server_non_block, readable_cb_args=(self.server.engine, ))
self.client.start_client ()
print "[start]", str (self)
"""核注清单Spider"""
import json
import time
from spider.utils.mail import error_2_send_email
from spider.utils.spider_base import BaseCls
from lib.log import getLogger
log = getLogger(__name__)
class NemsSpider(BaseCls):
def __init__(self, *args, **kwargs):
super(NemsSpider, self).__init__(*args, **kwargs)
# self.CookieUrl = r'http://sz.singlewindow.cn/dyck/swProxy/emspubserver/sw/ems/invt/Bws/query?ngBasePath=http%3A%2F%2Fsz.singlewindow.cn%3A80%2Fdyck%2FswProxy%2Femspubserver%2F'
self.CookieUrl = r'http://sz.singlewindow.cn/dyck/swProxy/deskserver/sw/deskIndex?menu_id=bws'
self.search_post_url = r'http://sz.singlewindow.cn/dyck/swProxy/sasserver/sw/ems/invt/Bws/list'
self.detial_post_url = r'http://sz.singlewindow.cn/dyck/swProxy/sasserver/sw/ems/invt/Bws/details/{}'
# self.check_login()
def get_web_cookie(self):
while not self.get_login_cookie():
log.info('登陆失败，1S后重新登陆..')
time.sleep(1)
headers = {
'Accept': 'application/json, text/javascript, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
'Content-Type': 'application/json',
'Referer': None,
}
# URL = 'https://matalgoapi-dev-ev2-b-track.mc-track-csase-dev-ev2-use0-b.p.azurewebsites.net/getResults'
URL = 'http://40.121.207.138:80/getResults'
POST_DATA_TEMPLATE = {
"SUPPLIER_ID": "313415",
"SUPPLIER_NAME": "PokerCentral, LLC",
"SUPPLIER_STREET_ADDRESS_1": "3960 HOWARD HUGHES PKWY STE 500 ",
"SUPPLIER_CITY": "LAS VEGAS,NV,US",
"SUPPLIER_ZIP": "",
"COUNTRY": "US",
"CORRELATION_ID": "587693d3-8bd9-42e8-a4e8-608ccfd3d232",
"SUPPLIER_STATE": "NV"
}
# Get 1000 datas
log = getLogger("Test5000")
setup_log = LoggingSetup(name="Test5000",
console_level=INFO,
file_level=ERROR,
daily_file=False)
setup_log.init_logging()
def get_1000_data():
headers = [
'RawOrganizationKey', 'CorrelationID', 'EIN', 'TaxCode',
'IDFromNetwork', 'IDFromBuyer', 'DunsNumber', 'ComplianceVendorID',
'CompanyNumber', 'OrganizationName', 'CurrentAlternativeLegalName',
'Branch', 'RegisteredBusinessName', 'Address1', 'Address2', 'City',
'State', 'Zip', 'Country', 'CompanyGeoLatLong', 'PhoneNumber',
'FaxNumber', 'CompanyURL', 'DateOfCompanyRegistration',
def __on_echo_server_error(conn, *args):
getLogger("server").error(conn.error)
