def test_tree(self):
ee = EventEmitter()
stack = []
@ee.on("max")
def handler1():
stack.append("max_1")
@ee.once("max")
def handler2():
stack.append("max_2")
self.assertEqual(ee.num_listeners, 2)
self.assertEqual(len(ee._event_tree.nodes["max"].listeners), 2)
ee.emit("max")
self.assertTrue(stack[-2] == "max_1")
self.assertTrue(stack[-1] == "max_2")
del stack[:]
ee.emit("max")
self.assertTrue(stack[-1] == "max_1")
del stack[:]
self.assertEqual(ee.num_listeners, 1)
self.assertTrue("max" in ee._event_tree.nodes)
self.assertEqual(len(ee._event_tree.nodes["max"].listeners), 1)
ee.off("max", handler1)
self.assertEqual(ee.num_listeners, 0)
class PyrebaseDatabase(object):
def __init__(self):
with open('pyrebase_config.json') as f:
config = json.load(f)
#print()
self.firebase = pyrebase.initialize_app(config)
self.db = self.firebase.database()
def start(self):
self.ee = EventEmitter()
self.new_data_listener(self.new_data_handler)
self.my_stream = self.db.child("users").child("0").stream(
self.stream_handler)
def stop(self):
print('closing stream to firebase')
self.my_stream.close()
def stream_handler(self, message):
print(message["event"]) # put
print(message["path"]) # /-K7yGTTEp7O549EzTYtI
print(message["data"]) # {'title': 'Pyrebase', "body": "etc..."}
s = json.dumps(message["data"])
self.ee.emit("new_data_event", s)
def new_data_handler(self, args):
print(args)
def new_data_listener(self, func):
self.ee.on("new_data_event", func)
class PyrebaseDatabase(object):
def __init__(self):
with open(
'/home/pi/Documents/american_gothic/american_gothic_1/python/pyrebase_config.json'
) as f:
config = json.load(f)
#print()
self.firebase = pyrebase.initialize_app(config)
self.db = self.firebase.database()
def start(self, node):
self.ee = EventEmitter()
self.new_data_listener(self.new_data_handler)
self.my_stream = self.db.child(node).stream(self.stream_handler)
def stop(self):
print('closing stream to firebase')
self.my_stream.close()
def stream_handler(self, message):
print(message["event"]) # put
print(message["path"]) # /-K7yGTTEp7O549EzTYtI
print(message["data"]) # {'title': 'Pyrebase', "body": "etc..."}
s = json.dumps(message["data"])
self.ee.emit("new_data_event", s)
def new_data_handler(self, args):
print(args)
def new_data_listener(self, func):
self.ee.on("new_data_event", func)
def send_data(self, data):
self.db.child("american_gothic").update({"buffer_1": data})
class Speed():
def __init__(self):
self.sl = SpeedListener()
self.ee = EventEmitter()
self.sl.setEM(self.ee)
self.channel = None
self.running = True
def em(self):
return self.ee
def start(self):
logging.debug('Starting bike events listener')
# Initialize
stick = driver.USB1Driver(SERIAL, log=LOG, debug=DEBUG)
self.antnode = node.Node(stick)
self.antnode.start()
self.antnode.registerEventListener(self.sl)
# Set network key
network = node.Network(key=NETKEY, name='N:ANT+')
self.antnode.setNetworkKey(0, network)
# Get the first unused channel. Returns an instance of the node.Channel class.
self.channel = self.antnode.getFreeChannel()
# Initialize it as a receiving channel using our network key
self.channel.assign(network, CHANNEL_TYPE_TWOWAY_RECEIVE)
# Now set the channel id for pairing with an ANT+ HR monitor
self.channel.setID(123, 0, 0)
# Listen forever and ever (not really, but for a long time)
self.channel.searchTimeout = TIMEOUT_NEVER
# We want a ~4.06 Hz transmission period
self.channel.period = 8118
# And ANT frequency 57
self.channel.frequency = 57
self.evm = event.EventMachine(driver)
self.channel.open()
self.ee.emit("connected", True)
def stop(self):
logging.info("Closing devices finally")
if self.channel is None:
return None
# Shutdown channel
self.channel.close()
self.channel.unassign()
# Shutdown
self.antnode.stop()
def test_on_all(self):
ee = EventEmitter(wildcard=True)
stack = []
@ee.on("on_all.*")
def handler():
stack.append("on_all")
ee.emit("on_all.foo")
self.assertTrue(stack[-1] == "on_all")
def test_async_decorator_usage(self):
ee = EventEmitter()
stack = []
@ee.on("async_decorator_usage")
async def handler(arg):
stack.append("async_decorator_usage_" + arg)
ee.emit("async_decorator_usage", "bar")
self.assertTrue(stack[-1] == "async_decorator_usage_bar")
def test_delimiter(self):
ee = EventEmitter(wildcard=True, delimiter=":")
stack = []
@ee.on("delimiter:*")
def handler():
stack.append("delimiter")
ee.emit("delimiter:foo")
self.assertTrue(stack[-1] == "delimiter")
def test_async_callback_usage(self):
ee = EventEmitter()
stack = []
async def handler(arg):
stack.append("async_callback_usage_" + arg)
ee.on("async_callback_usage", handler)
ee.emit("async_callback_usage", "foo")
self.assertTrue(stack[-1] == "async_callback_usage_foo")
class Main():
""" Python-Main class """
_instance = None
EVENT_RELOAD = "reload"
@classmethod
def i(cls):
if cls._instance is None:
cls._instance = cls()
return cls._instance
#######################################################
def __init__(self):
log("Constructing Bot")
from bot import Bot
self.ee = EventEmitter()
self.config = object() # __postinit
self.igp = [] # __postinit
self.bot = Bot()
self.__pi_done = False
def reload(self):
self.ee.emit(Main.EVENT_RELOAD)
def __postinit(self):
""" Post-init for everything that needs a ready main instance. """
if self.__pi_done:
return
self.config = JSON_File("config/settings.json")
self.igp = []
@self.config.ee.on(JSON_File.EVENT_RELOAD)
def _on_config_reload(cfg):
self.igp = [re.compile(r) for r in cfg["ignore-pattern"]]
self.__pi_done = True
self.reload()
def run(self):
self.__postinit()
from bot_modules import load
load()
self.bot.run(self.config["token"])
def test_ttl_once(self):
ee = EventEmitter()
stack = []
@ee.once("ttl_once")
def handler(arg):
stack.append("ttl_once_" + arg)
ee.emit("ttl_once", "foo")
self.assertTrue(stack[-1] == "ttl_once_foo")
ee.emit("ttl_once", "bar")
self.assertTrue(stack[-1] == "ttl_once_foo")
def test_on_any(self):
ee = EventEmitter()
stack = []
@ee.on("foo")
def handler1():
stack.append("foo")
@ee.on_any()
def handler2():
stack.append("bar")
ee.emit("foo")
self.assertEqual(tuple(stack), ("foo", "bar"))
def test_max(self):
ee = EventEmitter(max_listeners=1)
stack = []
@ee.on("max")
def handler1():
stack.append("max_1")
@ee.on("max")
def handler2():
stack.append("max_2")
ee.emit("max")
self.assertTrue(stack[-1] == "max_1")
def test_off_any(self):
ee = EventEmitter()
stack = []
@ee.on_any
def handler1():
stack.append("foo")
ee.emit("xyz")
self.assertEqual(tuple(stack), ("foo", ))
del stack[:]
ee.off_any(handler1)
ee.emit("xyz")
self.assertEqual(tuple(stack), ())
self.assertEqual(ee.num_listeners, 0)
class AulaActor:
def __init__(self, websocket_manager, rc_service, models):
self.websocket_manager = websocket_manager
self.rc_service = rc_service
self.models = models
self.event_emitter = EventEmitter()
self.bind_events()
def emit_event(self, event):
self.event_emitter.emit(event.get("action"), event)
def bind_events(self):
self.event_emitter.on("aula.add_room", self.on_add_room)
self.event_emitter.on("aula.move_student", self.on_move_student)
self.event_emitter.on_any(func=self.broadcast_to_aula)
def broadcast_to_aula(self, event):
""" Adapts AulaService's messages to the WebsocketManager's broadcast interface"""
active_session_id = event.get("active_session_id")
aula_channel = f"aula-{active_session_id}"
self.websocket_manager.broadcast(event, aula_channel)
def on_add_room(self, event):
room_name = event.get("data").get("room")
slug = event.get("data").get("slug")
channel_name = f"{room_name}-{slug}"
res = self.rc_service.create_channel(channel_name)
LOG.info(f"Result of adding room:", res)
def on_move_student(self, event):
data = event.get("data")
to_channel = data.get("to_room")
from_channel = data.get("from_room")
student_id = data.get("student")
slug = data.get("slug")
from_room_name = f"{from_channel}-{slug}"
to_room_name = f"{to_channel}-{slug}"
user = self.models.User.query.filter_by(id=student_id).one()
remove_res = self.rc_service.remove_user_from_channel(
user.rocketchat_id, channel_name=from_room_name)
add_res = self.rc_service.add_user_to_channel(
user.rocketchat_id, channel_name=to_room_name)
LOG.debug(f"User channel add result: {add_res}")
class InputCommands(object):
def __init__(self, func):
#Setup listeners for input
thread = Thread(target=self.key_listener)
thread.start()
self.ee = EventEmitter()
self.ee.on("exit_event", func)
#Input callbacks
def on_release(self, key):
print('{0} released'.format(key))
if key == keyboard.Key.esc:
self.ee.emit("exit_event")
return False
def key_listener(self):
# Collect events until released
with keyboard.Listener(on_release=self.on_release) as listener:
listener.join()
class MockedSpeed():
def __init__(self):
self.ee = EventEmitter()
self.running = True
def em(self):
return self.ee
def start(self):
logging.debug('Starting bike events listener')
try:
logging.info("Listening for device events...")
while self.running:
self.ee.emit("speed", SpeedEvent(10, 2.3456))
time.sleep(5)
finally:
self.running = False
def stop(self):
self.running = False
def test_on_reverse_pattern(self):
ee = EventEmitter(wildcard=True)
stack = []
@ee.on("foo.bar")
def handler1():
stack.append("on_foo_bar")
@ee.on("foo.baz")
def handler2():
stack.append("on_foo_baz")
@ee.on("foo.bar.baz.test")
def handler3():
stack.append("on_foo_bar_baz_test")
ee.emit("foo.ba?")
self.assertTrue(stack[-2] == "on_foo_bar")
self.assertTrue(stack[-1] == "on_foo_baz")
del stack[:]
ee.emit("foo.bar.*.test")
self.assertTrue(stack[-1] == "on_foo_bar_baz_test")
class AllTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(AllTestCase, self).__init__(*args, **kwargs)
self.ee1 = EventEmitter()
self.ee2 = EventEmitter(wildcard=True)
self.ee3 = EventEmitter(wildcard=True, delimiter=":")
self.ee4 = EventEmitter(new_listener=True)
self.ee5 = EventEmitter(max_listeners=1)
self.stack = []
def test_1_callback_usage(self):
def handler(arg):
self.stack.append("1_callback_usage_" + arg)
self.ee1.on("1_callback_usage", handler)
self.ee1.emit("1_callback_usage", "foo")
self.assertTrue(self.stack[-1] == "1_callback_usage_foo")
def test_1_decorator_usage(self):
@self.ee1.on("1_decorator_usage")
def handler(arg):
self.stack.append("1_decorator_usage_" + arg)
self.ee1.emit("1_decorator_usage", "bar")
self.assertTrue(self.stack[-1] == "1_decorator_usage_bar")
def test_1_ttl(self):
# same as once
@self.ee1.on("1_ttl", ttl=1)
def handler(arg):
self.stack.append("1_ttl_" + arg)
self.ee1.emit("1_ttl", "foo")
self.assertTrue(self.stack[-1] == "1_ttl_foo")
self.ee1.emit("1_ttl", "bar")
self.assertTrue(self.stack[-1] == "1_ttl_foo")
def test_2_on_all(self):
@self.ee2.on("2_on_all.*")
def handler():
self.stack.append("2_on_all")
self.ee2.emit("2_on_all.foo")
self.assertTrue(self.stack[-1] == "2_on_all")
def test_2_emit_all(self):
@self.ee2.on("2_emit_all.foo")
def handler():
self.stack.append("2_emit_all.foo")
self.ee2.emit("2_emit_all.*")
self.assertTrue(self.stack[-1] == "2_emit_all.foo")
def test_3_delimiter(self):
@self.ee3.on("3_delimiter:*")
def handler():
self.stack.append("3_delimiter")
self.ee3.emit("3_delimiter:foo")
self.assertTrue(self.stack[-1] == "3_delimiter")
def test_4_new(self):
@self.ee4.on("new_listener")
def handler(func, event=None):
self.stack.append((func, event))
def newhandler():
pass
self.ee4.on("4_new", newhandler)
self.assertTrue(self.stack[-1] == (newhandler, "4_new"))
def test_5_max(self):
@self.ee5.on("5_max")
def handler1():
self.stack.append("5_max_1")
@self.ee5.on("5_max")
def handler2():
self.stack.append("5_max_2")
self.ee5.emit("5_max")
self.assertTrue(self.stack[-1] == "5_max_1")
class ChannelHandler(threading.Thread):
context = None
CA_File = None
node_crt_file = None
node_key_file = None
ECDH_curve = "secp256k1"
ssock = None
host = None
port = None
request_counter = itertools.count()
logger = None
recvThread = None
sendThread = None
keepWorking = False
socketClosed = False
pushDispacher = None
def __init__(self, max_timeout=10, name="channelHandler"):
self.timeout = max_timeout
threading.Thread.__init__(self)
self.callbackEmitter = EventEmitter()
self.requests = []
self.name = name
self.blockNumber = 0
self.onResponsePrefix = "onResponse"
self.getResultPrefix = "getResult"
self.lock = threading.RLock()
def initTLSContext(self, ca_file, node_crt_file, node_key_file,
protocol=ssl.PROTOCOL_TLSv1_2,
verify_mode=ssl.CERT_REQUIRED):
try:
context = ssl.SSLContext(protocol)
context.check_hostname = False
context.load_verify_locations(ca_file)
context.load_cert_chain(node_crt_file, node_key_file)
# print(context.get_ca_certs())
context.set_ecdh_curve(self.ECDH_curve)
context.verify_mode = verify_mode
self.context = context
except Exception as e:
raise ChannelException(("init ssl context failed,"
" please check the certificatsreason: {}").
format(e))
def __del__(self):
self.finish()
def disconnect(self):
self.lock.acquire()
if self.ssock is not None and self.socketClosed is False:
self.socketClosed = True
self.ssock.shutdown(socket.SHUT_RDWR)
self.ssock.close()
self.logger.info(
"disconnect for read/write error, host: {}, port: {}".format(self.host, self.port))
self.lock.release()
def reconnect(self):
if self.socketClosed is True:
self.lock.acquire()
self.logger.info("reconnect, host: {}, port: {}".format(self.host, self.port))
try:
self.start_connect()
self.socketClosed = False
self.logger.info(
"reconnect success, host: {}, port: {}".format(self.host, self.port))
except Exception as e:
self.logger.error("reconnect failed, error: {}".format(e))
self.lock.release()
def finish(self):
self.disconnect()
self.ssock = None
if self.keepWorking is True:
self.keepWorking = False
self.join(timeout=1)
if self.recvThread is not None:
self.recvThread.finish()
self.recvThread.join(timeout=2)
if self.sendThread is not None:
self.sendThread.finish()
self.sendThread.join(timeout=2)
if self.pushDispacher is not None:
self.pushDispacher.finish()
def run(self):
try:
self.keepWorking = True
self.logger.debug(self.name + ":start thread-->")
while self.keepWorking:
try:
# try to reconnect
self.reconnect()
responsepack = self.recvThread.recvQueue.get_nowait() # pop msg from queue
if responsepack is None and self.keepWorking:
time.sleep(0.001)
continue
emitter_str = ChannelHandler.getEmitterStr(self.onResponsePrefix,
responsepack.seq, responsepack.type)
if emitter_str in self.requests:
self.lock.acquire()
self.callbackEmitter.emit(emitter_str, responsepack)
self.lock.release()
else:
# 并非客户端指定的等待接受的来自节点的包，可能是push来的消息，包括amop，event push等
# 独立接口处理
# print("push type ",hex(responsepack.type) )
self.pushDispacher.push(responsepack)
except Empty:
time.sleep(0.001)
except Exception as e:
self.logger.error("{} recv error {}".format(self.name, e))
finally:
self.logger.debug("{}:thread finished ,keepWorking = {}".format(
self.name, self.keepWorking))
def start_connect(self):
sock = socket.create_connection((self.host, self.port))
self.logger.debug("connect {}:{},as socket {}".format(self.host, self.port, sock))
# 将socket打包成SSL socket
self.ssock = self.context.wrap_socket(sock)
def start_channel(self, host, port):
try:
self.socketClosed = False
self.keepWorking = False
self.host = host
self.port = port
self.start_connect()
self.recvThread = ChannelRecvThread(self)
self.recvThread.start()
self.sendThread = ChannelSendThread(self)
self.sendThread.start()
self.pushDispacher = ChannelPushDispatcher()
self.pushDispacher.start()
super().start()
except Exception as e:
raise ChannelException(("start channelHandler Failed for {},"
" host: {}, port: {}").format(e,
self.host, self.port))
def decode_rpc_response(self, response):
text_response = to_text(response)
return FriendlyJsonSerde().json_decode(text_response)
def encode_rpc_request(self, method, params):
rpc_dict = {
"jsonrpc": "2.0",
"method": method,
"params": params or [],
"id": next(self.request_counter),
}
encoded = FriendlyJsonSerde().json_encode(rpc_dict)
return to_bytes(text=encoded)
'''
result:
0 成功
100 节点不可达
101 SDK不可达
102 超时
'''
errorMsg = dict()
errorMsg[0] = "success"
errorMsg[100] = "node unreachable"
errorMsg[101] = "sdk unreachable"
errorMsg[102] = "timeout"
def make_channel_request(self, data, packet_type,
response_type=None):
seq = ChannelPack.make_seq32()
request_pack = ChannelPack(packet_type, seq, 0, data)
self.send_pack(request_pack)
onresponse_emitter_str = ChannelHandler.getEmitterStr(self.onResponsePrefix,
seq, response_type)
# register onResponse emitter
self.lock.acquire()
self.callbackEmitter.on(onresponse_emitter_str, self.onResponse)
self.lock.release()
self.requests.append(onresponse_emitter_str)
# register onResponse emitter of RPC
rpc_onresponse_emitter_str = None
rpc_result_emitter_str = None
if response_type is ChannelPack.TYPE_TX_COMMITTED \
or response_type is ChannelPack.CLIENT_REGISTER_EVENT_LOG:
rpc_onresponse_emitter_str = ChannelHandler.getEmitterStr(self.onResponsePrefix,
seq, packet_type)
self.requests.append(rpc_onresponse_emitter_str)
rpc_result_emitter_str = ChannelHandler.getEmitterStr(self.getResultPrefix,
seq, packet_type)
self.lock.acquire()
self.callbackEmitter.on(rpc_onresponse_emitter_str, self.onResponse)
self.lock.release()
emitter_str = ChannelHandler.getEmitterStr(self.getResultPrefix,
seq, response_type)
def resolve_promise(resolve, reject):
"""
resolve promise
"""
# register getResult emitter
self.lock.acquire()
self.callbackEmitter.on(emitter_str, (lambda result, is_error: resolve(result)))
# 1. if send transaction failed, return the error message directly
# and erase the registered 0x1002 emitter
# 2. if send transaction success, remove the registered 0x12 emitter
if rpc_result_emitter_str is not None:
self.callbackEmitter.on(
rpc_result_emitter_str,
(lambda result, is_error:
resolve(result) and self.requests.remove(onresponse_emitter_str)
if is_error is True else self.requests.remove(rpc_onresponse_emitter_str)
if self.requests.count(rpc_onresponse_emitter_str) else None))
self.lock.release()
p = Promise(resolve_promise)
# default timeout is 60s
return p.get(60)
def make_channel_rpc_request(self, method, params, packet_type=ChannelPack.TYPE_RPC,
response_type=ChannelPack.TYPE_RPC):
rpc_data = self.encode_rpc_request(method, params)
#self.logger.debug("request rpc_data : {}".format(rpc_data))
return self.make_channel_request(rpc_data, packet_type, response_type)
def setBlockNumber(self, blockNumber):
"""
init block number
"""
self.blockNumber = blockNumber
def getBlockNumber(self, groupId):
"""
get block number notify
"""
block_notify_emitter = ChannelHandler.getEmitterStr(self.onResponsePrefix,
ChannelPack.get_seq_zero(),
ChannelPack.TYPE_TX_BLOCKNUM)
self.callbackEmitter.on(block_notify_emitter, self.onResponse)
self.logger.debug("block notify emitter: {}".format(block_notify_emitter))
self.requests.append(block_notify_emitter)
seq = ChannelPack.make_seq32()
topic = json.dumps(["_block_notify_{}".format(groupId)])
request_pack = ChannelPack(ChannelPack.TYPE_TOPIC_REPORT, seq, 0, topic)
self.send_pack(request_pack)
@staticmethod
def getEmitterStr(prefix, seq, response_type):
"""
get emitter str
"""
return "{}_{}_{}".format(prefix, str(seq), str(response_type))
def onResponse(self, responsepack):
"""
obtain the response of given type
"""
result = responsepack.result
data = responsepack.data.decode("utf-8")
# get onResponse emitter
onresponse_emitter = ChannelHandler.getEmitterStr(self.onResponsePrefix,
responsepack.seq, responsepack.type)
if onresponse_emitter in self.requests and responsepack.type != ChannelPack.TYPE_TX_BLOCKNUM:
self.requests.remove(onresponse_emitter)
emitter_str = ChannelHandler.getEmitterStr(self.getResultPrefix,
responsepack.seq, responsepack.type)
self.logger.debug("onResponse, emitter: {}".format(emitter_str))
if result != 0:
self.logger.error("response from server failed , seq: {}, type:{}, result: {}".
format(responsepack.seq, responsepack.type, result))
self.callbackEmitter.emit(emitter_str, result, True)
return
try:
# json packet
if responsepack.type == ChannelPack.TYPE_RPC or \
responsepack.type == ChannelPack.TYPE_TX_COMMITTED:
response = FriendlyJsonSerde().json_decode(data)
response_item = None
error_status = False
if 'error' in response.keys():
error_status = True
response_item = dict()
response_item["result"] = response
elif "result" not in response.keys():
response_item = dict()
response_item["result"] = response
else:
response_item = response
self.callbackEmitter.emit(emitter_str, response_item, error_status)
self.logger.debug("response from server , seq: {}, type:{}".
format(responsepack.seq, responsepack.type))
# block notify
elif responsepack.type == ChannelPack.TYPE_TX_BLOCKNUM:
number = int(data.split(',')[1], 10)
self.logger.debug("receive block notify: seq: {} type:{}, data:{}".
format(responsepack.seq, responsepack.type, data))
if self.blockNumber < number:
self.blockNumber = number
self.logger.debug("currentBlockNumber: {}".format(self.blockNumber))
elif responsepack.type == ChannelPack.CLIENT_REGISTER_EVENT_LOG:
self.logger.debug("receive event register result: seq: {} type:{}".
format(responsepack.seq, responsepack.type))
#print("receive event register result: seq: {} type:{}".format(responsepack.seq, responsepack.type))
self.callbackEmitter.emit(emitter_str, responsepack.data, 0)
elif responsepack.type == ChannelPack.EVENT_LOG_PUSH:
print("event log push:", responsepack.data)
except Exception as e:
self.logger.error("decode response failed, seq:{}, type:{}, error info: {}"
.format(responsepack.seq, responsepack.type, e))
error_msg = "decode response failed, seq:{}, type:{}, message: {}".format(
responsepack.seq, responsepack.type, result)
self.callbackEmitter.emit(emitter_str, error_msg, True)
def send_pack(self, pack):
if self.sendThread.packQueue.full():
self.logger.error("channel send Queue full!")
raise BcosError(-1, None, "channel send Queue full!")
self.sendThread.packQueue.put(pack)
class Watch(object):
# def __init__(self, gpio, trig, echo, func_in, func_out, offset):
def __init__(self, **kwargs):
""" Watch(gpio=GPIO, trig=23, echo=24, func_in=None, func_out=None, offset=200)
The Watch class.
Please always use *kwargs* in the constructor.
- *gpio*: Pass the GPIO object
- *trig*: Pin for trigger
- *echo*: Pin for trigger
- *func_in*: handler when a objects comes into field
- *func_out*: handler when a objects goes out of field
- *offset*: offset in cm to determine if the object is IN zone or OUT zone
"""
def dummy_func_in():
print("Dummy in function")
def dummy_func_out():
print("Dummy out function")
super(Watch, self).__init__()
func_in = kwargs.get("func_in", dummy_func_in)
func_out = kwargs.get("func_out", dummy_func_out)
self._ee = EventEmitter(wildcard=True,
new_listener=True,
max_listeners=-1)
self._ee.on("ObjectIn", func_in)
self._ee.on("ObjectOut", func_out)
self._gpio = kwargs.get("gpio", None)
self._trig = kwargs.get("trig", 23)
self._echo = kwargs.get("echo", 24)
self._offset = kwargs.get("offset", 200)
self._wasIn = False
self._observer = ThreadPoolExecutor(max_workers=1)
self._observer_on = True
self._future = None
def trigger_pin(self):
return self._trig
def echo_pin(self):
return self._echo
def ping(self):
distance = self.get_distance()
if distance < self._offset:
if not self._wasIn:
self._ee.emit("ObjectIn", distance)
self._wasIn = True
else:
if self._wasIn:
self._ee.emit("ObjectOut", distance)
self._wasIn = False
return
def poll(self):
while True:
if self._observer_on:
self.ping()
else:
return
def observe(self):
self._future = self._observer.submit(self.poll)
return
def stop(self):
self._observer_on = False
time.sleep(3)
try:
self._future.shutdown(wait=False)
except:
if self._future.running():
self._observer.shutdown(wait=False)
return
def get_distance(self):
print("Distance Measurement In Progress")
self._gpio.setup(self._trig, self._gpio.OUT)
self._gpio.setup(self._echo, self._gpio.IN)
self._gpio.output(self._trig, False)
print("Waiting For Sensor To Settle")
time.sleep(2)
self._gpio.output(self._trig, True)
time.sleep(0.00001)
self._gpio.output(self._trig, False)
pulse_start = time.time()
pulse_end = time.time()
while self._gpio.input(self._echo) == 0:
pulse_start = time.time()
while self._gpio.input(self._echo) == 1:
pulse_end = time.time()
pulse_duration = pulse_end - pulse_start
distance = pulse_duration * 17150
distance = round(distance, 2)
print("Distance:", distance, "cm")
self._gpio.cleanup()
return distance
def handler_foo2(arg):
print("foo handler 2 called with", arg)
@ee.on("foo.*", ttl=1)
def handler_fooall(arg):
print("foo.* handler called with", arg)
@ee.on("foo.bar")
def handler_foobar(arg):
print("foo.bar handler called with", arg)
@ee.on_any()
def handler_any(*args, **kwargs):
print("called every time")
print("emit foo")
ee.emit("foo", "test")
print(10 * "-")
print("emit foo.bar")
ee.emit("foo.bar", "test")
print(10 * "-")
print("emit foo.*")
ee.emit("foo.*", "test")
print(10 * "-")
from pymitter import EventEmitter
ee = EventEmitter()
# decorator usage
@ee.on("myevent")
def handler1(arg):
print("handler1 called with " + arg)
# callback usage
def handler2(arg):
print("handler2 called with " + arg)
ee.on("myotherevent", handler2)
# emit
ee.emit("myevent", "foo")
# -> "handler1 called with foo"
ee.emit("myotherevent", "bar")
# -> "handler2 called with bar"
print("foo handler 1 called with", arg)
@ee.on("foo")
def handler_foo2(arg):
print("foo handler 2 called with", arg)
@ee.on("foo.*", ttl=1)
def handler_fooall(arg):
print("foo.* handler called with", arg)
@ee.on("foo.bar")
def handler_foobar(arg):
print("foo.bar handler called with", arg)
@ee.on_any()
def handler_any(*args, **kwargs):
print("called every time")
print("emit foo")
ee.emit("foo", "test")
print(10 * "-")
print("emit foo.bar")
ee.emit("foo.bar", "test")
print(10 * "-")
print("emit foo.*")
ee.emit("foo.*", "test")
print(10 * "-")
@ee.on("foo.*", ttl=1)
def handler_fooall(arg):
print("foo.* handler called with", arg)
@ee.on("foo.bar")
def handler_foobar(arg):
print("foo.bar handler called with", arg)
@ee.on_any()
def handler_any(*args, **kwargs):
print("called every time with", args[0])
print("emit foo")
ee.emit("foo", "test")
print(20 * "-")
print("emit foo.bar")
ee.emit("foo.bar", "test")
print(20 * "-")
print("emit foo.*")
ee.emit("foo.*", "test")
print(20 * "-")
ee.emit("bullshit", "bullshit")
class Wuzei:
def __init__(self, config: WuzeiConfig, logger=None):
if not logger:
logger = partial(print, sep='\t')
self.logger = logger
self.ee = EventEmitter()
self.ee.on('lock', self._on_lock)
self.ee.on('unlock', self._on_unlock)
self.ee.on('hotkey', self._on_hotkey)
self.ee.on('timer', self._on_timer)
self.threads = []
self.running_event = InterruptibleEvent()
self._update_time()
self.config = config
self.paused = config.paused
self.interval = config.interval
self._sources = [path for key, path in config.sources.items()]
self.manager = WallpaperManager(paths=self._sources,
cache_dir=config.cache_dir,
blurred=config.blurred,
shuffled=config.shuffled,
logger=self.logger)
def _monitor_hotkeys(self):
actions = dict(
prev=Action.PREV_WALLPAPER,
next=Action.NEXT_WALLPAPER,
prev_source=Action.PREV_SOURCE,
next_source=Action.NEXT_SOURCE,
toggle_shuffle=Action.TOGGLE_SHUFFLE,
toggle_blur=Action.TOGGLE_BLUR,
blur=Action.BLUR,
exit=Action.EXIT,
pause=Action.PAUSE,
view=Action.VIEW,
)
hotkeys = {
hotkey: actions[name]
for name, hotkey in self.config.hotkeys.items()
}
for combination, action in hotkeys.items():
keyboard.add_hotkey(combination,
callback=self.ee.emit,
args=['hotkey', action])
def _monitor_session(self):
sm = SessionMonitor()
sm.subscribe(SessionEvent.SESSION_LOCK, self.ee.emit, 'lock')
sm.subscribe(SessionEvent.SESSION_UNLOCK, self.ee.emit, 'unlock')
sm.listen()
def _monitor_dirs(self):
cooldown = self.config.dir_monitor_cooldown
def make_callback(path: str):
@throttle(cooldown=cooldown + 5)
def cb(*args):
self.logger('SOURCE CHANGED', path)
time.sleep(cooldown)
self.manager.sync(path)
return cb
for path in self._sources:
callback = make_callback(path)
watcher = DirectoryWatcher(path=path,
on_deleted=callback,
on_created=callback,
on_renamed=callback)
while True:
time.sleep(10)
def _setup_timer(self):
while True and self.interval > 0:
time.sleep(1)
elapsed = time.time() - self.last_change
if elapsed < self.interval:
continue
if not self.paused:
self.ee.emit('timer')
def _on_timer(self):
self.logger('TIMER')
self.manager.next_wallpaper()
self._update_time()
def _on_lock(self):
self.logger('LOCKED')
self.manager.blur()
def _on_unlock(self):
self.logger('UNLOCKED')
keyboard.stash_state()
def _on_mouse(self, desktop: WindowSpy, taskbar: WindowSpy):
if desktop.is_under_mouse:
self.logger('DESKTOP CLICKED')
self.manager.toggle_blur()
if taskbar.is_under_mouse:
self.logger('TASKBAR CLICKED')
if keyboard.is_pressed('alt'):
self.manager.blur()
else:
self.manager.toggle_blur()
def _rehook(self):
while True:
time.sleep(self.config.hook_refresh_interval)
keyboard.stash_state()
def _hook_mouse(self):
desktop = WindowSpy.desktop()
taskbar = WindowSpy.taskbar()
mouse.on_double_click(self._on_mouse, args=(desktop, taskbar))
def _on_hotkey(self, action: Action):
self.logger('HOTKEY', action)
self.handle_action(action)
def handle_action(self, action: Action):
try:
handlers = {
Action.PREV_WALLPAPER: self.manager.prev_wallpaper,
Action.NEXT_WALLPAPER: self.manager.next_wallpaper,
Action.PREV_SOURCE: self.manager.prev_source,
Action.NEXT_SOURCE: self.manager.next_source,
Action.TOGGLE_SHUFFLE: self.manager.toggle_shuffle,
Action.TOGGLE_BLUR: self.manager.toggle_blur,
Action.BLUR: self.manager.blur,
Action.PAUSE: self.pause,
Action.VIEW: self.view_current,
Action.EXIT: self.exit,
}
handlers[action]()
self._update_time()
except KeyError:
self.logger('Unhandled action', action)
raise NotImplementedError
def _update_time(self):
self.last_change = time.time()
def pause(self):
self.paused = not self.paused
self.logger('PAUSED' if self.paused else 'RUNNING')
def view_current(self):
os.startfile(self.manager.wallpaper)
def exit(self):
self.running_event.set()
self.logger('Bye')
os._exit(0)
def run(self):
threads = dict(
keyboard=threading.Thread(target=self._monitor_hotkeys),
timer=threading.Thread(target=self._setup_timer),
rehook=threading.Thread(target=self._rehook),
)
if self.config.blur_on_lock:
threads['session'] = threading.Thread(target=self._monitor_session)
if self.config.monitor_dirs:
threads['directory'] = threading.Thread(target=self._monitor_dirs)
if self.config.hook_mouse:
threads['mouse'] = threading.Thread(target=self._hook_mouse)
self.threads = threads
for name, th in threads.items():
th.start()
self.running_event.wait()
class DartApiCrawler(object):
def __init__(self) -> None:
super().__init__()
self.ee = EventEmitter()
self.isLock = False
self.isCancelled = False
self.logger = Logger("DartApiCrawler")
def createUUID(self) -> str:
return str(uuid.uuid4())
async def downloadCodes(self, isCodeNew: bool, apiKey: str) -> Dict:
if "pytest" in sys.modules:
# savepath = Path('factors/codes.zip')
loadpath = Path('factors/codes')
datapath = Path("factors/codes/CORPCODE.xml")
else:
# savepath = Path('app/static/factors/codes.zip')
loadpath = Path('app/static/factors/codes')
datapath = Path("app/static/factors/codes/CORPCODE.xml")
if isCodeNew or not os.path.exists(datapath.resolve()):
# user_agent = UserAgent(cache=False, use_cache_server=True)
headers = {
'User-Agent':
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2'",
'accept-language': 'ko'
}
params = {"crtfc_key": apiKey}
url = "https://opendart.fss.or.kr/api/corpCode.xml"
async with aiohttp.ClientSession() as session:
async with session.get(url, params=params,
headers=headers) as response:
data = await response.read()
ZipFile(io.BytesIO(data)).extractall(loadpath.resolve())
tree = ET.parse(datapath.resolve())
codes: Dict[str, Any] = {}
for li in tree.findall("list"):
el = li.find("stock_code")
if el is not None:
stockCode = el.text
if isinstance(stockCode, str) and len(stockCode) == 6:
codeEl = li.find("corp_code")
nameEl = li.find("corp_name")
if codeEl is not None:
codes[stockCode] = {}
codes[stockCode]["corp_code"] = codeEl.text
if nameEl is not None:
codes[stockCode]["corp_name"] = nameEl.text
return codes
async def crawling(self, dto: DartApiCrawling) -> None:
# cpu bound 작업
try:
if dto.startYear < 2015:
dto.startYear = 2015
self.ee.emit(EVENT_DART_API_CRAWLING_ON_DOWNLOADING_CODES, dto)
codes = await asyncRetryNonBlock(5,
1,
self.downloadCodes,
isCodeNew=dto.isCodeNew,
apiKey=dto.apiKey)
# codes = self.downloadCodes(dto.isCodeNew, dto.apiKey)
self.ee.emit(EVENT_DART_API_CRAWLING_ON_CRAWLING_FACTOR_DATA, dto)
for year in range(dto.startYear, dto.endYear + 1):
self.ee.emit(EVENT_DART_API_CRAWLING_ON_CRAWLING_FACTOR_DATA,
dto)
self.logger.info("crawling", str(len(codes)))
for code in codes:
# newDf = self.getYearDf(dart, code, codes, year)
newDf = await asyncRetryNonBlock(5, 1, self.getYearDf,
dto.apiKey, code, codes,
year)
if self.isCancelled:
self.ee.emit(EVENT_DART_API_CRAWLING_ON_CANCEL, dto)
if newDf is not None:
self.logger.info("crawling", code)
self.ee.emit(
EVENT_DART_API_CRAWLING_ON_RESULT_OF_FACTOR, dto,
year, newDf.to_dict("records"))
# yearDf = await self.getYearDf(dart, code, codes, year, yearDf)
self.ee.emit(EVENT_DART_API_CRAWLING_ON_COMPLETE_YEAR, dto,
year)
self.logger.info("crawling", str(year))
except Exception as e:
raise e
async def getYearDf(self, apiKey: str, code: str, codes: Dict,
year: int) -> pd.DataFrame:
self.logger.info("getYearDf", f"crawling: {code}")
df = None
try:
url = 'https://opendart.fss.or.kr/api/fnlttSinglAcntAll.json'
# user_agent = UserAgent(cache=False, use_cache_server=True)
headers = {
'User-Agent':
"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.2 (KHTML, like Gecko) Chrome/22.0.1216.0 Safari/537.2'",
'accept-language': 'ko',
}
params = {
'crtfc_key': apiKey,
'corp_code': codes[code]["corp_code"],
'bsns_year': year, # 사업년도
'reprt_code': "11011", # "11011": 사업보고서
'fs_div': "CFS", # "CFS":연결재무제표, "OFS":재무제표
}
connector = aiohttp.TCPConnector(limit=50, force_close=True)
async with aiohttp.ClientSession(connector=connector) as session:
timeout = aiohttp.ClientTimeout(total=15)
# async with session.get(url, params=params, headers=headers) as response:
async with session.get(url,
params=params,
timeout=timeout,
headers=headers) as response:
data = await response.json()
if 'list' not in data:
return None
df = pd.json_normalize(data, 'list')
# df = dart.finstate_all(code, year)
# df = await asyncio.create_task(dart.finstate_all(code, year))
# df = await loop.run_in_executor(self.pool, dart.finstate_all, code, year)
except Exception as e:
self.logger.error("getYearDf", traceback.format_exc())
raise e
self.logger.info("df", str(df))
if df is not None:
df["crawling_year"] = year
df["crawling_code"] = code
df["crawling_name"] = codes[code]["corp_name"]
name = codes[code]["corp_name"]
self.logger.info("getYearDf",
f"{str(year)} {str(code)} {str(name)}")
return df
# allCodeDf = pd.concat([allCodeDf, df])
# return allCodeDf
return None
from pymitter import EventEmitter
#responsible for mapping
def map(input):
#return input + "processed"
if input == "a":
return "b"
else:
return input
#event handling
ee = EventEmitter()
@ee.on("handleInput")
def handler1(input):
print("your input was " + map(input))
# use some basic keyboard input as data source to map
run = True
while run:
keyInput = input("input (type X to end): ")
if keyInput == "X":
run = False
else:
ee.emit("handleInput", keyInput)
class Bot(discord.Client):
""" Discord-Bot Kenrnel class """
EVENT_READY = "reday"
EVENT_MESSAGE = "message"
EVENT_USER_CONNECT_VOICE = "userconnectvoice"
EVENT_USER_DISCONNECT_VOICE = "userdisconnectvoice"
EVENT_USER_SWITCHED_VOICE = "userswitchedvoice"
def __init__(self, *args, **kwargs):
super(Bot, self).__init__(*args, **kwargs)
self.ee = EventEmitter()
async def on_ready(self):
log("Bot online")
self.ee.emit(Bot.EVENT_READY, self)
async def on_message(self, msg):
self.ee.emit(Bot.EVENT_MESSAGE, self, msg)
async def execute(msg):
cmd_name = msg.content.split('!')[1].split(' ')[0].lower()
log(f"Command [{cmd_name}] from [{str(msg.author)}]")
try:
cmd = next(x for x in Command.commands if x.name == cmd_name)
await cmd.execute(msg)
except StopIteration:
await msg.channel.send("Bitte was soll ich machen? Den Command kenne ich nicht. Hast du dich vertippt?")
await asyncio.sleep(1)
await msg.channel.send("Wenn ich den kennen sollte, dann wende dich ans Klebi. Der hilft gerne.")
def îs_ignored_token(m):
for _p in Main.i().igp:
if re.match(_p, m):
return True
return False
if msg.author.bot:
return
if îs_ignored_token(msg.content):
return
if msg.content.startswith("!") and len(msg.content) >= 2 :
await asyncio.sleep(0.5)
try:
await execute(msg)
except Exception:
await msg.channel.send("Oh! Da hast du wohl einen Bug gefunden!")
await asyncio.sleep(1)
await msg.channel.send("Irgendwas stimmt hier nicht. Ich weiß leider auch nicht was...")
await asyncio.sleep(1)
await msg.channel.send(f"Hay {Main.i().config['developer-mention']}! Schau dir das mal bitte an!")
await asyncio.sleep(1)
await msg.channel.send("Ich will ja, dass mein Code ordentlich funktioniert!")
error_user = await self.fetch_user(Main.i().config["send-error-to-user-id"])
await error_user.send(f"```\n{traceback.format_exc()}\n```")
async def on_voice_state_update(self, member, before, after):
# If channel difference
if before.channel != after.channel:
if before.channel is None:
self.ee.emit(Bot.EVENT_USER_CONNECT_VOICE, member, after.channel)
elif after.channel is None:
self.ee.emit(Bot.EVENT_USER_DISCONNECT_VOICE, member, before.channel)
else:
self.ee.emit(Bot.EVENT_USER_SWITCHED_VOICE, member, before.channel, after.channel)
class MarcapCrawler(object):
def __init__(self) -> None:
super().__init__()
self.ee = EventEmitter()
self.logger = Logger("MarcapCrawler")
def createUUID(self) -> str:
return str(uuid.uuid4())
async def connectWebDriver(self, addr: str, uuid: str) -> WebDriver:
chrome_options = webdriver.ChromeOptions()
prefs = {
'profile.default_content_setting_values.automatic_downloads': 1,
'download.default_directory': f"/home/seluser/Downloads/{uuid}"
}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Remote(
command_executor=addr,
options=chrome_options,
)
driver.set_page_load_timeout(60)
driver.set_script_timeout(60)
self.logger.info("connectWebDriver", "create driver")
return driver
def connectLocalDriver(self, addr: str, uuid: str) -> WebDriver:
chrome_options = webdriver.ChromeOptions()
prefs = {
'profile.default_content_setting_values.automatic_downloads': 1,
'download.default_directory': f"/Users/iseongjae/Documents/PersonalProjects/fin-web/fin-crawling-server/server/downloads/{uuid}"
}
chrome_options.add_experimental_option("prefs", prefs)
driver = webdriver.Chrome(executable_path="/Users/iseongjae/Downloads/chromedriver", chrome_options=chrome_options)
return driver
async def crawling(self, dto: StockRunCrawling) -> None:
driver = None
downloadObserver = None
try:
uuid = self.createUUID()
self.logger.info("crawling", uuid)
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_CONNECTING_WEBDRIVER, dto)
downloadObserver = DownloadObserver()
path = await asyncRetryNonBlock(5, 1, downloadObserver.makePath, uuid)
downloadObserver.startObserver(path, self.ee)
self.logger.info("crawling", "create observer and start")
print("startObserver")
driver = await asyncRetryNonBlock(5, 1, self.connectWebDriver, dto.driverAddr, uuid)
print("connectWebDriver")
driver.get("http://data.krx.co.kr/contents/MDC/MDI/mdiLoader/index.cmd?menuId=MDC0201020101")
try:
alert = WebDriverWait(driver, timeout=3).until(EC.alert_is_present())
alert.accept()
except Exception as e:
print("예외발생:"+str(e))
print("start:"+dto.startDateStr)
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_START_CRAWLING, dto)
WebDriverWait(driver, timeout=20, poll_frequency=1).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "#mktId_0_1")))
date = datetime.strptime(dto.startDateStr, "%Y%m%d")
endDate = datetime.strptime(dto.endDateStr, "%Y%m%d")
while date <= endDate:
dateStr = date.strftime("%Y%m%d")
downloadTask = StockCrawlingDownloadTask(**{
"dateStr": dateStr,
"market": dto.market,
"uuid": uuid,
"taskId": dto.taskId,
"taskUniqueId": dto.taskUniqueId
})
self.logger.info("crawling", f"create downloadTask taskId: {dto.taskId} market: {dto.market} date: {dateStr} taskUniqueId: {dto.taskUniqueId}")
print(downloadTask.json())
downloadObserver.event_handler.setDownloadTask(downloadTask)
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_DOWNLOAD_START, downloadTask)
await asyncRetryNonBlock(5, 1, self.downloadData, downloadTask, downloadObserver, driver)
# await self.downloadData(downloadTask, downloadObserver, driver)
date = date + timedelta(days=1)
except Exception as e:
raise e
finally:
if downloadObserver:
downloadObserver.stopObserver()
if driver:
driver.quit()
async def downloadData(self, downloadTask: StockCrawlingDownloadTask, downloadObserver: DownloadObserver, driver: WebDriver) -> None:
self.logger.info("downloadData")
if driver is None:
return
# pymitter
before = driver.execute_script("return $('.CI-MDI-UNIT-TIME').text()")
if downloadTask.market == "kospi":
driver.execute_script('$("#mktId_0_1").click()')
elif downloadTask.market == "kosdaq":
driver.execute_script('$("#mktId_0_2").click()')
elif downloadTask.market == "konex":
driver.execute_script('$("#mktId_0_3").click()')
# driver.implicitly_wait(1)
driver.execute_script(f'$("#trdDd")[0].value = "{downloadTask.dateStr}"')
# driver.implicitly_wait(1)
driver.execute_script('$(".btn_component_search").click()')
# driver.implicitly_wait(1)
after = before
while before == after:
after = driver.execute_script('return $(".CI-MDI-UNIT-TIME").text()')
await sleepNonBlock(0.5)
# driver.implicitly_wait(1)
print("before:"+before)
print("after:"+after)
await sleepNonBlock(3)
WebDriverWait(driver, timeout=10, poll_frequency=2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "*[class='CI-MDI-UNIT-DOWNLOAD']")))
driver.execute_script("$('[class=\"CI-MDI-UNIT-DOWNLOAD\"]').click()")
WebDriverWait(driver, timeout=10, poll_frequency=2).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "*[data-type='csv']")))
driver.execute_script("$(\"[data-type='csv']\").click()")
print("wait:"+downloadTask.dateStr)
loop = asyncio.get_running_loop()
queue: asyncio.Queue = asyncio.Queue(maxsize=1, loop=loop)
async def fileResultOfData(event: FileCreatedEvent, downloadTask: StockCrawlingDownloadTask) -> None:
result = {}
result["event"] = event
result["downloadTask"] = downloadTask
await queue.put(result)
@self.ee.once(FILE_SYSTEM_HANDLER(downloadTask.uuid))
def downloadComplete(event: FileCreatedEvent, downloadTask: StockCrawlingDownloadTask) -> None:
loop.create_task(fileResultOfData(event, downloadTask))
try:
result = await asyncio.wait_for(queue.get(), timeout=30)
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_DOWNLOAD_COMPLETE, downloadTask)
await asyncio.create_task(self.makeMarcapData(result["event"], result["downloadTask"]))
except Exception as e:
raise e
finally:
queue.task_done()
def convertFileToDto(self, path: str, dto: StockMarketCapitalResult) -> None:
lines = []
with open(path, "r", encoding="utf-8") as f:
# p = Path(f.name)
# dto.date = p.stem
lines = f.readlines()
for i in range(1, len(lines)):
data = lines[i].replace('"', '').split(",")
if dto.market == "kospi":
marcap = StockMarketCapital(**{
"date": dto.date,
"market": dto.market,
"code": data[0].strip(),
"name": data[1].strip(),
"close": data[2].strip(),
"diff": data[3].strip(),
"percent": data[4].strip(),
"open": data[5].strip(),
"high": data[6].strip(),
"low": data[7].strip(),
"volume": data[8].strip(),
"price": data[9].strip(),
"marcap": data[10].strip(),
"number": data[11].strip()
})
else:
marcap = StockMarketCapital(**{
"date": dto.date,
"market": dto.market,
"code": data[0].strip(),
"name": data[1].strip(),
"close": data[3].strip(),
"diff": data[4].strip(),
"percent": data[5].strip(),
"open": data[6].strip(),
"high": data[7].strip(),
"low": data[8].strip(),
"volume": data[9].strip(),
"price": data[10].strip(),
"marcap": data[11].strip(),
"number": data[12].strip()
})
# print("append marcap: " + str(marcap))
dto.data.append(marcap)
async def isExistFile(self, path: str, ext: str = ".csv") -> bool:
isExist = path.endswith(ext)
restTimes = 3
while not isExist and restTimes >= 0:
await sleepNonBlock(1)
isExist = path.endswith(ext)
restTimes -= 1
return isExist
async def parseReceivedFile(self, event: FileCreatedEvent, downloadTask: StockCrawlingDownloadTask) -> None:
retdto = StockMarketCapitalResult()
date = downloadTask.dateStr
market = downloadTask.market
retdto.date = date
retdto.market = market
isExist = await self.isExistFile(event.src_path)
if not isExist:
return
print("created: " + date)
await sleepNonBlock(0.5)
dest_path = f'{os.path.dirname(event.src_path)}/{market+"-"+date}.csv'
if os.path.isfile(dest_path):
return
self.changeCharSet(event.src_path)
os.rename(event.src_path, dest_path)
self.convertFileToDto(dest_path, retdto)
retdto.result = "success"
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_PARSING_COMPLETE, True, retdto, downloadTask)
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_RESULT_OF_STOCK_DATA, downloadTask, retdto)
self.logger.info("parseFile", f"success, {downloadTask.taskUniqueId}")
async def makeMarcapData(self, event: FileCreatedEvent, downloadTask: StockCrawlingDownloadTask) -> None:
try:
await asyncRetry(3, 1, self.parseReceivedFile, event, downloadTask)
except Exception:
retdto = StockMarketCapitalResult()
retdto.result = "fail"
retdto.errorMsg = traceback.format_exc()
self.ee.emit(EVENT_MARCAP_CRAWLING_ON_PARSING_COMPLETE, False, retdto, downloadTask)
self.logger.error("parseFile", f"fail, {downloadTask.taskUniqueId} error: {traceback.format_exc()}")
finally:
self.logger.info("parseFile...")
def changeCharSet(self, path: str) -> None:
lines = None
with open(path, "r", encoding="euc-kr") as f:
lines = f.readlines()
with open(path, 'w', encoding="utf-8") as f:
f.writelines(lines)
class AllTestCase(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(AllTestCase, self).__init__(*args, **kwargs)
self.ee1 = EventEmitter()
self.ee2 = EventEmitter(wildcard=True)
self.ee3 = EventEmitter(wildcard=True, delimiter=":")
self.ee4 = EventEmitter(new_listener=True)
self.ee5 = EventEmitter(max_listeners=1)
self.stack = []
def test_1_callback_usage(self):
def handler(arg):
self.stack.append("1_callback_usage_" + arg)
self.ee1.on("1_callback_usage", handler)
self.ee1.emit("1_callback_usage", "foo")
self.assertTrue(self.stack[-1] == "1_callback_usage_foo")
def test_1_decorator_usage(self):
@self.ee1.on("1_decorator_usage")
def handler(arg):
self.stack.append("1_decorator_usage_" + arg)
self.ee1.emit("1_decorator_usage", "bar")
self.assertTrue(self.stack[-1] == "1_decorator_usage_bar")
def test_1_ttl(self):
# same as once
@self.ee1.on("1_ttl", ttl=1)
def handler(arg):
self.stack.append("1_ttl_" + arg)
self.ee1.emit("1_ttl", "foo")
self.assertTrue(self.stack[-1] == "1_ttl_foo")
self.ee1.emit("1_ttl", "bar")
self.assertTrue(self.stack[-1] == "1_ttl_foo")
def test_2_on_all(self):
@self.ee2.on("2_on_all.*")
def handler():
self.stack.append("2_on_all")
self.ee2.emit("2_on_all.foo")
self.assertTrue(self.stack[-1] == "2_on_all")
def test_2_emit_all(self):
@self.ee2.on("2_emit_all.foo")
def handler():
self.stack.append("2_emit_all.foo")
self.ee2.emit("2_emit_all.*")
self.assertTrue(self.stack[-1] == "2_emit_all.foo")
def test_3_delimiter(self):
@self.ee3.on("3_delimiter:*")
def handler():
self.stack.append("3_delimiter")
self.ee3.emit("3_delimiter:foo")
self.assertTrue(self.stack[-1] == "3_delimiter")
def test_4_new(self):
@self.ee4.on("new_listener")
def handler(func, event=None):
self.stack.append((func, event))
def newhandler():
pass
self.ee4.on("4_new", newhandler)
self.assertTrue(self.stack[-1] == (newhandler, "4_new"))
def test_5_max(self):
@self.ee5.on("5_max")
def handler1():
self.stack.append("5_max_1")
@self.ee5.on("5_max")
def handler2():
self.stack.append("5_max_2")
self.ee5.emit("5_max")
self.assertTrue(self.stack[-1] == "5_max_1")
class GuiViewModel():
def __init__(self):
self.ee = EventEmitter()
self.MiNombre = "..."
self.MiPuntaje = 0
self.MisCartas = []
self.MiSaldo = 0
self.MiEstado = ""
self.Validado = False
self.esMiTurno = False
self.Turno = ""
self.Acciones = []
self.Jugadores = []
self.lenguaje = "es"
#Evento que lanza la GUI para pedir conexion a traves del cliente
def onRequestConnection(self, servidor, puerto):
return self.ee.emit("requestConnectionEvent", servidor, puerto)
#Evento que envia la GUI para enviar un comando SOY
def onSoy(self, soy):
self.ee.emit("soyEvent", soy)
#Evento que envia el Cliente a la GUI para avisarle que fue aceptado en la Sala
def onSoyAceptado(self):
self.ee.emit("soyAceptadoEvent", )
#Evento que envia el Cliente a la GUI para avisarle que se rechazo el ingreso a la Sala
def onSoyRechazado(self, mensaje):
self.ee.emit("soyRechazadoEvent", mensaje)
#Evento que indica a la GUI que la conexion se establecio correctamente
def onConnected(self):
self.ee.emit("connectedEvent", )
#Evento que le indica a la GUI que fallo la conexion
def onConnectError(self, mensaje):
self.ee.emit("connectErrorEvent", mensaje)
#Evento que dispara la GUI para solicitar una carta
def onPedirCarta(self):
self.ee.emit("pedirCartaEvent", )
#Evento que dispara la GUI para plantarse
def onPlantarse(self):
self.ee.emit("plantarseEvent", )
#Evento que dispara la GUI para duplicar la apuesta
def onDoblar(self):
self.ee.emit("doblarEvent", )
#Evento que dispara la GUI para ingresar dinero
def onFondear(self, monto):
self.ee.emit("fondearEvent", monto)
#Evento que dispara la GUI para ingresar una apuesta
def onApostar(self, monto):
self.ee.emit("apostarEvent", monto)
#Evento que dispara la GUI para enviar un mensaje de chat
def onEnviarMensaje(self, mensaje):
self.ee.emit("enviarMensajeEvent", mensaje)
#Evento que informa a la GUI de los botones que tiene habilitados en cada momento
def onRefreshButtons(self, botones):
self.Acciones = botones
self.ee.emit("refreshButtonsEvent", (botones))
#Evento que indica a la GUI cuando cambia el turno de la partida
def onTurnoChanged(self, turno):
self.Turno = turno
self.esMiTurno = self.MiNombre == turno
self.ee.emit("turnoChangedEvent", turno)
#Evento que indica a la GUI cuando llega un mensaje entrante
def onMensajeEntrante(self, mensaje, tipo):
self.ee.emit("mensajeEntranteEvent", mensaje, tipo)
#Evento que indica cuando comienza una partida nueva
def onJuegoComenzado(self, mensaje):
self.ee.emit("juegoComenzadoEvent", mensaje)
#Evento que indica que la partida termino
def onJuegoTerminado(self):
self.ee.emit("juegoTerminadoEvent", )
#Evento que notifica que algun dato cambio, estado, puntaje, etc.
def onEstadoChanged(self, estado):
self.ee.emit("estadoChangedEvent", estado)
#Evento que indica que el status de los otros jugadores cambio
def onJugadoresRefreshed(self, status):
self.ee.emit("jugadoresRefreshedEvent", status)
#Evento que reporta el cambio de estado de la mano de la banca
def onPuntajeBancaChanged(self, puntaje, cartas):
self.ee.emit("puntajeBancaChangedEvent", puntaje, cartas)
def onSolicitarEstadisticas(self):
self.ee.emit("solicitarEstadisticasEvent", )
def onEstadisticasRecibidas(self, estadisticas):
self.ee.emit("estadisticasRecibidasEvent", estadisticas)
class Agent(object):
""" Agent implementation with a Finite State Machine.
The agent uses a Machine instance to move through the FSM that's
created based on a given strategy. The class describes with methods
all the possible operations the Agent can perform.
"""
def __init__(self, config='strategies.json', strategy='normal',
name='Pandora', testing=False, verbose=False):
""" Initializes the agent.
:param :name The name of the agent. Defaults to Pandora.
:param :strategy Defines the configuration that will be loaded from
the Agent.
:param :config A yaml/json file that contains the agent strategies.
The file should be located in the config folder of this
package.
"""
# Configuration folder
config_dir = RosPack().get_path(PKG) + '/config/'
self.name = name
self.verbose = verbose
self.testing = testing
self.strategy = strategy
self.config = config_dir + config
# Dispatcher for event based communication.
self.dispatcher = EventEmitter()
# SUBSCRIBERS.
self.world_model_sub = Subscriber(topics.world_model, WorldModel,
self.receive_world_model)
# ACTION CLIENTS.
self.explorer = clients.Explorer(self.dispatcher)
self.data_fusion = clients.DataFusion()
self.navigator = clients.Navigator(self.dispatcher)
self.gui_client = clients.GUI()
self.effector = clients.Effector()
if not self.testing:
Service('/gui/kill_agent', Empty, self.destroy_agent)
self.transform_listener = tf.TransformListener()
# State client
if not self.testing:
log.debug('Connecting to state manager.')
self.state_changer = StateClient()
self.state_changer.client_initialize()
log.debug('Connection established.')
self.state_changer.change_state_and_wait(RobotModeMsg.MODE_OFF)
# General information.
self.current_pose = Pose()
self.exploration_mode = DoExplorationGoal.TYPE_DEEP
# Victim information.
self.available_targets = []
self.gui_result = ValidateVictimGUIResult()
# Between-transition information.
self.base_converged = threading.Event()
self.poi_found_lock = threading.Lock()
# Utility Variables
self.MOVE_BASE_RETRIES = 0
self.target = Target(self.dispatcher)
# Expose client methods to class
setattr(self, 'test_end_effector', self.effector.test)
setattr(self, 'park_end_effector', self.effector.park)
setattr(self, 'preempt_end_effector', self.effector.cancel_all_goals)
setattr(self, 'preempt_explorer', self.explorer.cancel_all_goals)
setattr(self, 'scan', self.effector.scan)
self.generate_global_state_transitions()
self.load()
log.info('Agent initialized...')
######################################################
# UTILITIES #
######################################################
def set_breakpoint(self, state):
""" Stops the execution of the FSM after a given state.
Removes the implementation from the state.
:param :state The last state we want to go. After this state
the FSM will stop.
"""
# Removing the implementation of the given state.
self.machine.get_state(state).empty()
def wait_for_map(self):
sleep(5.0)
def load(self):
""" Loads the configuration file and sets up the FSM accordingly. """
try:
# Read the configuration file.
with open(self.config) as file_handler:
data = yaml.load(file_handler)
except IOError:
log.critical('Could not read configuration file.')
sys.exit(1)
try:
states = data[self.strategy]['states']
except KeyError:
log.critical('%s is not a valid strategy.', self.strategy)
sys.exit(1)
# Setting up the FSM
self.machine = Machine(model=self)
# Get all the states for the given strategy.
self.states = [state['name'] for state in states]
# Set up states tasks.
for state in states:
self.machine.add_states(state['name'], on_enter=state['tasks'],
on_exit=state['clean'])
# Create the transition table.
self.transitions = []
for state in states:
for transition in state['transitions']:
self.machine.add_transition(transition['trigger'],
state['name'], transition['to'],
before=transition['before'],
after=transition['after'],
conditions=transition['conditions']
)
# Sets up the initial state
self.machine.set_state(self.states[0])
log.debug('FSM has been loaded.')
def clean_up(self):
""" Kills agent and cleans the environment. """
self.explorer.cancel_all_goals()
self.navigator.cancel_all_goals()
self.effector.cancel_all_goals()
log.info('Agent is sleeping...')
def disable_events(self):
"""
Disable all EventEmitter events. It should be called before the
main tasks of a state.
"""
self.dispatcher.off_all()
######################################################
# DISPATCHER'S CALLBACKS #
######################################################
def exploration_success(self):
""" Called on 'exploration.success' event. The event is triggered from
the exploration client when the current goal has succeeded.
Enables the agent to move from the exploration state to end.
"""
if self.state == 'exploration':
log.warning('Map covered!')
self.map_covered()
else:
log.warning('Exploration success caught outside of exploration.')
def exploration_retry(self):
""" Called on 'exploration.retry' event. The event is triggered from
the exploration client when the current goal has failed and the
agent sends again a goal.
"""
if self.state == 'exploration':
log.warning('Retrying exploration goal.')
self.explore()
else:
log.warning('Exploration failure while not on exploration state.')
def poi_found(self):
""" Called on 'poi.found' event. The event is triggered when there are
available points of interest on the received world model. Enables
the agent to move from the exploration state to identification.
"""
if self.state != 'exploration':
# This is a bug.
log.warning('Called poi_found outside of exploration.')
return
# Ensure that poi found is not called twice with the same target.
self.poi_found_lock.acquire(False)
log.warning('A point of interest has been discovered.')
log.info('Pursuing target #%d.', self.target.info.id)
self.poi_found_lock.release()
self.point_of_interest_found()
def move_base_success(self, result):
""" The event is triggered from the move_base client when the
current goal has succeeded.
:param result: The result of the action.
"""
self.MOVE_BASE_RETRIES = 0
if self.state != 'identification':
log.error('Received move base success outside identification.')
return
log.warning('Approached potential victim.')
self.base_timer.cancel()
self.valid_victim()
def move_base_retry(self, status):
""" The event is triggered from the move_base client when the
current goal has failed and the agent will send again the goal.
After a number of failures the agent will change state and delete
the current victim.
:param :status The goal status.
"""
if self.state != 'identification':
log.error('Received move_base retry event outside identification.')
return
if self.MOVE_BASE_RETRIES < conf.MOVE_BASE_RETRY_LIMIT:
# The agent tries again.
log.warning('Retrying...%s goal', ACTION_STATES[status])
remain = conf.MOVE_BASE_RETRY_LIMIT - self.MOVE_BASE_RETRIES
log.warning('%d remaining before aborting the current target.',
remain)
self.MOVE_BASE_RETRIES += 1
self.navigator.move_base(self.target.info.victimPose)
else:
self.MOVE_BASE_RETRIES = 0
self.base_timer.cancel()
# The agent changes state.
if self.target.is_identified():
# The target is valid and the next state is hold_sensors.
log.warning('Victim with high probability identified.')
self.valid_victim()
elif self.base_converged.is_set():
log.warning('Base is close enough to the target.')
self.valid_victim()
else:
# The target is not valid and we delete it.
log.warning('Victim has been aborted.')
self.abort_victim()
def move_base_feedback(self, current, goal):
"""
The event is triggered from the move_base client when the
action server sends feedback with the current pose. Given
the goal and the current pose the agent checks if it is
close enough to the point of interest. This will work if the
move_base server takes too long to succeed.
:param :current The current PoseStamped received from the action
client.
:param :goal The PoseStamped goal of the current action.
"""
self.current_pose = current.base_position.pose
if distance_2d(goal.pose, self.current_pose) < conf.BASE_THRESHOLD:
log.warning('Base converged.')
self.base_converged.set()
def move_base_resend(self, pose):
"""
Send a new MoveBase goal. The old one is outdated because the
current target's pose has been updated.
:param pose: The new pose to go to.
"""
if self.state == 'identification':
log.warning('Move base goal is outdated...')
log.debug(pose)
self.navigator.cancel_all_goals()
self.approach_target()
######################################################
# SUBSCRIBER'S CALLBACKS #
######################################################
def destroy_agent(self, stop):
""" Kill the agent's process and stop all running goals. """
# Change to teleoperation
self.mode_teleoperated_locomotion()
# Cancel all goals
self.clean_up()
# Kill the process
pid = os.getpid()
p = psutil.Process(pid)
log.warning("Shutting down.")
p.terminate()
def receive_world_model(self, model):
""" Receives the world model from data fusion. """
# If no targets are available there is nothing to do.
if not model.victims:
return
# Remember the available targets.
self.available_targets = model.victims
if self.target.is_empty:
# Set a new target from the available ones.
new_target = self.choose_target(model.victims)
self.target.set(new_target)
else:
# Check for invalid target acquisition.
idx = self.target.info.id
for target in model.victims:
if idx == target.id:
break
else:
log.error('A non-existent target #%d has been acquired.', idx)
self.target.clean()
return
# Update the current target.
self.target.update(model.victims)
self.dispatcher.emit('poi.found')
######################################################
# AGENT'S ACTIONS #
######################################################
def reset_environment(self):
""" Sets the environment ready for the next exploration. """
self.gui_result.victimValid = False
self.target.clean()
def notify_data_fusion(self):
""" Notify data fusion about the current target. """
self.available_targets = self.data_fusion.announce_target(self.target.info.id)
def validate_victim(self):
""" Sends information about the current target. """
if not self.target.is_empty:
self.available_targets = self.data_fusion.validate_victim(self.target.info.id,
valid=self.gui_result.victimValid,
verified=self.target.is_verified())
else:
log.critical('Reached data fusion validation without target.')
def delete_victim(self):
"""
Send deletion request to DataFusion about the current target victim.
"""
self.available_targets = self.data_fusion.delete_victim(self.target.info.id)
self.victim_deleted()
def wait_for_verification(self):
"""
Check if the probability of the target exceeds the verification
threshold.
"""
log.info("Starting victim verification...")
if self.target.verified.wait(conf.VERIFICATION_TIMEOUT):
log.warning('Victim verified.')
self.verified()
else:
log.warning('Victim failed to be verified within %d secs.',
conf.VERIFICATION_TIMEOUT)
self.gui_result.victimValid = False
self.timeout()
def wait_for_operator(self):
if self.gui_client.send_request(self.target.info):
self.gui_result = self.gui_client.result()
else:
self.gui_result.victimValid = False
def approach_target(self):
""" The agent will try to approach the target's location and point
all sensors to its direction.
"""
log.info('Approaching victim...')
self.base_converged.clear()
# Move base to the target.
self.navigator.move_base(self.target.info.victimPose)
# Point sensors to the target.
self.effector.point_to(self.target.info.victimFrameId)
# Start timer to cancel all goals if the move base is unresponsive.
self.base_timer = threading.Timer(conf.MOVE_BASE_TIMEOUT,
self.timer_handler)
self.base_timer.start()
def slowly_track_target(self):
"""
The end effector slowly tracks the current target so the
captured image is staying as still as possible.
"""
# Point sensors to the target.
self.effector.slowly_point_to(self.target.info.victimFrameId)
def explore(self):
"""
Send exploration goal to the explorer. A different exploration
strategy is used depending on the global state.
"""
global_state = self.state_changer.get_current_state()
coverage = DoExplorationGoal.TYPE_DEEP
fast = DoExplorationGoal.TYPE_FAST
if global_state == RobotModeMsg.MODE_EXPLORATION_RESCUE:
log.info("** COVERAGE EXPLORATION **")
self.explorer.explore(exploration_type=coverage)
else:
log.info("** FAST EXPLORATION **")
self.explorer.explore(exploration_type=fast)
def check_for_targets(self):
"""
Check for available targets and choose one now. Don't wait
until the next update of the world model.
"""
log.debug("Checking for available targets...")
if self.target.is_empty:
if self.available_targets != []:
new_target = self.choose_target(self.available_targets)
self.target.set(new_target)
self.dispatcher.emit('poi.found')
else:
log.debug('World model is empty. Searching for targets...')
self.explore()
else:
if self.target.info in self.available_targets:
self.dispatcher.emit('poi.found')
else:
target_id = self.target.info.id
log.error('Acquired non existent target #%s' % (target_id))
self.target.clean()
if self.available_targets != []:
new_target = self.choose_target(self.available_targets)
self.target.set(new_target)
self.dispatcher.emit('poi.found')
else:
log.debug('World model is empty. Searching for targets...')
self.explore()
def timer_handler(self):
if self.state == 'identification':
log.warning('Move base is unresponsive or it takes too long.')
self.navigator.cancel_all_goals()
self.base_timer.cancel()
self.abort_victim()
else:
log.error('Timer fired outside of identification state.')
def print_results(self):
""" Prints results of the mission. """
log.info('The agent is shutting down...')
def enable_exploration_events(self):
"""
Enable EventEmitter events for the exploration state.
"""
self.dispatcher.on('exploration.success', self.exploration_success)
self.dispatcher.on('exploration.retry', self.exploration_retry)
self.dispatcher.on('poi.found', self.poi_found)
def enable_identification_events(self):
"""
Enable EventEmitter events for the identfication state.
"""
self.dispatcher.on('move_base.success', self.move_base_success)
self.dispatcher.on('move_base.retry', self.move_base_retry)
self.dispatcher.on('move_base.feedback', self.move_base_feedback)
self.dispatcher.on('move_base.resend', self.move_base_resend)
######################################################
# AGENT LOGIC #
######################################################
def choose_target(self, targets):
""" Choose the neareset possible target. """
# Should never be called with empty targets.
if not targets:
log.error('choose_target was called with no targets.')
return None
closest_target = targets[0]
min_distance = 1000
if len(targets) == 1:
return closest_target
# self.current_pose = self.explorer.pose_stamped.pose
for target in targets:
try:
(trans, rot) = self.transform_listener.lookupTransform(target.victimPose.header.frame_id,
'/base_footprint',
Time(0))
except:
log.error("Transform listener failed to acquire transformation")
return closest_target
self.current_pose = Pose()
self.current_pose.position.x = trans[0]
self.current_pose.position.y = trans[1]
self.current_pose.position.z = trans[2]
target_pose = target.victimPose.pose
target_distance = distance_2d(target_pose, self.current_pose)
if target_distance < min_distance:
min_distance = target_distance
closest_target = target
return closest_target
######################################################
# GLOBAL STATE TRANSITIONS #
######################################################
def generate_global_state_transitions(self):
""" Generates a function for every global state. The agent will
be able to call this function in order to change the
global state.
Reads all the available modes from the RobotModeMsg and creates
a function with the same name.
"""
for member, value in inspect.getmembers(RobotModeMsg):
if member.startswith('MODE_'):
func = partial(self.global_state_transition, mode=value)
setattr(self, member.lower(), func)
log.debug('Global state transitions have been generated.')
def global_state_transition(self, mode=0):
""" Is used to generate state_transition functions.
Given a desired mode the state_client will will try to change
the global state.
:param :mode A global mode from RobotModeMsg.
"""
params = (mode, conf.STATE_CHANGE_TIMEOUT)
while True:
success = self.state_changer.change_state_and_wait(*params)
if success:
log.info('==> %s', GLOBAL_STATES[mode])
break
sleep(2)
log.error('Failed to change the global state [%d]. Retrying...',
mode)
class TasksRepository(object):
def __init__(self, mongod: TaskMongoDataSource) -> None:
super().__init__()
self.mongod = mongod
self.logger = Logger("TasksRepository")
self.taskEventEmitter = EventEmitter()
self.tasksdto = ProcessTasks()
self.taskRunner: Optional[TaskRunner] = None
self.createTaskRunner()
# 태스크 러너를 만든다.
def createTaskRunner(self) -> None:
if self.taskRunner is None:
self.taskRunner = TaskRunner()
self.taskRunner.notifyCallback = self.onUpdatePoolInfo
self.logger.info("createTaskRunner", "created taskrunner")
# 태스크 풀 정보가 업데이트 될 떄 이벤트를 날린다.
def onUpdatePoolInfo(self, poolInfo: TaskPoolInfo) -> None:
self.taskEventEmitter.emit(EVENT_TASK_REPO_UPDATE_POOL_INFO, poolInfo)
self.logger.info("updatePoolInfo", f"{poolInfo.json()}")
# 테스크 풀 정보를 가져온다.
def getPoolInfo(self) -> None:
if self.taskRunner:
poolInfo = self.taskRunner.getPoolInfo()
self.taskEventEmitter.emit(EVENT_TASK_REPO_UPDATE_POOL_INFO,
poolInfo)
# 태스크 풀에 태스크를 등록한다.
def runTask(self, task: Task) -> None:
# print("runTask")
if self.taskRunner:
self.taskRunner.put(task)
# 추가된 태스크 정보를 저장한다.
def addTask(self, task: ProcessTask) -> None:
if task.taskId not in self.tasksdto.tasks:
self.tasksdto.tasks[task.taskId] = dict()
self.tasksdto.tasks[task.taskId]["list"] = dict()
self.tasksdto.tasks[task.taskId]["ids"] = []
self.tasksdto.taskIds.append(task.taskId)
self.tasksdto.tasks[task.taskId]["list"][task.taskUniqueId] = task
self.tasksdto.tasks[task.taskId]["ids"].append(task.taskUniqueId)
self.taskEventEmitter.emit(EVENT_TASK_REPO_UPDATE_TASKS, self.tasksdto)
self.logger.info("addTask", f"{task.taskUniqueId}")
# 갱신 태스크 정보를 저장한다.
def updateTask(self, task: ProcessTask) -> None:
self.tasksdto.tasks[task.taskId]["list"][task.taskUniqueId] = task
self.logger.info("updateTask", f"{task.taskUniqueId}")
self.mongod.upsertTask(task.dict())
self.taskEventEmitter.emit(EVENT_TASK_REPO_UPDATE_TASKS, self.tasksdto)
def updateAllTask(self) -> None:
self.taskEventEmitter.emit(EVENT_TASK_REPO_UPDATE_TASKS, self.tasksdto)
# 저장된 테스크 정보를 반환한다.
def getTask(self, taskId: str, taskUniqueId: str) -> ProcessTask:
if self.isExistTask(taskId, taskUniqueId):
return self.tasksdto.tasks[taskId]["list"][taskUniqueId]
return None
# 저장된 태스크가 있는지 확인한다.
def isExistTask(self, taskId: str, taskUniqueId: str) -> bool:
return taskId in self.tasksdto.tasks and taskUniqueId in self.tasksdto.tasks[
taskId]["list"]
# 저장된 태스크 정보를 삭제한다.
def deleteTask(self, task: ProcessTask) -> None:
if task.taskId in self.tasksdto.tasks:
if task.taskUniqueId in self.tasksdto.tasks[task.taskId]["list"]:
del self.tasksdto.tasks[task.taskId]["list"][task.taskUniqueId]
self.tasksdto.tasks[task.taskId]["ids"].remove(
task.taskUniqueId)
self.logger.info("deleteTask", f"{task.taskUniqueId}")
def errorTask(self, dto: TaskModel, errMsg: str) -> None:
task = self.getTask(dto.taskId, dto.taskUniqueId)
task.state = "error"
task.errMsg = errMsg
self.updateTask(task)
def completeFactorConvertFileToDbTask(self, task: ProcessTask) -> None:
self.success(task, 1)
self.updateTask(task)
self.deleteTask(task)
self.taskEventEmitter.emit(EVENT_TASK_REPO_TASK_COMPLETE, "factorFile",
None)
def completeFactorDart(self, task: ProcessTask, year: int) -> None:
self.success(task, 1)
self.updateTask(task)
if task.restCount <= 0:
self.deleteTask(task)
task.state = "complete"
self.updateTask(task)
self.logger.info("completeFactorDart", "complete")
self.taskEventEmitter.emit(
EVENT_TASK_REPO_TASK_COMPLETE, "factorDart",
StockUpdateState(
**{
"taskId": task.taskId,
"market": task.market,
"date": year,
"ret": 1
}))
# 완료된 태스크 정보를 처린한다.
def completeStockCrawlingTask(self, isSuccess: bool,
retdto: StockMarketCapitalResult,
dto: StockCrawlingDownloadTask) -> None:
self.logger.info("##############completeStockCrawlingTask",
str(isSuccess))
task = self.getTask(dto.taskId, dto.taskUniqueId)
if isSuccess:
self.success(task, 1)
else:
self.fail(task, 1)
if task.restCount <= 0:
self.deleteTask(task)
if retdto:
task.errMsg = retdto.errorMsg
task.state = "success"
self.updateTask(task)
self.logger.info("completeStockCrawlingTask", "complete")
self.taskEventEmitter.emit(
EVENT_TASK_REPO_TASK_COMPLETE, "marcap",
StockUpdateState(
**{
"taskId": dto.taskId,
"market": dto.market,
"date": dto.dateStr,
"ret": 1 if isSuccess else 2
}))
# 성공한 태스크 정보를 처리한다.
def success(self, task: ProcessTask, count: int) -> None:
task.successCount = task.successCount + count
task.restCount = task.restCount - count
i = 0
for _ in range(count):
task.tasksRet[task.index + i] = SUCCESS
i = i + 1
task.index = task.index + count
task.percent = (task.successCount + task.failCount) / task.count * 100
if task.restCount <= 0:
task.state = "success"
else:
task.state = "waiting next task"
self.logger.info("success", f"{task.taskUniqueId}")
# 실패한 태스크 정보를 처리한다.
def fail(self, task: ProcessTask, count: int) -> None:
task.failCount = task.failCount + count
task.restCount = task.restCount - count
i = 0
for _ in range(count):
left = task.tasks[task.index + i]
task.failTasks.append(left)
task.tasksRet[task.index + i] = FAIL
i = i + 1
task.index = task.index + count
task.percent = (task.successCount + task.failCount) / task.count * 100
if task.restCount <= 0:
task.state = "fail"
else:
task.state = "waiting next task"
self.logger.info("fail", f"{task.taskUniqueId}")
# 완료된 태스크 정보를 반환한다.
def getCompletedTask(self, dto: ListLimitDao) -> ListLimitDataDao:
taskData = self.mongod.getCompletedTask(dto)
print(taskData)
tasks: Dict = dict()
taskIds = []
for task in taskData.data:
if task["taskId"] not in tasks:
tasks[task["taskId"]] = dict()
tasks[task["taskId"]]["list"] = dict()
tasks[task["taskId"]]["ids"] = []
taskIds.append(task["taskId"])
tasks[task["taskId"]]["list"][task["taskUniqueId"]] = task
tasks[task["taskId"]]["ids"].append(task["taskUniqueId"])
stockCrawlingCompletedTasksDTO = StockCrawlingCompletedTasks(
**{
"history": tasks,
"historyIds": taskIds
})
taskData.data = stockCrawlingCompletedTasksDTO
self.logger.info("getCompletedTask", f"count: {len(taskIds)}")
return taskData
# 모든 태스크 상태를 반환한다.
def getAllTaskState(self, taskId: str) -> StockTaskState:
markets = ["kospi", "kosdaq"]
resultDict: YearData = YearData(**{"yearData": dict()})
resultDict.yearData[taskId] = dict()
for market in markets:
data = self.mongod.getAllTaskState(taskId, market)
compDict: Dict = {}
count: Dict = {}
for one in data:
for idx, taskDate in enumerate(one["tasks"]):
if taskDate in compDict.keys():
if compDict[taskDate]["ret"] == 1 or one["tasksRet"][
idx] == 1:
compDict[taskDate] = {"date": taskDate, "ret": 1}
else:
year = taskDate[0:4]
if year in count.keys():
count[year] = count[year] + 1
else:
count[year] = 1
compDict[taskDate] = {
"date": taskDate,
"ret": one["tasksRet"][idx]
}
collect: List = list(compDict.values())
collect = sorted(collect, key=lambda x: x["date"])
resultDict.yearData[taskId][market] = StockTaskState(
**{
"taskStates": compDict,
"taskKeys": compDict.keys(),
"stocks": collect,
"years": count,
"market": market,
"taskId": taskId
})
return resultDict
