def __init__(self, host, consumer_exchange, consumer_routing_keys,
consumer_queue_name, producer_exchange, producer_routing_key):
try:
self.handler = Handler()
log.info("init consumer for host=%s, exchange=%s, routing_keys=%s" %
(host, consumer_exchange, repr(consumer_routing_keys)))
self.host = host
self.exchange = consumer_exchange
self.routing_keys = consumer_routing_keys
self.queue_name = consumer_queue_name
if not consumer_routing_keys:
log.error("routing_keys can't be null...")
sys.exit(1)
# init producer
self.resend_producer = Producer(self.host,
producer_exchange,
producer_routing_key
)
# init pika connection
self._connection = self._connect()
self._closing = False
self._consumer_tag = None
except Exception as e:
log.error("Unexpected error:%r", e)
def handle(args: Namespace):
"""
Extracts all the compressed files in the data directory so they can be queried
@param args: The arguments for the extract handler
"""
Handler._directory_walk(dir_path=args.data_path,
action=ExtractHandler._extract_file)
def __init__(self, cli_args: argparse.Namespace, config: Dict[str, Any],
**kwargs: Any):
self.args = cli_args
self.config = config
self.prefixes = {self.config["default-prefix"]}
self.owners = set(self.config["owners"])
self.repo = git.Repo()
self._handler = Handler(self)
super().__init__(**kwargs)
# prevents bot from initislizing on reconnect
self._first_on_ready = True
def startProcess(conf, sock):
from handler.executor import Executor
from handler.handler import Handler
from server.server import Server
from asyncio import get_event_loop
loop = get_event_loop()
loop.add_signal_handler(signal.SIGINT, closeHandler, loop)
try:
info('running source with PID {}'.format(str(getpid())))
handler = Handler(conf.root_dir, Executor(conf.root_dir))
for _ in range(0, int(conf.threads)):
server = Server(config=conf, loop=loop, handler=handler, sock=sock)
loop.create_task(server.launch_server())
loop.run_forever()
except Exception:
loop.stop()
info('stop loop {}'.format(getpid()))
finally:
loop.stop()
info('stop loop {}'.format(getpid()))
def run(self):
handler = Handler(self.settings, self.middlewares)
handler.init()
try:
self.is_running = True
handler.run()
except KeyboardInterrupt:
handler.shutdown()
if Database.db and not Database.db.is_closed():
Database.db.close()
class Run(object):
def __init__(self):
self.cal = GoogleCalendar()
self.bot = Bot()
self.parser = Parser()
self.handler = Handler()
self.time_zone = 'Europe/Moscow'
# start_data_time = datetime.datetime(2017, 8, 14, hour=0, minute=30)
# cal.add_single_event('Событие1', '', start_data_time)
def get_events(self):
events = [
it[0].strftime("%d %B %H:%M") + ': ' + it[1]
for it in self.cal.get_events(10)
]
return '/n'.join(events)
def text_handler(self, text):
error, result = self.parser.parse(text, self.time_zone)
if error == '':
return self.handler.handle(result.intent_name, result.parameters,
self.time_zone)
else:
return error
def create(self):
locale.setlocale(locale.LC_ALL, ('RU', 'UTF8'))
cred_dir = os.path.join(os.path.expanduser('~'), '.config',
'credentials')
if not os.path.exists(cred_dir):
os.makedirs(cred_dir)
config_file = os.path.join(cred_dir, 'bot.json')
config = json.load(open(config_file, 'r'))
self.cal.create()
self.bot.create(config['telegram'], self.text_handler)
self.parser.create(config['apiai'])
self.handler.create(config['handler'])
return self
def run(self):
self.bot.run()
def configure_parser(parser: ArgumentParser):
"""
Configures the arguments for the inspect handler
@param parser: The parser to configure
"""
Handler.configure_parser(parser)
parser.add_argument(VERBOSE_ARG,
action=STORE_TRUE_ACTION,
required=False,
help=VERBOSE_ARG_HELP)
parser.add_argument(KEY_WORDS_ARG,
nargs='+',
action=STORE_ACTION,
required=True,
help=KEY_WORDS_ARG_HELP)
def run_app():
#initialize our app class with empty values for taxon and protein as we will set these after we decide what the user wishes to run and what advanced settings will be changed before setting taxon and protein query. The classes need to beinitialized in order to save any changes to the state
app = App("", "")
#here, the welcome information is printed, a list of what the user may want to run is displayed and features can be turned on or off, or advanced settings like bootstrapping, esearch retmax, max accessions processed can all be tweaked. Handler.path_list booleans are turned on or off here which will be used to navigate the program's steps in handle()
handler = Handler.welcome(app)
#have the user assign their taxon and protein choices, user input for this is handled in the user_input class
app.taxon_query = "taxonomy"
app.protein_query = "protein"
#run the function above with our basal set parameters
handle(app, handler)
def main():
config = Config()
config.read(CONFIG_FILE_NAME)
print(config)
handler = Handler(config.document_root)
server = Server(config, handler)
try:
print(f'Starting server\n')
server.launch()
except KeyboardInterrupt:
server.stop()
def handle(args: Namespace):
"""
Recursively looks through a directory and its sub directories searching for key words in csv files and their
file paths
@param args: The arguments for the inspect handler, including key words to search for
"""
assert InspectHandler._csv_objects is None
assert InspectHandler._key_words is None
assert InspectHandler._data_path is None
InspectHandler._csv_objects = {}
InspectHandler._key_words = args.key_words
InspectHandler._data_path = args.data_path
Handler._directory_walk(dir_path=args.data_path,
action=InspectHandler._inspect_file)
# Print out all the info in the csv objects
info: list = InspectHandler._get_info(verbose=args.verbose)
for output_line in info:
print(output_line)
def __init__(
self,
cli_args: argparse.Namespace,
config: Dict[str, Any],
**kwargs: Any,
):
self.args = cli_args
self.config = config
self.prefixes = {self.config["default-prefix"]}
self.owners = set(self.config["owners"])
self.repo = git.Repo()
self._handler = Handler(self)
super().__init__(**kwargs)
class Consumer(object):
"""
A consumer implementation that consumes specified
exchange and routing_keys
"""
def _on_connection_open(self, unused_connection):
"""This method is called by pika once the connection to RabbitMQ has
been established. It passes the handle to the connection object in
case we need it, but in this case, we'll just mark it unused.
:type unused_connection: pika.SelectConnection
"""
log.info("connection established")
#self._add_on_connection_close_callback()
self._open_channel()
def _close_channel(self):
"""Call to close the channel with RabbitMQ cleanly by issuing the
Channel.Close RPC command.
"""
log.info('Closing the channel')
self._channel.close()
def _open_channel(self):
"""Open a new channel with RabbitMQ by issuing the Channel.Open RPC
command. When RabbitMQ responds that the channel is open, the
on_channel_open callback will be invoked by pika.
"""
self._connection.channel(on_open_callback=self._on_channel_open)
def _on_channel_open(self, channel):
"""This method is invoked by pika when the channel has been opened.
The channel object is passed in so we can make use of it.
Since the channel is now open, we'll declare the exchange to use.
:param pika.channel.Channel channel: The channel object
"""
try:
log.info("connection channel created")
self._channel = channel
self._setup_exchange(self.exchange)
log.info("exchange %s declared as %s" % (self.exchange, 'topic'))
except Exception as e:
log.error("Unexpected error:%r", e)
raise e
def _setup_exchange(self, exchange_name):
"""Setup the exchange on RabbitMQ by invoking the Exchange.Declare RPC
command. When it is complete, the on_exchange_declareok method will
be invoked by pika.
:param str|unicode exchange_name: The name of the exchange to declare
"""
try:
log.info('Declaring exchange %s', exchange_name)
self._channel.exchange_declare(callback=self._on_exchange_declareok,
exchange=self.exchange,
type='topic',
durable=True,
auto_delete=False,)
except Exception as e:
log.error("Unexpected error:%r", e)
raise e
def _on_exchange_declareok(self, unused_frame):
"""Invoked by pika when RabbitMQ has finished the Exchange.Declare RPC
command.
:param pika.Frame.Method unused_frame: Exchange.DeclareOk response frame
"""
log.info('Exchange declared')
self._setup_queue(self.queue_name)
def _setup_queue(self, queue_name):
"""Setup the queue on RabbitMQ by invoking the Queue.Declare RPC
command. When it is complete, the on_queue_declareok method will
be invoked by pika.
:param str|unicode queue_name: The name of the queue to declare.
"""
try:
log.info('Declaring queue %s', queue_name)
# make queue and message durability
self._channel.queue_declare(queue=self.queue_name, durable=True,
exclusive=False, auto_delete=False,
callback=self._on_queue_declaredok)
except Exception as e:
log.error("Unexpected error:%r", e)
raise e
def _on_queue_declaredok(self, method_frame):
"""Called when RabbitMQ has told us our Queue has been declared,
frame is the response from RabbitMQ"""
try:
# bingding exchange and queue
for routing_key in self.routing_keys:
log.info("binding queue %s to exchange %s by routing_key %s" %
(self.queue_name, self.exchange, routing_key))
self._channel.queue_bind(exchange=self.exchange,
queue=self.queue_name,
routing_key=routing_key,
callback=self._on_bindok)
except Exception as e:
log.error("Unexpected error:%r", e)
raise e
def _on_bindok(self, unused_frame):
"""Invoked by pika when the Queue.Bind method has completed. At this
point we will start consuming messages by calling start_consuming
which will invoke the needed RPC commands to start the process.
:param pika.frame.Method unused_frame: The Queue.BindOk response frame
"""
log.info('Queue bound')
self._start_consuming()
def _start_consuming(self):
"""This method sets up the consumer by first calling
add_on_cancel_callback so that the object is notified if RabbitMQ
cancels the consumer. It then issues the Basic.Consume RPC command
which returns the consumer tag that is used to uniquely identify the
consumer with RabbitMQ. We keep the value to use it when we want to
cancel consuming. The on_message method is passed in as a callback pika
will invoke when a message is fully received.
"""
try:
log.info('Issuing consumer related RPC commands')
self._add_on_cancel_callback()
#self.channel.basic_qos(prefetch_count=1)
self._consumer_tag = self._channel.basic_consume(self.consume_msg_callback,
queue=self.queue_name)
except Exception as e:
log.error("Unexpected error:%r", e)
raise e
def _add_on_cancel_callback(self):
"""Add a callback that will be invoked if RabbitMQ cancels the consumer
for some reason. If RabbitMQ does cancel the consumer,
on_consumer_cancelled will be invoked by pika.
"""
log.info('Adding consumer cancellation callback')
self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
def on_consumer_cancelled(self, method_frame):
"""Invoked by pika when RabbitMQ sends a Basic.Cancel for a consumer
receiving messages.
:param pika.frame.Method method_frame: The Basic.Cancel frame
"""
log.info('Consumer was cancelled remotely, shutting down: %r', method_frame)
if self._channel:
self._channel.close()
def _connect(self):
"""This method connects to RabbitMQ, returning the connection handle.
When the connection is established, the on_connection_open method
will be invoked by pika.
:rtype: pika.SelectConnection
"""
parameters = pika.ConnectionParameters(host=self.host, heartbeat_interval=30)
while True:
try:
log.info('Connecting to %s', self.host)
return pika.SelectConnection(parameters, self._on_connection_open, stop_ioloop_on_close=False)
except Exception:
log.warning('%s cannot connect', self.host, exc_info=True)
time.sleep(10)
continue
def __init__(self, host, consumer_exchange, consumer_routing_keys,
consumer_queue_name, producer_exchange, producer_routing_key):
try:
self.handler = Handler()
log.info("init consumer for host=%s, exchange=%s, routing_keys=%s" %
(host, consumer_exchange, repr(consumer_routing_keys)))
self.host = host
self.exchange = consumer_exchange
self.routing_keys = consumer_routing_keys
self.queue_name = consumer_queue_name
if not consumer_routing_keys:
log.error("routing_keys can't be null...")
sys.exit(1)
# init producer
self.resend_producer = Producer(self.host,
producer_exchange,
producer_routing_key
)
# init pika connection
self._connection = self._connect()
self._closing = False
self._consumer_tag = None
except Exception as e:
log.error("Unexpected error:%r", e)
def start_consuming(self):
""" main consuming loop """
#self.channel.start_consuming()
# Loop so we can communicate with RabbitMQ
self._connection.ioloop.start()
def _on_cancelok(self, unused_frame):
"""This method is invoked by pika when RabbitMQ acknowledges the
cancellation of a consumer. At this point we will close the channel.
This will invoke the on_channel_closed method once the channel has been
closed, which will in-turn close the connection.
:param pika.frame.Method unused_frame: The Basic.CancelOk frame
"""
log.info('RabbitMQ acknowledged the cancellation of the consumer')
self._close_channel()
def stop_consuming(self):
"""Tell RabbitMQ that you would like to stop consuming by sending the
Basic.Cancel RPC command.
"""
if self._channel:
log.info('Sending a Basic.Cancel RPC command to RabbitMQ')
self._channel.basic_cancel(self._on_cancelok, self._consumer_tag)
def consume_msg_callback(self, ch, method, properties, body):
"""
send msg to jpush, if failed, push back to MQ for retry
"""
try:
"""
*** IMPLEMENT the handler 'pre_process' and 'do_process' func ***
"""
log.info(" [x] %r:%r", method.routing_key, body,)
if self.handler.pre_process(body):
if self.handler.do_process():
self.__reply(self.handler._ret_msg)
except Exception as e:
log.error("Unexpected error:%r", e)
finally:
# MUST have this ACK statement, otherwise the mem will be filled up by
# RabbitMQ's unack queue. USE
# sudo rabbitmqctl list_queues name messages_ready messages_unacknowledged
# to check this may came bug
log.info('Acknowledging message %s', method.delivery_tag)
self._channel.basic_ack(delivery_tag=method.delivery_tag)
#ch.basic_ack(delivery_tag=method.delivery_tag)
def __reply(self, message):
""" reply the message """
log.info("reply:%s", message)
self.responser.produce_send_msg(message)
def close(self):
"""Cleanly shutdown the connection to RabbitMQ by stopping the consumer
with RabbitMQ. When RabbitMQ confirms the cancellation, on_cancelok
will be invoked by pika, which will then closing the channel and
connection. The IOLoop is started again because this method is invoked
when CTRL-C is pressed raising a KeyboardInterrupt exception. This
exception stops the IOLoop which needs to be running for pika to
communicate with RabbitMQ. All of the commands issued prior to starting
the IOLoop will be buffered but not processed.
"""
log.info('Stopping')
self._closing = True
self.stop_consuming()
self._connection.ioloop.start()
log.info('Stopped')
from config_reader.config_reader import read_file
from server.server import Server
from handler.handler import Handler
if __name__ == '__main__':
config = read_file('httpd.conf')
# config = read_file('test_conf.conf')
print(config)
handler = Handler(config['files'])
server = Server(config['host'], config['port'], handler)
try:
server.start(int(config['cpu']), int(config['threads']))
print('Server started')
except KeyboardInterrupt:
server.stop()
print('Server stopped')
# !/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time : 2018/7/21 10:32
# @Author : zhe
# @FileName : twisted_server.py
# @Project : PyCharm
from twisted.internet import reactor, threads, defer
from twisted.internet.protocol import Protocol, Factory
from handler.handler import Handler
from config import server_port
from socket_fun import Logger
handler = Handler(to_send_enable=True)
logger = Logger()
class MyProtocal(Protocol):
def connectionMade(self):
logger.std_log('客户端连接 %s:%s' % self.transport.client)
def connectionLost(self, reason):
if self in handler.login_client:
handler.logout(self)
logger.std_log('客户端断开 %s:%s' % self.transport.client)
def dataReceived(self, data):
d = self.handler(data)
d.addCallback(self.callback)
def handler(self, line):
def __init__(self):
self.cal = GoogleCalendar()
self.bot = Bot()
self.parser = Parser()
self.handler = Handler()
self.time_zone = 'Europe/Moscow'
class TarakaniaRPG(discord.AutoShardedClient):
def __init__(self, cli_args: argparse.Namespace, config: Dict[str, Any],
**kwargs: Any):
self.args = cli_args
self.config = config
self.prefixes = {self.config["default-prefix"]}
self.owners = set(self.config["owners"])
self.repo = git.Repo()
self._handler = Handler(self)
super().__init__(**kwargs)
# prevents bot from initislizing on reconnect
self._first_on_ready = True
def run(self, *args: Any, **kwargs: Any) -> None:
if self.args.production:
token = self.config["discord-token"]
else:
token = self.config["discord-beta-token"]
if not token:
log.fatal(
f"Discord {'' if self.args.production else 'beta '}token is missing from config"
)
sys.exit(1)
super().run(token, *args, **kwargs)
async def on_ready(self) -> None:
if not self._first_on_ready:
log.info("Reconnected")
return
self._first_on_ready = False
await start_updater(self)
self.redis = await create_redis_pool(self.config["redis"])
self.pg = await create_pg_connection(self.config["postgresql"])
await self._handler.prepare_prefixes()
await self._handler.load_all_commands()
log.info(
f"Running in {'production' if self.args.production else 'debug'} mode"
)
for line in TARAKANIA_RPG_ASCII_ART.split("\n"):
log.info(line)
log.info(
f"Ready to operate as {self.user}. Prefix: {self.config['default-prefix']}"
)
async def on_message(self, msg: discord.Message) -> None:
await self._handler.process_message(msg)
async def on_message_edit(self, old: discord.Message,
new: discord.Message) -> None:
if old.pinned != new.pinned:
return
self._handler.cancel_command(new.id)
await self._clear_responses(new.id)
await self._handler.process_message(new)
async def on_raw_message_edit(
self, event: discord.RawMessageUpdateEvent) -> None:
if "content" not in event.data: # embed update
return
if event.cached_message is not None: # cached, on_message_edit will be called
return
channel = self.get_channel(event.channel_id)
if channel is None:
return
try:
# might be an issue if bot will get big, global message cache has limited
# size
message = await channel.fetch_message(event.message_id)
except discord.HTTPException:
return
# call this ourselves since message was not cached
#
# pin/unpin events will trigger this, there is no easy way to fix it
await self.on_message_edit(message, message)
async def on_raw_message_delete(
self, event: discord.RawMessageDeleteEvent) -> None:
self._handler.cancel_command(event.message_id)
await self._clear_responses(event.message_id)
async def on_error(self, event: str, *args: Any, **kwargs: Any) -> None:
log.exception(f"Error during event {event} execution")
async def _clear_responses(self, message_id: int) -> None:
address = f"message_responses:{message_id}"
entries = await self.redis.execute("LRANGE", address, 0, -1)
await self.redis.execute("DEL", address)
for entry in reversed(entries):
type_, data = entry.decode().split(":", 1)
if type_ == "message":
channel_id, message_id = data.split(":")
with suppress(discord.HTTPException):
await self.http.delete_message(int(channel_id),
int(message_id))
elif type_ == "reaction":
channel_id, message_id, reaction = data.split(":", 2)
if reaction.isdigit():
e = self.get_emoji(int(reaction))
if e is None:
continue
emote = f"{'a:' if e.animated else ''}{e.name}:{e.id}"
else:
emote = reaction
with suppress(discord.HTTPException):
await self.http.remove_own_reaction(
int(channel_id), int(message_id), emote)
else:
capture_message(
f"Unknown response type: {type_}. Entry: {entry}")