def main(client_secret_filepath, publish_address, subscribe_address):
if not _GOOGLE_API_INSTALLED:
logging.error('`google-api-python-client` required to use youtube. Install using `pip install google-api-python-client')
return
# TODO: Determine if this try/except pattern has become outdated
# with new `connect` methods being called rather than the old bind
try:
messaging = ZmqMessaging('youtube',
publish_address,
subscribe_address,
'youtube')
messaging.start_messaging()
except ZMQError:
return
# signal.signal(signal.SIGINT, handle_close)
# signal.signal(signal.SIGTERM, handle_close)
# handle_close = _handle_close(messaging)
scope = ['https://www.googleapis.com/auth/youtube',
'https://www.googleapis.com/auth/youtube.force-ssl',
'https://www.googleapis.com/auth/youtube.readonly']
youtube_api = _youtube_authentication(client_secret_filepath, scope)
parts = 'snippet'
livestream_response = youtube_api.liveBroadcasts().list(mine=True,
part=parts,
maxResults=1).execute()
live_chat_id = livestream_response.get('items')[0]['snippet']['liveChatId']
livechat_response = youtube_api.liveChatMessages().list(liveChatId=live_chat_id, part='snippet').execute()
next_token = livechat_response.get('nextPageToken')
polling_interval = livechat_response.get('pollingIntervalMillis')
polling_interval = _convert_to_seconds(polling_interval)
messaging.send_status('CONNECTED')
event_loop = asyncio.get_event_loop()
asyncio.ensure_future(_recv_loop(messaging,
youtube_api,
live_chat_id,
next_token,
polling_interval))
asyncio.ensure_future(_run(messaging,
youtube_api.liveChatMessages(),
live_chat_id))
atexit.register(_send_disconnect(messaging))
event_loop.run_forever()
messaging.send_status('DISCONNECTED')
def main(client_secret_filepath, publish_address, subscribe_address):
if not _GOOGLE_API_INSTALLED:
logging.error(
'`google-api-python-client` required to use youtube. Install using `pip install google-api-python-client'
)
return
# TODO: Determine if this try/except pattern has become outdated
# with new `connect` methods being called rather than the old bind
try:
messaging = ZmqMessaging('youtube', publish_address, subscribe_address,
'youtube')
messaging.start_messaging()
except ZMQError:
return
# signal.signal(signal.SIGINT, handle_close)
# signal.signal(signal.SIGTERM, handle_close)
# handle_close = _handle_close(messaging)
scope = [
'https://www.googleapis.com/auth/youtube',
'https://www.googleapis.com/auth/youtube.force-ssl',
'https://www.googleapis.com/auth/youtube.readonly'
]
youtube_api = _youtube_authentication(client_secret_filepath, scope)
parts = 'snippet'
livestream_response = youtube_api.liveBroadcasts().list(
mine=True, part=parts, maxResults=1).execute()
live_chat_id = livestream_response.get('items')[0]['snippet']['liveChatId']
livechat_response = youtube_api.liveChatMessages().list(
liveChatId=live_chat_id, part='snippet').execute()
next_token = livechat_response.get('nextPageToken')
polling_interval = livechat_response.get('pollingIntervalMillis')
polling_interval = _convert_to_seconds(polling_interval)
messaging.send_status('CONNECTED')
event_loop = asyncio.get_event_loop()
asyncio.ensure_future(
_recv_loop(messaging, youtube_api, live_chat_id, next_token,
polling_interval))
asyncio.ensure_future(
_run(messaging, youtube_api.liveChatMessages(), live_chat_id))
atexit.register(_send_disconnect(messaging))
event_loop.run_forever()
messaging.send_status('DISCONNECTED')
class WebSocket(WebSocketApp):
def __init__(self,
streamer_name,
namespace,
website_url,
publish_address,
subscribe_address,
service_name):
self.log = logging.getLogger(__name__)
self.log.setLevel(0)
if not _WEBSOCKET_INSTALLED:
self.log.error('Must install `websocket`')
if not _REQUESTS_INSTALLED:
self.log.error('Must install `requests')
self.messaging = ZmqMessaging(service_name,
publish_address,
subscribe_address,
service_name)
self.messaging.start_messaging()
self._streamer_name = streamer_name
self.namespace = namespace
self._website_url = website_url
self.log.info('Getting Socket IO key!')
self.key, heartbeat = self._connect_to_server_helper()
self.log.info('Socket IO key got!')
self.command_manager = AdapterCommandManager(self.messaging)
self._thread = Thread(target=self.handle_subscription)
self._thread.daemon = True
self._thread.start()
# alters URL to be more websocket...ie
self._website_socket = self._website_url.replace('http', 'ws')
self._website_socket += 'websocket/'
self.nick = None
super().__init__(self._website_socket + self.key,
on_open=self.on_open,
on_close=self.on_close,
on_message=self.on_message,
on_error=self.on_error)
def handle_subscription(self):
while True:
frame = self.messaging.sub_socket.recv_multipart()
message = decode_vex_message(frame)
if message.type == 'CMD':
self.command_manager.parse_commands(message)
if message.type == 'RSP':
data = {}
data['name'] = 'message'
data['args'] = [message.contents.get('response'),
self._streamer_name]
self.send_packet_helper(5, data)
def repeat_run_forever(self):
while True:
try:
self.run_forever()
except (KeyboardInterrupt, SystemExit):
break
except Exception as e:
self.log.info('Socket IO errors: {}'.format(e))
self.messaging.send_status('DISCONNECTED')
sleep(3)
key, _ = self._connect_to_server_helper()
self.url = self._website_socket + key
def _connect_to_server_helper(self):
r = requests.post(self._website_url)
params = r.text
# unused variables are connection_timeout and supported_formats
key, heartbeat_timeout, _, _ = params.split(':')
heartbeat_timeout = int(heartbeat_timeout)
return key, heartbeat_timeout
def on_open(self, *args):
logging.info('Websocket open!')
def on_close(self, *args):
logging.info('Websocket closed :(')
def on_message(self, *args):
message = args[1].split(':', 3)
key = int(message[0])
# namespace = message[2]
if len(message) >= 4:
data = message[3]
else:
data = ''
if key == 1 and args[1] == '1::':
self.send_packet_helper(1)
elif key == 1 and args[1] == '1::{}'.format(self.namespace):
self.send_packet_helper(5, data={'name': 'initialize'})
data = {'name': 'join',
'args': ['{}'.format(self._streamer_name)]}
self.send_packet_helper(5, data=data)
self.log.info('Connected to channel with socket io!')
self.messaging.send_status('CONNECTED')
elif key == 2:
self.send_packet_helper(2)
elif key == 5:
data = json.loads(data, )
if data['name'] == 'message':
message = data['args'][0]
sender = html.unescape(message['sender'])
message = html.unescape(message['text'])
self.messaging.send_message(author=sender, message=message)
elif data['name'] == 'join':
self.nick = data['args'][1]
def on_error(self, *args):
logging.error(args[1])
def disconnect(self):
callback = ''
data = ''
# '1::namespace'
self.send(':'.join([str(self.TYPE_KEYS['DISCONNECT']),
callback, self.namespace, data]))
def send_packet_helper(self,
type_key,
data=None):
if data is None:
data = ''
else:
data = json.dumps(data)
# NOTE: callbacks currently not implemented
callback = ''
message = ':'.join([str(type_key), callback, self.namespace, data])
self.send(message)
class JavascriptWebscraper:
def __init__(self,
url=None,
comment_element_id=None,
author_class_name=None,
message_class_name=None,
publish_address='',
subscribe_address='',
service_name=''):
"""
`comment_element_id` is the css element where all the comments are,
i.e., 'all-comments' for youtube
`author_class_name` is the css class which holds the comment author
username i.e., 'yt-user-name' for youtube
`message_class_name` is the css class which holds the comment test
ie., 'comment-text' for youtube
"""
self.messaging = ZmqMessaging(service_name,
publish_address,
subscribe_address,
'javascriptwebscraper')
self.messaging.start_messaging()
self.log = logging.getLogger(__name__)
self.log.setLevel(logging.NOTSET)
self.url = url
self._number_of_messages = 0
self.comment_element_id = comment_element_id
self.author_class_name = author_class_name
self.message_class_name = message_class_name
self._driver = None
self._kill = False
signal.signal(signal.SIGINT, self._exit_gracefully)
signal.signal(signal.SIGTERM, self._exit_gracefully)
def _exit_gracefully(self, *args, **kwargs):
if self._driver is not None:
self._driver.quit()
self._kill = True
def run_forever(self):
while True:
try:
self.log.info('Starting javascript scraper!')
self.run()
except selenium.common.exceptions.NoSuchElementException:
self.log.error('Youtube parameters wrong, shutting down :(')
break
except Exception as e:
if self._kill:
break
else:
self.log.exception('Javascript error!', e)
def run(self):
if self._driver:
self._driver.quit()
self._driver = None
self.log.info('starting up phantom javascript!')
self._driver = webdriver.PhantomJS()
# TODO: see if this is needed or not
self._driver.set_window_size(1000, 1000)
self._driver.get(self.url)
# NOTE: need some time for comments to load
self.log.info('youtube sleeping for 5 seconds!')
sleep(5)
self.log.info('youtube done sleeping')
all_comments = self._driver.find_element_by_id(self.comment_element_id)
# TODO: add in a signal here that all is connected!
# NOTE: make sure this is ok if using for anything other than youtube
comments = all_comments.find_elements_by_tag_name('li')
self._number_of_messages = len(comments)
self.messaging.send_status('CONNECTED')
while True:
sleep(1)
comments = all_comments.find_elements_by_tag_name('li')
comments_length = len(comments)
if comments_length > self._number_of_messages:
# NOTE: this number is intentionally NEGATIVE
msgs_not_parsed = self._number_of_messages - comments_length
self._number_of_messages = len(comments)
comments = comments[msgs_not_parsed:]
for comment in comments:
find_elem = comment.find_element_by_class_name
author = find_elem(self.author_class_name).text
message = find_elem(self.message_class_name).text
self.messaging.send_message(author=author, message=message)
self.messaging.send_status('DISCONNECTED')
class XMPPBot(ClientXMPP):
def __init__(self,
jid,
password,
room,
publish_address,
subscribe_address,
service_name,
bot_nick='EchoBot',
**kwargs):
# Initialize the parent class
if not _SLEEKXMPP_INSTALLED:
logging.error('must install sleekxmpp')
super().__init__(jid, password)
self.messaging = ZmqMessaging(service_name,
publish_address,
subscribe_address,
service_name)
self.messaging.start_messaging()
self.command_manager = AdapterCommandManager(self.messaging)
self.room = room
self.nick = bot_nick
self.log = logging.getLogger(__file__)
# One-shot helper method used to register all the plugins
self._register_plugin_helper()
self.add_event_handler("session_start", self.start)
self.add_event_handler("groupchat_message", self.muc_message)
self.add_event_handler('connected', self._connected)
self.add_event_handler('disconnected', self._disconnected)
self._thread = Thread(target=self.run)
self._thread.daemon = True
self._thread.start()
def run(self):
while True:
frame = self.messaging.sub_socket.recv_multipart()
message = decode_vex_message(frame)
if message.type == 'CMD':
self.command_manager.parse_commands(message)
elif message.type == 'RSP':
channel = message.contents.get('channel')
contents = message.contents.get('response')
self.send_message(channel, contents, mtype='groupchat')
def _disconnected(self, *args):
self.messaging.send_status('DISCONNECTED')
def _connected(self, *args):
self.messaging.send_status('CONNECTED')
def _register_plugin_helper(self):
"""
One-shot helper method used to register all the plugins
"""
# Service Discovery
self.register_plugin('xep_0030')
# XMPP Ping
self.register_plugin('xep_0199')
# Multiple User Chatroom
self.register_plugin('xep_0045')
def start(self, event):
self.log.info('starting xmpp')
self.send_presence()
self.plugin['xep_0045'].joinMUC(self.room,
self.nick,
wait=True)
self.get_roster()
def muc_message(self, msg):
self.messaging.send_message(author=msg['mucnick'],
message=msg['body'],
channel=msg['from'].bare)
class WebSocket(WebSocketApp):
def __init__(self, streamer_name, namespace, website_url, publish_address,
subscribe_address, service_name):
self.log = logging.getLogger(__name__)
self.log.setLevel(0)
if not _WEBSOCKET_INSTALLED:
self.log.error('Must install `websocket`')
if not _REQUESTS_INSTALLED:
self.log.error('Must install `requests')
self.messaging = ZmqMessaging(service_name, publish_address,
subscribe_address, service_name)
self.messaging.start_messaging()
self._streamer_name = streamer_name
self.namespace = namespace
self._website_url = website_url
self.log.info('Getting Socket IO key!')
self.key, heartbeat = self._connect_to_server_helper()
self.log.info('Socket IO key got!')
self.command_manager = AdapterCommandManager(self.messaging)
self._thread = Thread(target=self.handle_subscription)
self._thread.daemon = True
self._thread.start()
# alters URL to be more websocket...ie
self._website_socket = self._website_url.replace('http', 'ws')
self._website_socket += 'websocket/'
self.nick = None
super().__init__(self._website_socket + self.key,
on_open=self.on_open,
on_close=self.on_close,
on_message=self.on_message,
on_error=self.on_error)
def handle_subscription(self):
while True:
frame = self.messaging.sub_socket.recv_multipart()
message = decode_vex_message(frame)
if message.type == 'CMD':
self.command_manager.parse_commands(message)
if message.type == 'RSP':
data = {}
data['name'] = 'message'
data['args'] = [
message.contents.get('response'), self._streamer_name
]
self.send_packet_helper(5, data)
def repeat_run_forever(self):
while True:
try:
self.run_forever()
except (KeyboardInterrupt, SystemExit):
break
except Exception as e:
self.log.info('Socket IO errors: {}'.format(e))
self.messaging.send_status('DISCONNECTED')
sleep(3)
key, _ = self._connect_to_server_helper()
self.url = self._website_socket + key
def _connect_to_server_helper(self):
r = requests.post(self._website_url)
params = r.text
# unused variables are connection_timeout and supported_formats
key, heartbeat_timeout, _, _ = params.split(':')
heartbeat_timeout = int(heartbeat_timeout)
return key, heartbeat_timeout
def on_open(self, *args):
logging.info('Websocket open!')
def on_close(self, *args):
logging.info('Websocket closed :(')
def on_message(self, *args):
message = args[1].split(':', 3)
key = int(message[0])
# namespace = message[2]
if len(message) >= 4:
data = message[3]
else:
data = ''
if key == 1 and args[1] == '1::':
self.send_packet_helper(1)
elif key == 1 and args[1] == '1::{}'.format(self.namespace):
self.send_packet_helper(5, data={'name': 'initialize'})
data = {'name': 'join', 'args': ['{}'.format(self._streamer_name)]}
self.send_packet_helper(5, data=data)
self.log.info('Connected to channel with socket io!')
self.messaging.send_status('CONNECTED')
elif key == 2:
self.send_packet_helper(2)
elif key == 5:
data = json.loads(data, )
if data['name'] == 'message':
message = data['args'][0]
sender = html.unescape(message['sender'])
message = html.unescape(message['text'])
self.messaging.send_message(author=sender, message=message)
elif data['name'] == 'join':
self.nick = data['args'][1]
def on_error(self, *args):
logging.error(args[1])
def disconnect(self):
callback = ''
data = ''
# '1::namespace'
self.send(':'.join([
str(self.TYPE_KEYS['DISCONNECT']), callback, self.namespace, data
]))
def send_packet_helper(self, type_key, data=None):
if data is None:
data = ''
else:
data = json.dumps(data)
# NOTE: callbacks currently not implemented
callback = ''
message = ':'.join([str(type_key), callback, self.namespace, data])
self.send(message)
