def get_messages(self, channel):
"""
Thanks to the hipchat API, this function allows to retrieve all messages sent into a specific channel and,
save them into a dedicated queue.
It takes a single argument :
- channel : Hipchat channel name.
"""
# needed to avoid API rate limits
time.sleep(10)
try:
room = self.connection.get_room(channel.name)
except hypchat.requests.HttpNotFound as e:
logger.error("room %s at %s not found" %
(channel.name, self.server))
return None
except requests.exceptions.ConnectionError as e:
self.connection = hypchat.HypChat(self.token, endpoint=self.server)
room = self.connection.get_room(channel.name)
except hypchat.requests.HttpGatewayTimeout as e:
self.connection = hypchat.HypChat(self.token, endpoint=self.server)
room = self.connection.get_room(channel.name)
try:
messages = list(room.history(maxResults=90).contents())
except hypchat.requests.HttpGatewayTimeout as e:
logger.error(e)
return
old_cursor = channel.cursor
logger.info("Fetching message from %s (%s)" %
(channel.name, self.server))
scrap_counter.labels('hipcat', room['name']).inc()
for message in messages:
d = message['date']
message_date = datetime(d.year, d.month, d.day, d.hour, d.minute,
d.second, d.microsecond, None)
if message_date <= old_cursor:
continue
if message_date > old_cursor:
old_cursor = message_date
if type(message['from']) == unicode:
msg = "%s@%s | %s" % \
(message['from'], channel.name, message['message'])
else:
msg = "%s@%s | %s" % \
(message['from']['name'],
channel.name, message['message'])
if channel.include_pattern and \
not self.match_pattern(
channel.include_pattern, message['message']):
msg = 'Message skipped as not in include_pattern'
logger.info(msg)
channel.cursor = old_cursor
continue
self.enqueue(queue=channel.queue, message=msg)
read_msg_counter.labels('hipchat', room['name']).inc()
channel.cursor = old_cursor
def __init__(self, *args, **kwargs):
self.token = kwargs.pop('token')
self.endpoint = kwargs.pop('endpoint')
if self.endpoint is None:
self.hypchat = hypchat.HypChat(self.token)
else:
# We could always pass in the endpoint, with a default value if it's
# None, but this way we support hypchat<0.18
self.hypchat = hypchat.HypChat(self.token, endpoint=self.endpoint)
super().__init__(*args, **kwargs)
def __init__(self, *args, **kwargs):
self.token = kwargs.pop('token')
self.endpoint = kwargs.pop('endpoint')
self._cached_users = None
verify = kwargs.pop('verify')
if verify is None:
verify = True
if self.endpoint is None:
self.hypchat = hypchat.HypChat(self.token, verify=verify)
else:
# We could always pass in the endpoint, with a default value if it's
# None, but this way we support hypchat<0.18
self.hypchat = hypchat.HypChat(self.token, endpoint=self.endpoint, verify=verify)
super().__init__(*args, **kwargs)
def message_room(token, room, name, location):
hc_account = hypchat.HypChat(token)
hc_room = hc_account.get_room(room)
hc_room.topic('Fertilizer Update!')
hc_room.message(
name + ' has fertilized the ' + str(location) + ' (tree) (poo)',
'green', True, 'text')
def _create_connection(self):
""" Initialises a connection to the HipChat server """
# TODO this connection should be cached and handle reconnects, etc..
self.chatconn = None
try:
# HypChat() doesn't initiate a connection, invoke rooms() to test
conn = hypchat.HypChat(self.token)
conn.rooms()
self.chatconn = conn
except hypchat.requests.HttpUnauthorized:
logging.exception("HipChat Connection failed - Invalid API Token")
except Exception: # pylint:disable=broad-except
logging.exception("Unknown error")
def monitor_queue(queue_name,
host,
port,
room,
sleep_time=TIME_GAP_IN_SECONDS,
stop_once_empty=False):
sleep_time = int(sleep_time)
if sleep_time < 1:
raise ValueError(
'Sleep time too low. Please specify sleep time in seconds.')
if room is None:
raise ValueError('Please specify a valid room name')
logger.info('Connecting to hipchat')
hip_token = os.environ[
'HIP_TOKEN'] # Please specify your token using: export HIP_TOKEN=my_hipchat_token
hc = hypchat.HypChat(hip_token)
room = hc.get_room(room)
room.notification(
'Starting to monitor queue={queue} on port={port}'.format(
queue=queue_name, port=port),
color='yellow', # yellow, green, red, purple, gray, random
notify=True)
items_in_queue = APPROX_QUEUE_SIZE
last_notify_items_length = 0
last_items_length = 0
rd = qmon.status.redis_connection_cached(host, port)
while not stop_once_empty or items_in_queue:
items_in_queue = qmon.status.get_items_in_queue(queue_name,
redis_conn=rd,
default=0)
if last_items_length != items_in_queue:
message = '{} queue status: {:,d}'.format(queue_name,
items_in_queue)
notified = notify(message, items_in_queue,
last_notify_items_length, room)
if notified:
last_notify_items_length = items_in_queue
last_items_length = items_in_queue
time.sleep(sleep_time)
room.notification(
'All done for {queue}.'.format(queue=queue_name),
color='yellow', # yellow, green, red, purple, gray, random
notify=True)
def __init__(self, api_url, token, username, *rooms):
"""Create the bot. api_url and token are required
:api_url: TODO
:token: TODO
:rooms: All of the roomnames the hipbot needs to watch
"""
self._api_url = api_url
self._token = token
self._room_names = rooms
self._poll_amt = 10
self._username = username
self._rooms_map = {}
self._rooms_last_msg = {}
self._running = threading.Event()
self._log = logging.getLogger("hipbot")
self._hipchat = hypchat.HypChat(token, endpoint=api_url)
self._reactives = []
self._non_reactives = []
def start(token,
room='salt',
aliases=None,
valid_users=None,
valid_commands=None,
control=False,
trigger="!",
tag='salt/engines/hipchat/incoming',
api_key=None,
api_url=None,
max_rooms=None,
wait_time=None,
output_type='file',
outputter='nested'):
'''
Listen to Hipchat messages and forward them to Salt.
token
The HipChat API key. It requires a key for global usgae,
assigned per user, rather than room.
room
The HipChat room name.
aliases
Define custom aliases.
valid_users
Restrict access only to certain users.
valid_commands
Restrict the execution to a limited set of commands.
control
Send commands to the master.
trigger: ``!``
Special character that triggers the execution of salt commands.
tag: ``salt/engines/hipchat/incoming``
The event tag on the Salt bus.
api_url: ``https://api.hipchat.com``
The URL to the HipChat API.
.. versionadded:: 2017.7.0
max_rooms: ``1000``
Maximum number of rooms allowed to fetch. If set to 0,
it is able to retrieve the entire list of rooms.
wait_time: ``5``
Maximum wait time, in seconds.
output_type: ``file``
The type of the output. Choose bewteen:
- ``file``: save the output into a temporary file and upload
- ``html``: send the output as HTML
- ``code``: send the output as code
This can be overriden when executing a command, using the ``--out-type`` argument.
.. versionadded:: 2017.7.0
outputter: ``nested``
The format to display the data, using the outputters available on the CLI.
This argument can also be overriden when executing a command, using the ``--out`` option.
.. versionadded:: 2017.7.0
HipChat Example:
.. code-block:: text
! test.ping
! test.ping target=minion1
! test.ping --out=nested
! test.ping --out-type=code --out=table
'''
target_room = None
if __opts__.get('__role') == 'master':
fire_master = salt.utils.event.get_master_event(
__opts__,
__opts__['sock_dir']).fire_event
else:
fire_master = None
def fire(tag, msg):
'''
fire event to salt bus
'''
if fire_master:
fire_master(msg, tag)
else:
__salt__['event.send'](tag, msg)
def _eval_bot_mentions(all_messages, trigger):
''' yield partner message '''
for message in all_messages:
message_text = message['message']
if message_text.startswith(trigger):
fire(tag, message)
text = message_text.replace(trigger, '').strip()
yield message['from']['mention_name'], text
token = token or api_key
if not token:
raise UserWarning("Hipchat token not found")
runner_functions = sorted(salt.runner.Runner(__opts__).functions)
if not api_url:
api_url = _DEFAULT_API_URL
hipc = hypchat.HypChat(token, endpoint=api_url)
if not hipc:
raise UserWarning("Unable to connect to hipchat")
log.debug('Connected to Hipchat')
rooms_kwargs = {}
if max_rooms is None:
max_rooms = _DEFAULT_MAX_ROOMS
rooms_kwargs['max_results'] = max_rooms
elif max_rooms > 0:
rooms_kwargs['max_results'] = max_rooms
# if max_rooms is 0 => retrieve all (rooms_kwargs is empty dict)
all_rooms = hipc.rooms(**rooms_kwargs)['items']
for a_room in all_rooms:
if a_room['name'] == room:
target_room = a_room
if not target_room:
log.debug("Unable to connect to room %s", room)
# wait for a bit as to not burn through api calls
time.sleep(30)
raise UserWarning("Unable to connect to room {0}".format(room))
after_message_id = target_room.latest(maxResults=1)['items'][0]['id']
while True:
try:
new_messages = target_room.latest(
not_before=after_message_id)['items']
except hypchat.requests.HttpServiceUnavailable:
time.sleep(15)
continue
after_message_id = new_messages[-1]['id']
for partner, text in _eval_bot_mentions(new_messages[1:], trigger):
# bot summoned by partner
if not control:
log.debug("Engine not configured for control")
return
# Ensure the user is allowed to run commands
if valid_users:
if partner not in valid_users:
target_room.message('{0} not authorized to run Salt commands'.format(partner))
return
args = []
kwargs = {}
cmdline = salt.utils.args.shlex_split(text)
cmd = cmdline[0]
# Evaluate aliases
if aliases and isinstance(aliases, dict) and cmd in aliases.keys():
cmdline = aliases[cmd].get('cmd')
cmdline = salt.utils.args.shlex_split(cmdline)
cmd = cmdline[0]
# Parse args and kwargs
if len(cmdline) > 1:
for item in cmdline[1:]:
if '=' in item:
(key, value) = item.split('=', 1)
kwargs[key] = value
else:
args.append(item)
# Check for target. Otherwise assume *
if 'target' not in kwargs:
target = '*'
else:
target = kwargs['target']
del kwargs['target']
# Check for tgt_type. Otherwise assume glob
if 'tgt_type' not in kwargs:
tgt_type = 'glob'
else:
tgt_type = kwargs['tgt_type']
del kwargs['tgt_type']
# Check for outputter. Otherwise assume nested
if '--out' in kwargs:
outputter = kwargs['--out']
del kwargs['--out']
# Check for outputter. Otherwise assume nested
if '--out-type' in kwargs:
output_type = kwargs['--out-type']
del kwargs['--out-type']
# Ensure the command is allowed
if valid_commands:
if cmd not in valid_commands:
target_room.message('Using {0} is not allowed.'.format(cmd))
return
ret = {}
if cmd in runner_functions:
runner = salt.runner.RunnerClient(__opts__)
ret = runner.cmd(cmd, arg=args, kwarg=kwargs)
# Default to trying to run as a client module.
else:
local = salt.client.LocalClient()
ret = local.cmd('{0}'.format(target), cmd, args, kwargs, tgt_type='{0}'.format(tgt_type))
nice_args = (' ' + ' '.join(args)) if args else ''
nice_kwargs = (' ' + ' '.join('{0}={1}'.format(key, value) for (key, value) in six.iteritems(kwargs))) \
if kwargs else ''
message_string = '@{0} Results for: {1}{2}{3} on {4}'.format(partner,
cmd,
nice_args,
nice_kwargs,
target)
if output_type == 'html':
_publish_html_message(token, room, ret, message=message_string, outputter=outputter, api_url=api_url)
elif output_type == 'code':
_publish_code_message(token, room, ret, message=message_string, outputter=outputter, api_url=api_url)
else:
tmp_path_fn = salt.utils.files.mkstemp()
with salt.utils.files.fopen(tmp_path_fn, 'w+') as fp_:
salt.utils.json.dump(ret, fp_, sort_keys=True, indent=4)
_publish_file(token, room, tmp_path_fn, message=message_string, api_url=api_url)
salt.utils.files.safe_rm(tmp_path_fn)
time.sleep(wait_time or _DEFAULT_SLEEP)
def connect(self):
"""
This function allows to create a connexion to a hipchat server through his API.
"""
return hypchat.HypChat(self.token, endpoint=self.server)
def __init__(self, key, path):
hypchat.Linker = HipDump.BasicLinker
self.hc = hypchat.HypChat(key)
self.path = path
HipDump.mkdir(path)
def get_instance():
global instance
if instance is None:
instance = hypchat.HypChat(get_hipchat_token())
return instance
#Listener for webhook messages from HipChat
import nss
import hypchat
import json
#SENSITIVE INFORMATION: Personal key
hc = hypchat.HypChat('6gHVzFsQA5wiZOkbeSFZiG0KKlL5tOL4Fa3UKnVr')
rm = hc.get_room('Tech Nerd Chat')
class EchoManager(nss.HTTPManager):
def Handle(self, client, req, server):
#print req.payload
try:
obj = json.loads(req.payload)
except ValueError:
raise nss.HTTPError(500, 'No JSON data present')
rm.notification('<' + obj['item']['room']['name'] + '> ' +
obj['item']['message']['message'],
format='text')
return nss.HTTPResponse('', 204)
def error():
if auth.is_logged_in():
user_id, username = auth.user.id, auth.user.email
else:
user_id, username = '******', '(Not logged-in)'
if IS_DEV:
ticket_url = URL('admin',
'default',
'ticket',
args=[request.vars.ticket],
scheme=True if request.is_local else 'https',
host=True)
else:
ticket_url = URL('web',
'admin',
'default',
args=['ticket', request.vars.ticket],
scheme='https',
host=True)
if not IS_TEST or os.environ.get(
'CIRCLECI'
): # If environment is test (unless CI), don't do Hipchat/email
server_type = "CircleCI" if os.environ.get(
'CIRCLECI') else TalentPropertyManager.get_env().upper()
# HipChat
import urllib2
import hypchat
try:
# If on local, private-message the user. Otherwise, send it to Codebox.
hc = hypchat.HypChat(HIPCHAT_TOKEN)
if TalentPropertyManager.get_env() == 'dev':
hipchat_user = hc.get_user(TalentPropertyManager.get_email())
"""
:type: hypchat.restobject.User
"""
if not hipchat_user:
logger.error(
"I tried sending you a Hipchat message at %s, but no Hipchat user with that email exists! Make sure that email belongs to a getTalent Hipchat user",
TalentPropertyManager.get_email())
else:
hipchat_message = """You got a localhost error!<br /> <a href="%s">Error link</a> <br /> Username: %s""" % \
(ticket_url, username)
hipchat_user.message(message=hipchat_message,
message_format='html',
notify=True)
else:
hipchat_codebox_room = hc.get_room("Codebox")
"""
:type: hypchat.restobject.Room
"""
hipchat_message = """Error occurred on Talent Web %s <br /> <a href="%s">Error link</a> <br /> Username: %s""" % \
(server_type, ticket_url, username)
hipchat_codebox_room.notification(message=hipchat_message,
color='red',
notify=True,
format='html')
# hipster.message_room(room_id='Codebox',
# message_from='Talent Web %s' % server_type,
# message=hipchat_message,
# message_format='html',
# color='red',
# notify=True)
except urllib2.HTTPError:
logger.exception(
"Received exception posting error to HipChat. Error link: %s",
ticket_url)
# Email
email_message = """
Error occurred on Talent Web %s
Link to error ticket: %s \n
User ID: %s\n
Username: %s\n
URL: %s\n
""" % (server_type, ticket_url, user_id, username,
request.vars.request_url)
from TalentReporting import email_error_to_admins
email_error_to_admins(email_message, subject="Error ticket")
response.status = "%s Internal Service Error" % request.vars.code
if request.vars.request_url and '.json' in request.vars.request_url:
response.headers['Content-Type'] = 'application/json'
return json.dumps({
'error': {
'message': 'Internal server error',
'ticket': request.vars.ticket or ""
}
})
# request_url = request.vars.request_url
# if request_url:
# args = request_url.split('/')
# if args[0] == '':
# args.pop(0)
# error_data = dict(error=dict(system_error=0), ticket=ticket_url)
# if '.xml' in args[-1]:
# response.headers['Content-Type'] = 'text/xml'
# return response.render('generic.xml', error_data)
# elif '.json' in args[-1]:
# response.headers['Content-Type'] = 'application/json'
# return response.json(error_data)
return dict(ticket_url=ticket_url)
def setUpHipChat(self): self.hipchat = hypchat.HypChat(self.access_token)
def start(token,
room='salt',
aliases=None,
valid_users=None,
valid_commands=None,
control=False,
trigger="!",
tag='salt/engines/hipchat/incoming',
api_key=None,
api_url=None,
max_rooms=None,
wait_time=None):
'''
Listen to Hipchat messages and forward them to Salt
'''
target_room = None
if __opts__.get('__role') == 'master':
fire_master = salt.utils.event.get_master_event(
__opts__, __opts__['sock_dir']).fire_event
else:
fire_master = None
def fire(tag, msg):
'''
fire event to salt bus
'''
if fire_master:
fire_master(msg, tag)
else:
__salt__['event.send'](tag, msg)
def _eval_bot_mentions(all_messages, trigger):
''' yield partner message '''
for message in all_messages:
message_text = message['message']
if message_text.startswith(trigger):
fire(tag, message)
text = message_text.replace(trigger, '').strip()
yield message['from']['mention_name'], text
token = token or api_key
if not token:
raise UserWarning("Hipchat token not found")
runner_functions = sorted(salt.runner.Runner(__opts__).functions)
if not api_url:
api_url = _DEFAULT_API_URL
hipc = hypchat.HypChat(token, endpoint=api_url)
if not hipc:
raise UserWarning("Unable to connect to hipchat")
log.debug('Connected to Hipchat')
rooms_kwargs = {}
if max_rooms is None:
max_rooms = _DEFAULT_MAX_ROOMS
rooms_kwargs['max_results'] = max_rooms
elif max_rooms > 0:
rooms_kwargs['max_results'] = max_rooms
# if max_rooms is 0 => retrieve all (rooms_kwargs is empty dict)
all_rooms = hipc.rooms(**rooms_kwargs)['items']
for a_room in all_rooms:
if a_room['name'] == room:
target_room = a_room
if not target_room:
log.debug("Unable to connect to room {0}".format(room))
# wait for a bit as to not burn through api calls
time.sleep(30)
raise UserWarning("Unable to connect to room {0}".format(room))
after_message_id = target_room.latest(maxResults=1)['items'][0]['id']
while True:
try:
new_messages = target_room.latest(
not_before=after_message_id)['items']
except hypchat.requests.HttpServiceUnavailable:
time.sleep(15)
continue
after_message_id = new_messages[-1]['id']
for partner, text in _eval_bot_mentions(new_messages[1:], trigger):
# bot summoned by partner
if not control:
log.debug("Engine not configured for control")
return
# Ensure the user is allowed to run commands
if valid_users:
if partner not in valid_users:
target_room.message(
'{0} not authorized to run Salt commands'.format(
partner))
return
args = []
kwargs = {}
cmdline = salt.utils.shlex_split(text)
cmd = cmdline[0]
# Evaluate aliases
if aliases and isinstance(aliases, dict) and cmd in aliases.keys():
cmdline = aliases[cmd].get('cmd')
cmdline = salt.utils.shlex_split(cmdline)
cmd = cmdline[0]
# Parse args and kwargs
if len(cmdline) > 1:
for item in cmdline[1:]:
if '=' in item:
(key, value) = item.split('=', 1)
kwargs[key] = value
else:
args.append(item)
# Check for target. Otherwise assume *
if 'target' not in kwargs:
target = '*'
else:
target = kwargs['target']
del kwargs['target']
# Check for tgt_type. Otherwise assume glob
if 'tgt_type' not in kwargs:
tgt_type = 'glob'
else:
tgt_type = kwargs['tgt_type']
del kwargs['tgt_type']
# Ensure the command is allowed
if valid_commands:
if cmd not in valid_commands:
target_room.message(
'Using {0} is not allowed.'.format(cmd))
return
ret = {}
if cmd in runner_functions:
runner = salt.runner.RunnerClient(__opts__)
ret = runner.cmd(cmd, arg=args, kwarg=kwargs)
# Default to trying to run as a client module.
else:
local = salt.client.LocalClient()
ret = local.cmd('{0}'.format(target),
cmd,
args,
kwargs,
tgt_type='{0}'.format(tgt_type))
tmp_path_fn = salt.utils.files.mkstemp()
with salt.utils.fopen(tmp_path_fn, 'w+') as fp_:
fp_.write(json.dumps(ret, sort_keys=True, indent=4))
message_string = '@{0} Results for: {1} {2} {3} on {4}'.format(
partner, cmd, args, kwargs, target)
_publish_file(token,
room,
tmp_path_fn,
message=message_string,
api_url=api_url)
salt.utils.safe_rm(tmp_path_fn)
time.sleep(wait_time or _DEFAULT_SLEEP)
def convert_room_data(raw_data: List[ZerverFieldsT],
subscriber_handler: SubscriberHandler,
stream_id_mapper: IdMapper,
user_id_mapper: IdMapper,
realm_id: int,
api_token: Optional[str] = None) -> List[ZerverFieldsT]:
flat_data = [d['Room'] for d in raw_data]
def get_invite_only(v: str) -> bool:
if v == 'public':
return False
elif v == 'private':
return True
else:
raise Exception('unexpected value')
streams = []
for in_dict in flat_data:
now = int(timezone_now().timestamp())
stream_id = stream_id_mapper.get(in_dict['id'])
invite_only = get_invite_only(in_dict['privacy'])
stream = build_stream(
date_created=now,
realm_id=realm_id,
name=in_dict['name'],
description=in_dict['topic'],
stream_id=stream_id,
deactivated=in_dict['is_archived'],
invite_only=invite_only,
)
if invite_only:
users: Set[int] = {
user_id_mapper.get(key)
for key in in_dict['members'] if user_id_mapper.has(key)
}
if user_id_mapper.has(in_dict['owner']):
owner = user_id_mapper.get(in_dict['owner'])
users.add(owner)
else:
users = set()
if api_token is not None:
hc = hypchat.HypChat(api_token)
room_data = hc.fromurl('{}/v2/room/{}/member'.format(
hc.endpoint, in_dict['id']))
for item in room_data['items']:
hipchat_user_id = item['id']
zulip_user_id = user_id_mapper.get(hipchat_user_id)
users.add(zulip_user_id)
if users:
subscriber_handler.set_info(
stream_id=stream_id,
users=users,
)
# unmapped fields:
# guest_access_url: no Zulip equivalent
# created: we just use "now"
# participants: no good sample data
streams.append(stream)
return streams
def __init__(self, *args, **kwargs):
self.token = kwargs.pop('token')
self.debug = kwargs.pop('debug')
self.hypchat = hypchat.HypChat(self.token)
super().__init__(*args, **kwargs)
