class Audit:
def __init__(self, hostname='localhost', port='5672',
userid='', password='', virtual_host='graylog',
exchange=None):
self.hostname = hostname
self.port = port
self.userid = userid
self.password = password
self.virtual_host = virtual_host
self.connection = BrokerConnection(virtual_host=virtual_host)
self.exchange_setup = exchange or ExchangeSetup()
def custom_exchange(self, exchange, exchange_type, routing_key, queue):
"""Broker exchange can be set after the object has been instantiated.
Args:
exchange (str): Exchange name
exchange_type (str): AMQP exchange type, see your broker manual
routing_key (str)
queue (str)
"""
self.exchange_setup.exchange = exchange
self.exchange_setup.exchange_type = exchange_type
self.exchange_setup.routing_key = routing_key
self.exchange_setup.queue = queue
def log(self, message):
"""Pushes argument object to message broker.
Args:
message (json/gelp): Message can depend on third-party log software
Graylog uses gelp format: https://www.graylog.org/resources/gelf/
"""
if (type(message) is not str) or (message == ''):
print 'Unable to log empty message'
return False
if len(message) > 8192:
print 'Message size too large'
return False
self.connection.connect()
channel = self.connection.channel()
exchange = Exchange(self.exchange_setup.exchange,
type=self.exchange_setup.exchange_type)
bound_exchange = exchange(channel)
bound_exchange.declare()
# example_message = '{"short_message":"Kombu", "host":"example.org"}'
message = bound_exchange.Message(message)
bound_exchange.publish(message, routing_key=self.exchange_setup.routing_key)
self.connection.release()
def connect_rabbitmq():
message = 'hello, this function is connection to RabbitMQ'
print message
print current_app.config['CELERY_BROKER_URL']
conn = BrokerConnection(current_app.config['CELERY_BROKER_URL'],
heartbeat=int(10))
pdf_request.apply_async(['hello'], connection=conn)
conn.release()
return jsonify({
'error_code':
0,
'error_desc':
'Cannot call to IAPI server, check connection again'
})
class Messaging(object):
amqp_opts = {
'amqp_queue': '', # do not send to queue by default
'amqp_topic': 'notify',
'amqp_url': 'amqp://*****:*****@localhost:5672//', # RabbitMQ
# 'amqp_url': 'mongodb://*****:*****@' # AWS SQS (must define amqp_queue)
# 'amqp_sqs_region': 'eu-west-1' # required if SQS is used
}
def __init__(self):
config.register_opts(Messaging.amqp_opts)
if CONF.debug:
setup_logging(loglevel='DEBUG', loggers=[''])
self.connection = None
self.connect()
def connect(self):
if not CONF.amqp_url:
return
if CONF.amqp_url.startswith('sqs://'):
CONF.amqp_url = 'sqs://' + CONF.amqp_url[6:].replace('/', '%2F')
if CONF.amqp_sqs_region:
transport_options = {'region': CONF.amqp_sqs_region}
else:
transport_options = {}
self.connection = BrokerConnection(
CONF.amqp_url,
transport_options=transport_options
)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', CONF.amqp_url, e)
sys.exit(1)
LOG.info('Connected to broker %s', CONF.amqp_url)
def disconnect(self):
return self.connection.release()
def is_connected(self):
return self.connection.connected
class Messaging(object):
amqp_opts = {
'amqp_queue': 'alerts',
'amqp_topic': 'notify',
'amqp_url': 'amqp://*****:*****@localhost:5672//', # RabbitMQ
# 'amqp_url': 'mongodb://localhost:27017/kombu', # MongoDB
# 'amqp_url': 'redis://localhost:6379/', # Redis
}
def __init__(self):
config.register_opts(Messaging.amqp_opts)
self.connection = None
self.channel = None
self.connect()
def connect(self):
if not CONF.amqp_url:
return
self.connection = BrokerConnection(CONF.amqp_url)
try:
self.connection.connect()
except Exception as e:
LOG.error('Failed to connect to AMQP transport %s: %s', CONF.amqp_url, e)
sys.exit(1)
self.channel = self.connection.channel()
LOG.info('Connected to broker %s', CONF.amqp_url)
def disconnect(self):
return self.connection.release()
def is_connected(self):
return self.connection.connected
class event2amqp():
def __init__(self,host,port,user,password,virtual_host, exchange_name,identifier,maxqueuelength,queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (self.user,self.password,self.host,self.port,self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connect(self):
logger.info("[Canopsis] connection with : %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name , "topic", durable=True, auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def create_producer(self):
try:
self.producer = Producer(
channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host
)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
def postmessage(self,message,retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"]
)
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"],
message["connector_name"],
message["event_type"],
message["source_type"],
message["component"],
message["resource"]
)
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.info("[Canopsis] using routing key %s" % key)
logger.info("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message, compression=None, routing_key=key, exchange=self.exchange_name)
return True
except:
logger.error("[Canopsis] Not connected, going to queue messages until connection back")
self.queue.append({"key":key,"message":message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg="[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (str(len(self.queue)),str(self.maxqueuelength))
logger.info(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key":key,"message":message})
logger.info("[Canopsis] Queue length : %d" % len(self.queue))
return True
else:
logger.error("[Canopsis] Maximum retention for event queue %s reached" % str(self.maxqueuelength))
return False
def errback(self,exc,interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" % (exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.info("[Canopsis] Pop item from queue [%s] : %s" % (str(len(self.queue)),str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"], compression=None, routing_key=item["key"], exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" % (error,func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile="%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
class Connection:
connection = None
def __init__(self, url):
self.url = url
self.__connection = None
self.__running = True
self.channel = None
self.sleep_time = 10
self.reconnect(url)
@staticmethod
def get_instance():
return Connection.connection
def __connect(self):
self.__connection = BrokerConnection(self.url)
self.channel = self.get_channel()
self.rpc_factory = rpc.RpcFactory(self.channel)
self.publisher_factory = publisher.PublisherFactory(self.channel)
self.consumer_factory = consumer.ConsumerFactory(self.channel)
self.__running = True
Connection.connection = self
def get_broker_connection(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection
def get_channel(self):
if self.channel is None:
self.channel = self.get_new_channel()
return self.channel
def get_new_channel(self):
if self.__connection is None:
self.reconnect(self.url)
return self.__connection.channel()
def get_rpc_factory(self):
return self.rpc_factory
def reconnect(self, url=None):
cc.acquire()
if self.__connection is not None:
self.release()
if url is not None:
self.url = url
logger.debug("reconnect connection")
attempt = 0
while True:
try:
self.__connect()
cc.release()
return
except Exception as e:
logging.exception(e)
logging.debug("retry again in %s s" % self.sleep_time)
time.sleep(self.sleep_time)
cc.release()
def drain_events(self):
self.__connection.drain_events()
def release(self):
Connection.connection = None
self.__running = False
self.__connection.release()
self.__connection.close()
self.channel = None
self.__connection = None
class event2amqp():
def __init__(self, host, port, user, password, virtual_host, exchange_name,
identifier, maxqueuelength, queue_dump_frequency):
self.host = host
self.port = port
self.user = user
self.password = password
self.virtual_host = virtual_host
self.exchange_name = exchange_name
self.identifier = identifier
self.maxqueuelength = maxqueuelength
self.queue_dump_frequency = queue_dump_frequency
self.connection_string = None
self.connection = None
self.channel = None
self.producer = None
self.exchange = None
self.queue = deque([])
self.tickage = 0
self.load_queue()
def create_connection(self):
self.connection_string = "amqp://%s:%s@%s:%s/%s" % (
self.user, self.password, self.host, self.port, self.virtual_host)
try:
self.connection = BrokerConnection(self.connection_string)
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connect(self):
logger.info("[Canopsis] connection with: %s" % self.connection_string)
try:
self.connection.connect()
if not self.connected():
return False
else:
self.get_channel()
self.get_exchange()
self.create_producer()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def disconnect(self):
try:
if self.connected():
self.connection.release()
return True
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def connected(self):
try:
if self.connection.connected:
return True
else:
return False
except:
return False
def get_channel(self):
try:
self.channel = self.connection.channel()
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def get_exchange(self):
try:
self.exchange = Exchange(self.exchange_name,
"topic",
durable=True,
auto_delete=False)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def create_producer(self):
try:
self.producer = Producer(channel=self.channel,
exchange=self.exchange,
routing_key=self.virtual_host)
except:
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
def postmessage(self, message, retry=False):
# process enqueud events if possible
self.pop_events()
if message["source_type"] == "component":
key = "%s.%s.%s.%s.%s" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"])
else:
key = "%s.%s.%s.%s.%s[%s]" % (
message["connector"], message["connector_name"],
message["event_type"], message["source_type"],
message["component"], message["resource"])
# connection management
if not self.connected():
logger.error("[Canopsis] Create connection")
self.create_connection()
self.connect()
# publish message
if self.connected():
logger.debug("[Canopsis] using routing key %s" % key)
logger.debug("[Canopsis] sending %s" % str(message))
try:
self.producer.revive(self.channel)
self.producer.publish(body=message,
compression=None,
routing_key=key,
exchange=self.exchange_name)
return True
except:
logger.error(
"[Canopsis] Not connected, going to queue messages until connection back"
)
self.queue.append({"key": key, "message": message})
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
# logger.error(str(traceback.format_exc()))
return False
else:
errmsg = "[Canopsis] Not connected, going to queue messages until connection back (%s items in queue | max %s)" % (
str(len(self.queue)), str(self.maxqueuelength))
logger.error(errmsg)
#enqueue_cano_event(key,message)
if len(self.queue) < int(self.maxqueuelength):
self.queue.append({"key": key, "message": message})
logger.debug("[Canopsis] Queue length: %d" % len(self.queue))
return True
else:
logger.error(
"[Canopsis] Maximum retention for event queue %s reached" %
str(self.maxqueuelength))
return False
def errback(self, exc, interval):
logger.warning("Couldn't publish message: %r. Retry in %ds" %
(exc, interval))
def pop_events(self):
if self.connected():
while len(self.queue) > 0:
item = self.queue.pop()
try:
logger.debug("[Canopsis] Pop item from queue [%s]: %s" %
(str(len(self.queue)), str(item)))
self.producer.revive(self.channel)
self.producer.publish(body=item["message"],
compression=None,
routing_key=item["key"],
exchange=self.exchange_name)
except:
self.queue.append(item)
func = sys._getframe(1).f_code.co_name
error = str(sys.exc_info()[0])
logger.error("[Canopsis] Unexpected error: %s in %s" %
(error, func))
return False
else:
return False
def hook_tick(self, brok):
self.tickage += 1
# queue retention saving
if self.tickage >= int(self.queue_dump_frequency) and len(
self.queue) > 0:
# flush queue to disk if queue age reach queue_dump_frequency
self.save_queue()
self.tickage = 0
return True
def save_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd() #:fixme: use path.join
logger.info("[Canopsis] saving to %s" % retentionfile)
filehandler = open(retentionfile, 'w')
pickle.dump(self.queue, filehandler)
filehandler.close()
return True
def load_queue(self):
retentionfile = "%s/canopsis.dat" % os.getcwd()
logger.info("[Canopsis] loading from %s" % retentionfile)
filehandler = open(retentionfile, 'r')
try:
self.queue = pickle.load(filehandler)
except:
pass
return True
def encode_video():
'''
encoding video when user clicked encode from inside site
input: none
returns status of encoding process
'''
if not request.json or not 'origin_id' in request.json or not 'user' in request.json:
return abort(400)
origin_id = request.json.get('origin_id')
valid = ObjectId.is_valid(origin_id)
if valid == False:
return jsonify({'result':'False', 'info':'origin id is not valid, cannot encode video!', 'error': int(0)})
sub_url = request.json.get('sub_url')
logo_url = request.json.get('logo_url')
user = request.json.get('user')
profiles_id = request.json.get('profiles_id') #if profiles_id not null, follow on this
priority = request.json.get('priority')
'''
for croping video
'''
crop = request.json.get('crop')
crop_arr = []
if crop:
crop_arr = [int(c) for c in crop.split(',')]
profile_array = []
is_new_profiles = int(0)
if profiles_id:
profile_array = [p.strip(' ') for p in profiles_id.split(',')]
for profile_id in profile_array:
valid = ObjectId.is_valid(profile_id)
if valid == False:
return jsonify({'result':'False', 'info':'profile id is not valid, cannot add origin video!', 'error': int(1)})
is_new_profiles = int(1)
origin_video = mdb.OriginVideo.get_origin_video(origin_id)
if origin_video:
if len(crop_arr) != 4:
crop_arr = origin_video['crop']
#prevent spam form bad user
#if origin_video['status_encode'] == 1:
#return jsonify({'result':'False', 'info': 'cannot encode video, because the same process encoding is working', 'error': int(2)})
if priority:
origin_video['priority'] = int (priority)
if origin_video['priority']:
conn = BrokerConnection(current_app.config['CELERY_BROKER_URL_PRI'], heartbeat=int(10))
else:
conn = BrokerConnection(current_app.config['CELERY_BROKER_URL'], heartbeat=int(10))
update_origin_video(origin_id, 5)
encode_video_task.apply_async([origin_video['video_url'], sub_url, logo_url, profile_array if profiles_id else origin_video['profiles_id'], origin_video['priority'], user, origin_id, is_new_profiles, crop_arr], connection=conn)
conn.release()
if profiles_id:
if not mdb.EncodingJob.get_jobs_with_origin_video_id(origin_id).count():
origin_video['profiles_id'] = profile_array
else:
profiles_id_old = origin_video['profiles_id']
for p in profile_array:
if p not in profiles_id_old:
profiles_id_old.append(p)
origin_video['profiles_id'] = profiles_id_old
origin_video['status_encode'] = int(5) #video have been pushed to queue
origin_video['sub_url'] = sub_url if sub_url else ''
origin_video['logo_url'] = logo_url if logo_url else ''
origin_video['user'] = user
origin_video['priority'] = int(priority) if priority else int(0)
origin_video['crop'] = crop_arr
origin_video.save()
return jsonify({'result':'Done', 'info': 'encoding video successfull!'})
return jsonify({'result':'False', 'info': 'cannot encode video, origin id not found', 'error': int(4)})
