def parse_platform(platform_json, to_platform_id=None, source='platform_json'):
"""Parse Platform from platform definition.
If *to_platform_id* is specified, update existing platform specified by
*to_platform_id* instead of creating a new platform.
If *to_platform_id* is a callable. The result of the call is used as the
platform_id.
"""
validate_platform_schema(platform_json)
# Validate plaform-specific schema
provider = get_messaging_provider(
PlatformTypeEnum(platform_json['type_enum']))
try:
jsonschema.validate(platform_json['config'],
provider.get_config_schema())
except jsonschema.exceptions.ValidationError:
logger.error('Platform config validate failed for `%s\'!', source)
raise
if callable(to_platform_id):
to_platform_id = to_platform_id(platform_json)
bot_id = platform_json.get('bot_id')
if bot_id:
bot = Bot.get_by(id=bot_id,
account_id=platform_json['account_id'],
single=True)
if not bot:
raise RuntimeError('bot does not exist')
if to_platform_id:
# Update existing platform.
logger.info(u'Updating existing Platform(id=%d, name=%s) from %s ...',
to_platform_id, platform_json['name'], source)
platform = Platform.get_by(id=to_platform_id, single=True)
platform.bot_id = bot_id
platform.name = platform_json['name']
platform.deployed = platform_json['deployed']
platform.type_enum = platform_json['type_enum']
platform.provider_ident = platform_json['provider_ident']
platform.config = platform_json['config']
else:
# Create a new platform.
logger.info(u'Creating new Platform(name=%s) from %s ...',
platform_json['name'], source)
platform = Platform(**platform_json).add()
DatabaseManager.flush()
return platform
def setup_prerequisite(self):
DatabaseManager.reset()
self.bot = Bot(name=u'test', description=u'test',
interaction_timeout=120, session_timeout=86400).add()
DatabaseManager.commit()
config = {
'access_token': 'EAAP0okfsZCVkBAI3BCU5s3u8O0iVFh6NAwFHa7X2bKZCGQ'
'Lw6VYeTpeTsW5WODeDbekU3ZA0JyVCBSmXq8EqwL1GDuZBO'
'7aAlcNEHQ3AZBIx0ZBfFLh95TlJWlLrYetzm9owKNR8Qju8'
'HF6qra20ZC6HqNXwGpaP74knlNvQJqUmwZDZD'
}
platform = Platform(name=u'Test Platform',
bot_id=self.bot.id,
type_enum=PlatformTypeEnum.Facebook,
provider_ident='1155924351125985',
config=config).add()
DatabaseManager.commit()
self.user = User(platform_id=platform.id,
platform_user_ident='1153206858057166',
last_seen=datetime.datetime.now()).add()
DatabaseManager.commit()
def find_platform_by_name(platform_desc):
platform = Platform.get_by(
provider_ident=platform_desc['provider_ident'], single=True)
if platform:
return platform.id
else:
return None
def setup_prerequisite(self):
DatabaseManager.reset()
self.bot = Bot(name=u'test',
description=u'test',
interaction_timeout=120,
session_timeout=86400).add()
DatabaseManager.commit()
config = {
'access_token':
'iHRMgmp3zRLOc6kPCbPNMwEDHyFqLGSy0tyG3uZxnkNlhMKg'
'8GVFqMGslcOkmgOAFLlBvvYuXmKF9odhXtsCm3tBxRcPryKr'
'kOvzHBcBvS2zrVGiVmZGh5EBcqazgurYMwVSdgNSrhCm/qp6'
'2aR7HAdB04t89/1O/w1cDnyilFU=',
'channel_secret':
'335c901df3a1969ca28a48bf6ddcc333'
}
platform = Platform(bot_id=self.bot.id,
name=u'Line',
type_enum=PlatformTypeEnum.Line,
provider_ident='aitjcize.line',
config=config).add()
DatabaseManager.commit()
self.user = User(
platform_id=platform.id,
platform_user_ident='U7200f33369e7e586c973c3a9df8feee4',
last_seen=datetime.datetime.now()).add()
DatabaseManager.commit()
def setup_prerequisite(self, bot_file):
InputTransformation.clear()
self.bot = reset_and_setup_bots([bot_file])[0]
self.user = User(platform_id=Platform.get_by(id=1, single=True).id,
platform_user_ident='blablabla',
last_seen=datetime.datetime.now()).add()
DatabaseManager.commit()
def get_account_platform_by_id(platform_id):
platform = Platform.get_by(id=platform_id,
account_id=g.account.id,
single=True)
if platform is None:
raise AppError(HTTPStatus.STATUS_CLIENT_ERROR,
CustomError.ERR_NOT_FOUND,
'platform_id: %d not found' % platform_id)
return platform
def facebook_webhook_task():
with statsd.timed('facebook_webhook_task', tags=[config.ENV_TAG]):
engine = Engine()
for entry in request.json.get('entry', []):
page_id = entry['id']
platform = Platform.get_by(type_enum=PlatformTypeEnum.Facebook,
provider_ident=page_id,
single=True)
if platform is None:
raise RuntimeError('no associate bot found for Facebook '
'Platform with page_id = %s' % page_id)
bot = platform.bot
g.ga_id = bot.ga_id
for messaging in entry['messaging']:
msg = messaging.get('message', None)
sender = messaging['sender']['id']
recipient = messaging['recipient']['id']
user_input = UserInput.FromFacebookMessage(messaging)
if msg and msg.get('is_echo'):
# Ignore message sent by ourself
if msg.get('app_id', None):
continue
user = User.get_by(platform_id=platform.id,
platform_user_ident=recipient,
single=True)
engine.process_admin_reply(bot, user, user_input)
else:
user = User.get_by(platform_id=platform.id,
platform_user_ident=sender,
eager=['platform'],
single=True)
if not user:
user = add_user(platform, sender)
if user_input:
engine.step(bot, user, user_input)
# Don't measure the time of the GA call
send_ga_track_info()
def line_webhook_task(provider_ident):
with statsd.timed('line_webhook_task', tags=[config.ENV_TAG]):
engine = Engine()
for entry in request.json.get('events', []):
if entry['source']['type'] != 'user':
raise RuntimeError('line_receive: only user source is support '
'for now')
sender = entry['source']['userId']
platform = Platform.get_by(provider_ident=provider_ident,
single=True)
if platform is None:
raise RuntimeError('no associate bot found for Line '
'Platform with ident = %s' % provider_ident)
digest = hmac.new(str(platform.config['channel_secret']),
request.data, hashlib.sha256).digest()
signature = base64.b64encode(digest)
if signature != request.headers['X-Line-Signature']:
raise RuntimeError('line_receive: failed to verify message')
bot = platform.bot
g.ga_id = bot.ga_id
user = User.get_by(platform_id=platform.id,
platform_user_ident=sender,
single=True)
if not user:
user = add_user(platform, sender)
user_input = UserInput.FromLineMessage(entry)
if user_input:
g.line_reply_token = entry['replyToken']
g.line_messages = []
engine.step(bot, user, user_input)
line.flush_message(platform)
# Don't measure the time of the GA call
send_ga_track_info()
def validate_platform_schema(platform_json, source='platform_json'):
try:
jsonschema.validate(platform_json, Platform.schema())
except jsonschema.exceptions.ValidationError:
logger.error('Validation failed for `%s\'!', source)
raise
def parse_bot(bot_json, to_bot_id=None, source='bot_json'):
"""Parse Bot from bot definition.
If *to_bot_id* is specified, update existing bot specified by *to_bot_id*
instead of creating a new bot.
If *to_bot_id* is a callable. The result of the call is used as the bot_id.
"""
validate_bot_schema(bot_json, source=source)
bot_desc = bot_json['bot']
if callable(to_bot_id):
to_bot_id = to_bot_id(bot_desc)
if to_bot_id:
# Update existing bot.
logger.info(u'Updating existing Bot(id=%d, name=%s) from %s ...',
to_bot_id, bot_desc['name'], source)
bot = Bot.get_by(id=to_bot_id, single=True)
bot.delete_all_nodes()
bot.name = bot_desc['name']
bot.description = bot_desc['description']
bot.interaction_timeout = bot_desc['interaction_timeout']
bot.admin_interaction_timeout = bot_desc['admin_interaction_timeout']
bot.session_timeout = bot_desc['session_timeout']
bot.ga_id = bot_desc.get('ga_id', None)
bot.settings = bot_desc['settings']
DatabaseManager.flush()
else:
# Create a new bot
logger.info(u'Creating new Bot(name=%s) from %s ...', bot_desc['name'],
source)
bot = Bot(
name=bot_desc['name'],
description=bot_desc['description'],
interaction_timeout=bot_desc['interaction_timeout'],
admin_interaction_timeout=bot_desc['admin_interaction_timeout'],
session_timeout=bot_desc['session_timeout'],
ga_id=bot_desc.get('ga_id', None),
settings=bot_desc['settings']).add()
DatabaseManager.flush()
# Bind Platform with Bot
platforms = bot_json.get('platforms', {})
for unused_name, provider_ident in platforms.iteritems():
platform = Platform.get_by(provider_ident=provider_ident, single=True)
if platform is None:
raise RuntimeError('associated platform `%s\' does not exist',
provider_ident)
# Bind
platform.bot_id = bot.id
provider = get_messaging_provider(platform.type_enum)
if not platform.deployed or (config.DEPLOY and platform.deployed):
provider.apply_settings(platform.config, bot.settings)
DatabaseManager.flush()
nodes = bot_desc['nodes']
id_map = {} # Mapping of stable_id to id
# Build node
for stable_id, node in nodes.iteritems():
try:
cm = ContentModule.get_by(id=node['content_module']['id'],
single=True)
if cm is None:
raise RuntimeError('Content_module `%d\' does not exist',
node['content_module']['id'])
jsonschema.validate(node['content_module']['config'],
cm.get_module().schema())
except jsonschema.exceptions.ValidationError:
logger.error(
'Node `%s\' content module config validation '
'failed', stable_id)
raise
n = Node(stable_id=stable_id,
bot_id=bot.id,
name=unicode(node['name']),
description=unicode(node['description']),
expect_input=node['expect_input'],
content_module_id=node['content_module']['id'],
content_config=node['content_module']['config']).add()
if 'parser_module' in node:
n.parser_module_id = node['parser_module']['id']
DatabaseManager.flush()
id_map[stable_id] = n.id
# Validate that parser module linkages are present in this bot file.
for stable_id, node in nodes.iteritems():
n = Node.get_by(id=id_map[stable_id], single=True)
if n.parser_module is not None:
nodes = bot_json['bot']['nodes']
n.parser_config = node['parser_module']['config']
pm = n.parser_module.get_module()
try:
jsonschema.validate(n.parser_config, pm.schema())
except jsonschema.exceptions.ValidationError:
logger.error(
'Node `%s\' parser module config validation '
'failed', stable_id)
raise
for end_node_id in pm.get_linkages(n.parser_config):
if end_node_id is not None:
if re.search(HAS_VARIABLE_RE, end_node_id):
logger.info('Rendered end_node_id `%s\', check '
'skipped ...' % end_node_id)
continue
if end_node_id not in id_map.keys():
raise RuntimeError('end_node_id `%s\' is invalid' %
end_node_id)
DatabaseManager.flush()
return bot
def test_schema_sanity(self):
"""Populate data into all tables and make sure there are no error."""
DatabaseManager.reset()
account = Account(name=u'Test Account',
email='*****@*****.**', passwd='test_hashed').add()
bot = Bot(name=u'test', description=u'test', interaction_timeout=120,
session_timeout=86400).add()
account.bots.append(bot)
content = ContentModule(id='test', name='Content1', description='desc',
module_name='', ui_module_name='').add()
parser = ParserModule(id='test', name='Parser1', description='desc',
module_name='passthrough', ui_module_name='',
variables={}).add()
# Test for oauth schema
oauth1 = OAuthInfo(provider=OAuthProviderEnum.Facebook,
provider_ident='mock-facebook-id').add()
oauth2 = OAuthInfo(provider=OAuthProviderEnum.Github,
provider_ident='mock-github-id').add()
account.oauth_infos.append(oauth1)
account.oauth_infos.append(oauth2)
DatabaseManager.commit()
account_ = Account.get_by(id=account.id, single=True)
self.assertNotEquals(account_, None)
self.assertEquals(len(account_.oauth_infos), 2)
oauth_ = OAuthInfo.get_by(provider_ident='mock-facebook-id',
single=True)
self.assertNotEquals(oauth_, None)
self.assertNotEquals(oauth_.account, None)
self.assertEquals(oauth_.account.id, account.id)
# Test for bot
account.bots.append(bot)
DatabaseManager.commit()
self.assertNotEquals(Account.get_by(id=account.id, single=True), None)
self.assertNotEquals(Bot.get_by(id=bot.id, single=True), None)
self.assertNotEquals(ContentModule.get_by(id=content.id, single=True),
None)
self.assertNotEquals(ParserModule.get_by(id=parser.id, single=True),
None)
# Check acccount_bot association table
self.assertEquals(len(account.bots), 1)
self.assertEquals(account.bots[0].id, bot.id)
platform = Platform(name=u'Test platform',
bot_id=bot.id,
type_enum=PlatformTypeEnum.Facebook,
provider_ident='facebook_page_id',
config={}).add()
DatabaseManager.commit()
self.assertNotEquals(Platform.get_by(id=platform.id, single=True),
None)
self.assertEquals(len(bot.platforms), 1)
self.assertEquals(bot.platforms[0].id, platform.id)
node1 = Node(stable_id='node1', name=u'1', bot_id=bot.id,
expect_input=True, content_module_id=content.id,
content_config={}, parser_module_id=parser.id,
parser_config={}).add()
node2 = Node(stable_id='node2', name=u'2', bot_id=bot.id,
expect_input=True, content_module_id=content.id,
content_config={}, parser_module_id=parser.id,
parser_config={}).add()
node3 = Node(stable_id='node3', name=u'3', bot_id=bot.id,
expect_input=True, content_module_id=content.id,
content_config={}, parser_module_id=parser.id,
parser_config={}).add()
bot.orphan_nodes.append(node3)
DatabaseManager.commit()
self.assertNotEquals(Node.get_by(id=node1.id, single=True), None)
self.assertNotEquals(Node.get_by(id=node2.id, single=True), None)
self.assertNotEquals(Node.get_by(id=node3.id, single=True), None)
# Test bot_node association table
self.assertEquals(bot.orphan_nodes[0].id, node3.id)
user = User(platform_id=platform.id,
platform_user_ident='',
last_seen=datetime.datetime.now()).add()
DatabaseManager.commit()
self.assertNotEquals(User.get_by(id=user.id, single=True), None)
event = Event(bot_id=bot.id, user_id=user.id, event_name='event',
event_value={}).add()
DatabaseManager.commit()
self.assertNotEquals(Event.get_by(id=event.id, single=True), None)
collected_datum = CollectedDatum(user_id=user.id,
key='key', value={}).add()
DatabaseManager.commit()
self.assertNotEquals(CollectedDatum.get_by(id=collected_datum.id,
single=True), None)
self.assertEquals(len(user.colleted_data), 1)
self.assertEquals(user.colleted_data[0].id, collected_datum.id)
conversation = Conversation(bot_id=bot.id, user_id=user.id,
sender_enum=SenderEnum.Bot, msg={}).add()
DatabaseManager.commit()
self.assertNotEquals(Conversation.get_by(id=conversation.id,
single=True), None)
# Broadcast
bc = Broadcast(account_id=account.id, bot_id=bot.id,
name=u'New broadcast', messages=[],
scheduled_time=datetime.datetime.utcnow()).add()
DatabaseManager.commit()
self.assertNotEquals(Broadcast.get_by(id=bc.id, single=True), None)
# PublicFeed, Feed
account = Account(name=u'Test Account - 1',
email='*****@*****.**', passwd='test_hashed').add()
feed1 = Feed(url='example.com/rss', type=FeedEnum.RSS,
title=u'foo.com', image_url='foo.com/logo').add()
feed2 = Feed(url='example.com/rss', type=FeedEnum.RSS,
title=u'bar.com', image_url='bar.com/logo').add()
feed3 = Feed(url='example.com/rss', type=FeedEnum.RSS,
title=u'baz.com', image_url='baz.com/logo').add()
account.feeds.append(feed1)
account.feeds.append(feed2)
account.feeds.append(feed3)
DatabaseManager.commit()
self.assertNotEquals(Feed.get_by(id=feed1.id, single=True), None)
feeds = Feed.search_title('ba')
self.assertEquals(len(list(feeds)), 2)
pfeed = PublicFeed(url='example.com/rss', type=FeedEnum.RSS,
title=u'example.com',
image_url='example.com/logo').add()
DatabaseManager.commit()
self.assertNotEquals(PublicFeed.get_by(id=pfeed.id, single=True), None)