def on_direct_message(self, status):
LOGGER.debug(u"[%s] direct message: %s" % (self.stream_id, status))
if 'id' in status:
log_state(self.channel_id, str(status['id']),
PostState.ARRIVED_IN_BOT)
self.last_status_id = status['id']
self.bot.on_direct_message(status,
self.stream,
channel_id=self.channel_id)
def create_post(user, sync=False, **kw):
""" Creates a proper platform Post given a user and post components.
Special args:
sync - <bool:default=False> forces synchronous postprocessing
skip_acl_check - <bool:default=False> creates post w/o checking acl permissions on parent channel (e.g. bots)
"""
""" Creates a proper platform Post given a user and post components.
Special args:
sync - <bool:default=False> forces synchronous postprocessing
"""
# Set user in thread local storage
from solariat_bottle.db.user import set_user
set_user(user)
from solariat_bottle.db.post.utils import get_platform_class
from solariat_bottle.utils.post import get_language
from solariat_bottle.db.channel.base import Channel
from solariat_bottle.utils.posts_tracking import log_state, PostState, get_post_natural_id
log_state(kw.get('channel', kw.get('channels', None)),
get_post_natural_id(kw), PostState.DELIVERED_TO_TANGO)
post_lang = get_language(kw)
if post_lang.support_level == Support.NO:
logger.info("Detect message for unsupported language: %s" %
post_lang.lang)
logger.info("Unsupported message value is: %s" % str(kw))
return
else:
kw['lang'] = post_lang
kw = normalize_post_params(user, kw)
klass = get_platform_class(kw['_platform'], event_type=kw['event_type'])
# we have channels resolved in normalize_post_params
channels = kw['channels']
accounts = set([ch.account for ch in channels])
for account in accounts:
if _check_account_volume(user, account):
msg = u"Account '{} ({})' has reached its monthly volume threshold.".format(
account.name, account.id)
LOGGER.warning(msg)
if _check_account_daily_volume(user, account):
msg = u"Account '{} ({})' has reached its daily volume threshold.".format(
account.name, account.id)
LOGGER.warning(msg)
return klass.objects.create_by_user(user,
safe_create=True,
sync=sync,
**kw)
def push(self, post, channel_id):
try:
self.create(id=self.make_id(channel_id, post.id),
channel_id=channel_id,
created_at=datetime.utcnow(),
post_data=post.data,
reserved_until=datetime(1970, 1, 1))
except DuplicateKeyError:
LOGGER.warning("Trying to push post %s twice on channel %s" %
(post, channel_id))
else:
log_state(channel_id, post.native_id,
PostState.DELIVERED_TO_GSE_QUEUE)
def on_direct_message(self, dm, stream=None, channel_id=None):
self.checker.inc_received()
if self.dedup(dm):
if self.use_kafka:
KafkaDataPreparation.on_message(self.username,
(DIRECT_MESSAGE, dm),
self.kwargs)
else:
self.creators.receive((DIRECT_MESSAGE, dm))
log_state(channel_id, str(dm['id']),
PostState.ADDED_TO_WORKER_QUEUE)
else:
self.logger.debug(u"Duplicated direct message %s", dm['id'])
def on_message(self, event_json, stream=None):
self.dump_message(event_json)
if 'id' in event_json:
log_state(None, str(event_json['id']), PostState.ARRIVED_IN_BOT)
self.checker.inc_received()
if self.use_kafka:
KafkaDataPreparation.on_message(self.username,
(PUBLIC_TWEET, event_json),
self.kwargs)
else:
self.dispatch(event_json)
if 'id' in event_json:
log_state(None, str(event_json['id']),
PostState.ADDED_TO_WORKER_QUEUE)
return True
def push_posts(self, tweets, post_data_format=None, post_data_dump=None):
insert_data = dict(subscription=self.subscription,
post_data_format=post_data_format)
for tweet in tweets:
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ARRIVED_IN_RECOVERY)
try:
insert_data.update({
"post_data":
post_data_dump(tweet),
"timestamp":
datetime_to_timestamp(parse_datetime(tweet['created_at']))
})
QueuedHistoricData.objects.create(**insert_data)
log_state(self.subscription.channel.id, str(tweet['id']),
PostState.ADDED_TO_WORKER_QUEUE)
except:
LOGGER.exception(u'QueuedHistoricData.objects.create: %s' %
insert_data)
return len(tweets)
def assign_channels(self, post_fields):
if not post_fields.get('channel', False) and not post_fields.get(
'channels', False):
if 'twitter' in post_fields:
LOGGER.debug('resolving tracked channels')
channels = self.get_tracked_channels('Twitter', post_fields)
LOGGER.debug(channels)
post_fields['channels'] = channels
log_state(
post_fields.get('channel', post_fields.get('channels', None)),
get_post_natural_id(post_fields),
PostState.REMOVED_FROM_WORKER_QUEUE)
channels_assigned = post_fields.get(
'channel', False) or post_fields.get('channels', False)
if channels_assigned:
# clean up just to be on the safe side
post_fields.pop('direct_message', None)
post_fields.pop('sender_handle', None)
post_fields.pop('recipient_handle', None)
return channels_assigned
return False
def fetch_and_post(self):
ch_id = self.subscription.channel.id
for posts_processed, entry in enumerate(self._query_posts(),
start=self.posts_processed +
1):
if self.subscriber.stopped():
break
post_fields = entry.solariat_post_data
if not post_fields:
LOGGER.warning('no post_fields in: %s', entry)
self.flush_buffer()
continue
try:
log_state(ch_id, get_post_natural_id(post_fields),
PostState.REMOVED_FROM_WORKER_QUEUE)
except KeyError:
LOGGER.error('cannot get post id: %s', post_fields)
channels = self._find_channels(post_fields)
if channels:
post_fields['channels'] = channels
self.post_queue.put(post_fields) # blocked by queue maxsize
self.posts_queued += 1
self._fetch_buffer.append(entry)
else:
LOGGER.warn("No channels found for queued post %s\n"
"queue item id: %s" % (post_fields, entry.id))
self.posts_processed = posts_processed
if posts_processed % self.UPDATE_PROGRESS_EVERY == 0:
self.update_progress()
self.subscriber.aggregate_state(self,
{'running': self.progress})
self.flush_buffer()
self.subscriber.update_status(SUBSCRIPTION_RUNNING)