def print_progress(self):
info = {
self.worker1.cls_name: self.worker1.get_progress(),
self.worker2.cls_name: self.worker2.get_progress(),
self.serial.cls_name: self.serial.get_progress(),
}
LOGGER.debug(json.dumps(info, indent=4))
def test_fetch_by_date(self):
# check start_date. let fetcher made 2 requests (max resp len: 200)
one_sec = timedelta(seconds=1)
FakeTwitterApi.restore_settings()
api = FakeTwitterApi()
_, start_date = api.DM[300]
start_date += one_sec
LOGGER.debug('++++++ DATES: (%s)', (start_date, api.DM[200][1], api.DM[100][1]))
res = DirectMessagesFetcher(api, **{"start_date": start_date})
statuses = list(res.fetch())
self.assertEqual(len(statuses), 300)
# checking end_date filter
FakeTwitterApi.restore_settings()
api = FakeTwitterApi()
_, start_date = api.DM[300]
_, end_date = api.DM[150]
start_date += one_sec
res = DirectMessagesFetcher(api, **{'start_date': start_date, 'end_date': end_date})
statuses = list(res.fetch())
self.assertEqual(len(statuses), 150)
from solariat.utils.timeslot import parse_datetime
self.assertTrue(all(start_date <= parse_datetime(s['created_at']) <= end_date for s in statuses))
def _fetch_media(self, media_url, base64encoded=True, data_uri=False):
import base64
import json
import requests
from solariat_bottle.settings import LOGGER
auth = self.api.base_api.auth.apply_auth()
resp = requests.get(media_url, auth=auth, stream=True)
LOGGER.debug(u"Response headers: %s", resp.headers)
if resp.status_code and not 200 <= resp.status_code < 300:
try:
error_msg = self.api.base_api.parser.parse_error(resp.text)
except Exception:
error_msg = "Twitter error response: status code = %s" % resp.status_code
raise tweepy.error.TweepError(error_msg)
data = resp.raw.read()
if base64encoded:
data = base64.b64encode(data)
if data_uri:
content_type = resp.headers.get('content-type', 'image/png')
data = "data:%s;base64,%s" % (content_type, data)
return {
"media_data": data,
"twitter_response_headers": json.dumps(dict(resp.headers))
}
def _search(self, user, *args, **kwargs):
error_desc = "Requests should contain a channel id and some query text: {'channel': <>, 'query': <>}"
required_params = ['channel', 'query']
LOGGER.debug('Searching for FAQs with params: %s' % kwargs)
for entry in required_params:
if entry not in kwargs:
error_msg = "Missing required parameter '%s'" % entry
raise exc.InvalidParameterConfiguration(error_msg,
description=error_desc)
try:
channel = Channel.objects.get(kwargs['channel'])
except Channel.DoesNotExist:
raise exc.ResourceDoesNotExist(
"No channel with id=%s found in the system." %
kwargs['channel'])
result = FAQ.objects.text_search(channel, kwargs['query'], limit=100)
LOGGER.debug('Search results for FAQs: %s' % result)
from solariat.utils.timeslot import parse_datetime, now
_created = parse_datetime(kwargs.get('_created', now()), default=now())
faq_event = FAQQueryEvent.objects.create_by_user(user,
query=kwargs['query'],
channels=[channel.id],
_created=_created)
if 'customer_id' in kwargs:
CustomerProfile = user.account.get_customer_profile_class()
customer = CustomerProfile.objects.get(kwargs['customer_id'])
# import ipdb
# ipdb.set_trace()
# actions = NextBestActionEngine.upsert(account_id=customer.account_id).search(faq_event, customer)
# TODO: THink how we can implement this properly with new advisors API
return dict(list=result,
event=faq_event.to_dict()) #, actions=actions)
else:
return dict(list=result, event=faq_event.to_dict())
def handle_post_reply(user,
creative,
outbound_channel_id,
post_id,
dry_run=False,
prefix='',
suffix='',
response_type=None):
"""Reply is different from Response that it is arbitrary text posted to arbitrary channel
in reply to some post.
"""
from solariat_bottle.db.channel.base import Channel
from solariat_bottle.db.post.base import Post
outbound_channel = Channel.objects.get(id=outbound_channel_id)
post = Post.objects.get(id=post_id)
if suffix:
# suffix is added with space
creative = "%s%s" % (creative, suffix)
# prefix is added in send_message
if outbound_channel.is_dispatchable:
is_direct = None
if response_type == 'direct':
is_direct = True
elif response_type is not None:
is_direct = False
outbound_channel.send_message(dry_run,
creative,
post,
user=user,
direct_message=is_direct)
else:
LOGGER.debug(
'Warning. No post has been dispacthed because the channel %s is not dispatchable.'
% outbound_channel.title)
pass
def run(self):
inp_queue = self.inp_queue
start_time = time.time()
while not self.stopped():
# make sure we intercept all errors
try:
task = inp_queue.get()
if task is self.QUIT or task == 'QUIT':
LOGGER.debug('received QUIT signal %s' % self)
break
start_time = time.time()
self._busy = True # Just started doing our post processing
post_fields = self.preprocess_post(task)
if not post_fields:
LOGGER.warning('no post_fields in: %s', task)
continue
# LOGGER.debug('creating post %r %s', post_fields.get('content'), inp_queue.qsize())
if self.assign_channels(post_fields):
self.create_post(**post_fields)
else:
LOGGER.info('skipping post %r' %
post_fields.get('content'))
self.inc_skipped()
self._busy = False # Just Finished doing our post processing
except Exception, err:
LOGGER.error(err, exc_info=True)
pass
finally:
def on_receive(self, data):
"""
Called whenever a new direct message is recieved. Keep buffering data
until valid message is recieved fully. After that extract whatever is
required from the full json and push to the `incoming_posts_queue` for
further concurent processing.
"""
if self.__stopped:
self.mstream.stop()
return -1
if data:
self.keep_alive()
if data.strip():
if len(data) <= 60:
log_val = data
else:
log_val = '%s...%s' % (data[:40], data[-17:])
LOGGER.debug('received %r (%u bytes)', log_val, len(data))
# do not append empty data (\r\n) in buffer
self.buff += data
if data.endswith("\r\n") and self.buff.strip():
self.mstream.received_message(
self.buff, self) # Pass it on to bot for processing
self.buff = ""
def acquire_for_stream(self, ref):
LOGGER.info(u"Acquiring auth for stream %s" % ref)
with self._lock:
auth = self._resource.get()
self._in_use[ref.key] = auth
LOGGER.debug(u"In use: {}".format(self._in_use))
return auth
def on_direct_message(self, status):
LOGGER.debug(u"[%s] direct message: %s" % (self.stream_id, status))
if 'id' in status:
log_state(self.channel_id, str(status['id']),
PostState.ARRIVED_IN_BOT)
self.last_status_id = status['id']
self.bot.on_direct_message(status,
self.stream,
channel_id=self.channel_id)
def release_for_stream(self, ref):
LOGGER.info(u"Releasing auth for stream %s" % ref)
with self._lock:
if ref.key in self._in_use:
auth = self._in_use.pop(ref.key)
self.put(auth)
else:
auth = None
LOGGER.debug(u"In use: {}".format(self._in_use))
return auth
def _method(*args, **kwargs):
queued_data[:] = [
get_id_date_pair(i.solariat_post_data) for i in
QueuedHistoricData.objects(subscription=subscription)
]
LOGGER.debug('QUEUED_POSTS, len: %s', len(queued_data))
self.assertTrue(len(queued_data) == FakeTwitterApi.ALL_DATA_LENGTH,
msg="len=%d %s" % (len(queued_data), queued_data))
self.assertEqual(set(queued_data), set(fetched_data),
msg=u"\nqueued =%s\nfetched=%s" % (queued_data, fetched_data))
return method(*args, **kwargs)
def __opensocket(self, family, socktype, protocol, *args, **kwargs):
" set a timer, 90 second TCP level socket timeout "
LOGGER.debug('__opensocket(%s, %s, %s)' % (family, socktype, protocol))
LOGGER.debug('__opensocket args=%s kwargs=%s' % (args, kwargs))
sock = socket.socket(family, socktype, protocol)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 90)
sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 10)
sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 2)
return sock
def filters_fulfilled(self):
start_date = self.filters.get('start_date')
end_date = self.filters.get('end_date')
predicates = []
p = predicates.append
if start_date:
p(utc(self.timeline_min_date) <= utc(start_date))
if end_date:
p(utc(self.timeline_max_date) >= utc(end_date))
res = all(predicates)
LOGGER.debug('[:::: filters_fulfilled ::::] %s', res)
return res
def run(self):
LOGGER.debug('[%s restored?] my_status=%s', self.cls_name,
self.my_status)
self.worker.start()
while not self.stopped():
self.do_work()
self.wait(self.DELAY)
self.print_progress()
self.worker.stop()
self.worker.join()
def do_post(self, path, wrap_response=True, version=None, **kw):
"Emulate POST request"
path = get_api_url(path, version=self._get_version(version)) + '?token=%s' % self.auth_token
base_url = kw.pop('base_url', 'https://localhost')
data = json.dumps(kw)
LOGGER.debug("Performing POST to %s with %s" % (path, data))
response = self.client.post(path, data=data, base_url=base_url, content_type='application/json')
if wrap_response:
return self._handle_http_response(response)
else:
return response
def aggregate_state(self, worker, wstate):
LOGGER.debug('[aggregate state invoked] worker: %s', worker)
state = self.state
if worker in state:
name, _, _ = state[worker]
state[worker] = [name, wstate, now()]
else:
idx = len([1 for w in state if type(w) == type(worker)])
name = '%s_%s' % (type(worker).__name__, idx)
state[worker] = [name, wstate, now()]
# res = {name: [wstate, dt] for name, wstate, dt in state.values()}
return {name: wstate}
def extract_upsert_data(self, ProfileCls, data):
logger.debug('extract_upsert_data(%s, %s)', ProfileCls, data)
if data.get('id'):
query = {'id': data.get('id')}
else:
query = {'id': str(ObjectId())}
update = {
'native_id': ProfileCls.extract_native_id(data),
'platform_data': data,
'updated_at': now()
}
return query, update
def do_post(self, path, **kw):
"Emulate POST request"
path = restapi.get_angel_url(path)
kw['api_token'] = self.api_token
data = json.dumps(kw)
LOGGER.debug("Performing POST to %s with %s" % (path, data))
response = self.client.post(path,
data=data,
content_type='application/json')
self.assertEqual(response.status_code, 200)
return json.loads(response.data)
def run(self):
for message in self.consumer:
LOGGER.debug('kafka message consumed: %s', message.value)
try:
object = kafka_serializer.deserialize(message.value)
if 'task' in object:
self.kafka_handler.handle_create_post(
object['task'], object['username'], object['kwargs'])
if 'event' in object:
self.kafka_handler.on_event_handler(
object['event'], object['stream_data'])
except Exception as e:
LOGGER.exception(e)
def test_user_creation(self):
# user is created successfuly
data = {
'account_name': 'Account-Name',
'email': '*****@*****.**',
'cxb_id': '1234-abc',
}
resp = self.do_post('users', **data)
self.assertTrue(resp['ok'])
u = resp['user']
self.assertEqual(u['email'], data['email'])
self.assertEqual(u['external_id'], 'cxb_id:' + data['cxb_id'])
a = resp['account']
LOGGER.debug(a)
self.assertEqual(a['name'], data['account_name'])
self.assertEqual(a['account_type'], 'Angel')
# errors if some parameters are not specified
for k in data:
bad_data = copy(data)
bad_data.pop(k)
resp = self.do_post('users', **bad_data)
self.assertFalse(resp['ok'])
# a user with duplicate email is not created
different_data = {
'account_name': 'Account-Name-2',
'email': '*****@*****.**',
'cxb_id': '1234-abc2',
}
bad_data = copy(different_data)
bad_data['email'] = data['email']
resp = self.do_post('users', **bad_data)
self.assertFalse(resp['ok'])
# a user with duplicated external id is not created
bad_data = copy(different_data)
bad_data['account_name'] = data['account_name']
resp = self.do_post('users', **bad_data)
self.assertFalse(resp['ok'])
# a user with duplicated account name is not created
bad_data = copy(different_data)
bad_data['cxb_id'] = data['cxb_id']
resp = self.do_post('users', **bad_data)
print resp
self.assertFalse(resp['ok'])
def run(self):
cmd_queue = self.cmd_queue
cur_hash = None
ds_client = None
while not self.stopped():
# make sure we intercept all errors
try:
# react on commands simultaneously making a 10 sec pause
try:
cmd, arg = cmd_queue.get(block=True, timeout=10 if not get_var('ON_TEST') else 1)
LOGGER.debug('received %s command', cmd)
if cmd == 'CLIENT':
ds_client = arg
cur_hash = None
elif cmd == 'QUIT':
break
except Empty:
LOGGER.debug('timeout (it\'s okay)')
pass
if ds_client is None:
continue
if ds_client.terminated:
LOGGER.warning('ds_client is terminated')
ds_client = None
continue
# get current datasift hash from the db
ds_hash = get_datasift_hash()
if not ds_hash:
continue
# subscibe/unsubscribe if necessary
if not cur_hash:
ds_client.subscribe(ds_hash)
elif cur_hash != ds_hash:
ds_client.unsubscribe(cur_hash)
ds_client.subscribe(ds_hash)
# remember the current hash
cur_hash = ds_hash
except Exception, err:
LOGGER.error(err, exc_info=True)
pass
def test_multi_agent_scenario(self):
sc = self.make_service_channel("SC1", ['brand1', 'brand2'])
dispatch_channel_1 = self.make_dispatch_channel("D1", "brand1")
dispatch_channel_2 = self.make_dispatch_channel("D2", "brand2")
#try:
# ch = sc.get_outbound_channel(self.user)
# self.assertFalse(True, "This should not be possible: %s" % (ch.title if ch else "NONE"))
#except AppException, exc:
# pass
# Now, remove access from one of the channels, and confirm it works
dispatch_channel_2.del_perm(self.user)
LOGGER.debug("SC IS %s", sc)
self.assertEqual(sc.get_outbound_channel(self.user),
dispatch_channel_1)
def share_post(self, post, user, dry_run=False):
# self.sync_contacts(post.user_profile)
from solariat_bottle.tasks.twitter import tw_share_post
post_content = post.plaintext_content
status_id = post.native_id
if dry_run is False and not get_var('ON_TEST') and get_var(
'APP_MODE') == 'prod':
tw_share_post.ignore(self,
status_id=status_id,
screen_name=post.user_profile.user_name)
else:
create_outbound_post(user, self, "RT: %s" % post_content, post)
LOGGER.debug("Retweet '%s' using %s", post_content, self)
def wrapper(*args, **kwargs):
#LOGGER.debug("API Request Args: {} | Params: {}".format(args, kwargs))
assert args
assert isinstance(args[0], BaseAPIView)
view = args[0] # Method decorator
try:
args = args[1:]
except IndexError:
args = ()
params = _get_request_data()
params.update(kwargs) # Pass URL variables to the view function
start_execution = datetime.utcnow()
# Execute API method
try:
# Assert authentication
LOGGER.debug("Started executing API call: %s.%s(%s, %s) " % (
view.__class__.__name__, func.__name__, args, kwargs))
if allow_basic_auth is True:
user = _get_user()
if user is None:
user = authenticate_api_user(params)
if user is None:
raise api_exc.AuthorizationError("User is not authenticated. Parameters for call: " + str(params))
else:
user = authenticate_api_user(params)
# Set user in thread local storage
set_user(user)
resp = func(view, user, *args, **params)
elapsed = datetime.utcnow() - start_execution
if elapsed.total_seconds() >= 2:
log = LOGGER.info
else:
log = LOGGER.debug
log("API call: %s.%s(%s,%s) finished after %s Parameters: %s" % (view.__class__.__name__, func.__name__,
str(args)[:500], str(kwargs)[:500],
elapsed, str(params)[:500]))
# auth token expiration and other auth errors
except api_exc.AuthorizationError, exc:
LOGGER.info(exc)
return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
def send_message(self, dry_run, creative, post, user, direct_message=None):
# self.sync_contacts(post.user_profile)
from solariat_bottle.tasks.twitter import tw_normal_reply, tw_direct_reply
is_dm = False # Assume this is not a DM
if direct_message is not None:
# If we specifically passed the fact that we want a direct message, use DM
# Otherwise decide based on post type
is_dm = direct_message
else:
if post.message_type == 'direct':
is_dm = True
if not is_dm:
status = "@%s %s" % (post.user_profile.user_name, creative)
else:
status = creative
if len(status) > 140:
msg = (
'Sorry, you have exceeded your 140 character limit by %d characters. '
'Please correct your reply and try again.') % (len(status) -
140)
raise AppException(msg)
status_id = post.native_id
# Update the engagement history
post.user_profile.update_history(self)
LOGGER.debug("For current message, direct message flag is %s", is_dm)
if not dry_run and not get_var('ON_TEST'):
if is_dm:
tw_direct_reply.ignore(self,
status=status,
screen_name=post.user_profile.user_name)
else:
tw_normal_reply.ignore(self,
status=status,
status_id=status_id,
post=post)
else:
create_outbound_post(user, self, creative, post)
LOGGER.debug("Sent '%s' to %s using %s", creative,
post.user_profile.user_name, self.title)
def do_get(self, path, version=None, **kw):
"Emulate GET request"
kw['token'] = self.auth_token
path = get_api_url(path, version=self._get_version(version=version))
if '?' in path:
path += '&'
else:
path +='?'
base_url = kw.pop('base_url', 'https://localhost')
path += urllib.urlencode(kw)
LOGGER.debug(
"Performing GET with %s" % path)
return self._handle_http_response(
self.client.get(path, base_url=base_url))
def set_dynamic_class(self, inheritance):
''' Support dynamic events '''
if DynamicEvent.__name__ in inheritance:
# from solariat_bottle.db.dynamic_event import EventType
from solariat_bottle.db.events.event_type import BaseEventType
event_type_name = self.data[DynamicEvent.event_type.db_field]
# event_type = EventType.objects.find_one(event_type_id)
# TODO: check using get_user() is correct here
acc_id = get_user().account.id
LOGGER.debug('Set event dynamic class, use acc: %s for find event types', acc_id)
event_type = BaseEventType.objects.find_one_by_display_name(acc_id, event_type_name)
if event_type:
dyn_cls = event_type.get_data_class() # initialize class in Registry
self.__class__ = dyn_cls
else:
LOGGER.error('Cannot find suitable event type for dynamic class!')
def send_email(self, attachments):
from solariat_bottle.utils.mailer import \
send_mail, Message, _get_sender, render_template
rcps = self.recipient_emails
self.export_item.update(set__recipient_emails=rcps)
msg = Message(subject=DataExportMailer.EMAIL_SUBJECT,
sender=_get_sender(),
recipients=rcps)
params = self.get_email_parameters()
msg.html = render_template("mail/export/posts.html", **params)
msg.body = render_template("mail/export/posts.txt", **params)
from solariat_bottle.settings import LOGGER
LOGGER.debug(msg.body)
for (filename, mimetype, stream) in attachments:
msg.attach(filename, mimetype, stream.getvalue())
send_mail(msg)
def generate_csv(cls, iterable_data, context):
header = [
'Channel Type', 'Channel Name', 'Inbound/Outbound', 'Post/Comment',
'Initial Post', 'Date/Time (UTC)', 'Smart Tags', 'Intentions',
'Post Status', 'Topics'
]
def get_post_content(p):
_get = lambda p, attr: \
p.get(attr) if isinstance(p, dict) else getattr(p, attr)
attr = 'url' if context.get('platform') == 'Twitter' else 'content'
return unwrap_hidden(_get(p, attr))
def get_initial_post(p):
if p['parent']:
return get_post_content(p['parent'])
return ''
value_getters = [
cls.PrefetchedValue(context.get('platform')),
cls.PrefetchedValue(context.get('channel_name')),
cls.PrefetchedValue(context.get('channel_direction')),
get_post_content, get_initial_post, 'created_at', 'smart_tags',
'intentions', 'filter_status', 'topics'
]
stream = StringIO()
writer = UnicodeWriter(stream)
writer.writerow(header)
from solariat_bottle.settings import LOGGER
for item in iterable_data:
try:
row = cls.csv_row(value_getters, item)
except Exception:
LOGGER.exception(u"Cannot generate csv tuple for %s" % item)
else:
LOGGER.debug(row)
writer.writerow(row)
stream.seek(0)
return stream
def sync_streams(self):
"""
In case new channels are added, or old ones are suspended / logged out
we need to also reflect those changes in the UserStreams we are tracking.
"""
LOGGER.debug('synchronizing streams')
_channels = {}
for ustream in self.handles:
_channels[ustream.channel] = ustream
new_channels = {}
# Get a list of all the EnterpriseTwitterChannel channels that are currently active
for channel in EnterpriseTwitterChannel.objects(status='Active'):
try:
if (channel.access_token_key and channel.access_token_secret):
options = self._get_stream_opts(channel)
new_channels[options.channel] = options
except Exception, ex:
LOGGER.error("'%s' channel failed: %s", channel, ex)
