def print_progress(self): info = { self.worker1.cls_name: self.worker1.get_progress(), self.worker2.cls_name: self.worker2.get_progress(), self.serial.cls_name: self.serial.get_progress(), } LOGGER.debug(json.dumps(info, indent=4))
def test_fetch_by_date(self): # check start_date. let fetcher made 2 requests (max resp len: 200) one_sec = timedelta(seconds=1) FakeTwitterApi.restore_settings() api = FakeTwitterApi() _, start_date = api.DM[300] start_date += one_sec LOGGER.debug('++++++ DATES: (%s)', (start_date, api.DM[200][1], api.DM[100][1])) res = DirectMessagesFetcher(api, **{"start_date": start_date}) statuses = list(res.fetch()) self.assertEqual(len(statuses), 300) # checking end_date filter FakeTwitterApi.restore_settings() api = FakeTwitterApi() _, start_date = api.DM[300] _, end_date = api.DM[150] start_date += one_sec res = DirectMessagesFetcher(api, **{'start_date': start_date, 'end_date': end_date}) statuses = list(res.fetch()) self.assertEqual(len(statuses), 150) from solariat.utils.timeslot import parse_datetime self.assertTrue(all(start_date <= parse_datetime(s['created_at']) <= end_date for s in statuses))
def _fetch_media(self, media_url, base64encoded=True, data_uri=False): import base64 import json import requests from solariat_bottle.settings import LOGGER auth = self.api.base_api.auth.apply_auth() resp = requests.get(media_url, auth=auth, stream=True) LOGGER.debug(u"Response headers: %s", resp.headers) if resp.status_code and not 200 <= resp.status_code < 300: try: error_msg = self.api.base_api.parser.parse_error(resp.text) except Exception: error_msg = "Twitter error response: status code = %s" % resp.status_code raise tweepy.error.TweepError(error_msg) data = resp.raw.read() if base64encoded: data = base64.b64encode(data) if data_uri: content_type = resp.headers.get('content-type', 'image/png') data = "data:%s;base64,%s" % (content_type, data) return { "media_data": data, "twitter_response_headers": json.dumps(dict(resp.headers)) }
def _search(self, user, *args, **kwargs): error_desc = "Requests should contain a channel id and some query text: {'channel': <>, 'query': <>}" required_params = ['channel', 'query'] LOGGER.debug('Searching for FAQs with params: %s' % kwargs) for entry in required_params: if entry not in kwargs: error_msg = "Missing required parameter '%s'" % entry raise exc.InvalidParameterConfiguration(error_msg, description=error_desc) try: channel = Channel.objects.get(kwargs['channel']) except Channel.DoesNotExist: raise exc.ResourceDoesNotExist( "No channel with id=%s found in the system." % kwargs['channel']) result = FAQ.objects.text_search(channel, kwargs['query'], limit=100) LOGGER.debug('Search results for FAQs: %s' % result) from solariat.utils.timeslot import parse_datetime, now _created = parse_datetime(kwargs.get('_created', now()), default=now()) faq_event = FAQQueryEvent.objects.create_by_user(user, query=kwargs['query'], channels=[channel.id], _created=_created) if 'customer_id' in kwargs: CustomerProfile = user.account.get_customer_profile_class() customer = CustomerProfile.objects.get(kwargs['customer_id']) # import ipdb # ipdb.set_trace() # actions = NextBestActionEngine.upsert(account_id=customer.account_id).search(faq_event, customer) # TODO: THink how we can implement this properly with new advisors API return dict(list=result, event=faq_event.to_dict()) #, actions=actions) else: return dict(list=result, event=faq_event.to_dict())
def handle_post_reply(user, creative, outbound_channel_id, post_id, dry_run=False, prefix='', suffix='', response_type=None): """Reply is different from Response that it is arbitrary text posted to arbitrary channel in reply to some post. """ from solariat_bottle.db.channel.base import Channel from solariat_bottle.db.post.base import Post outbound_channel = Channel.objects.get(id=outbound_channel_id) post = Post.objects.get(id=post_id) if suffix: # suffix is added with space creative = "%s%s" % (creative, suffix) # prefix is added in send_message if outbound_channel.is_dispatchable: is_direct = None if response_type == 'direct': is_direct = True elif response_type is not None: is_direct = False outbound_channel.send_message(dry_run, creative, post, user=user, direct_message=is_direct) else: LOGGER.debug( 'Warning. No post has been dispacthed because the channel %s is not dispatchable.' % outbound_channel.title) pass
def run(self): inp_queue = self.inp_queue start_time = time.time() while not self.stopped(): # make sure we intercept all errors try: task = inp_queue.get() if task is self.QUIT or task == 'QUIT': LOGGER.debug('received QUIT signal %s' % self) break start_time = time.time() self._busy = True # Just started doing our post processing post_fields = self.preprocess_post(task) if not post_fields: LOGGER.warning('no post_fields in: %s', task) continue # LOGGER.debug('creating post %r %s', post_fields.get('content'), inp_queue.qsize()) if self.assign_channels(post_fields): self.create_post(**post_fields) else: LOGGER.info('skipping post %r' % post_fields.get('content')) self.inc_skipped() self._busy = False # Just Finished doing our post processing except Exception, err: LOGGER.error(err, exc_info=True) pass finally:
def on_receive(self, data): """ Called whenever a new direct message is recieved. Keep buffering data until valid message is recieved fully. After that extract whatever is required from the full json and push to the `incoming_posts_queue` for further concurent processing. """ if self.__stopped: self.mstream.stop() return -1 if data: self.keep_alive() if data.strip(): if len(data) <= 60: log_val = data else: log_val = '%s...%s' % (data[:40], data[-17:]) LOGGER.debug('received %r (%u bytes)', log_val, len(data)) # do not append empty data (\r\n) in buffer self.buff += data if data.endswith("\r\n") and self.buff.strip(): self.mstream.received_message( self.buff, self) # Pass it on to bot for processing self.buff = ""
def acquire_for_stream(self, ref): LOGGER.info(u"Acquiring auth for stream %s" % ref) with self._lock: auth = self._resource.get() self._in_use[ref.key] = auth LOGGER.debug(u"In use: {}".format(self._in_use)) return auth
def on_direct_message(self, status): LOGGER.debug(u"[%s] direct message: %s" % (self.stream_id, status)) if 'id' in status: log_state(self.channel_id, str(status['id']), PostState.ARRIVED_IN_BOT) self.last_status_id = status['id'] self.bot.on_direct_message(status, self.stream, channel_id=self.channel_id)
def release_for_stream(self, ref): LOGGER.info(u"Releasing auth for stream %s" % ref) with self._lock: if ref.key in self._in_use: auth = self._in_use.pop(ref.key) self.put(auth) else: auth = None LOGGER.debug(u"In use: {}".format(self._in_use)) return auth
def _method(*args, **kwargs): queued_data[:] = [ get_id_date_pair(i.solariat_post_data) for i in QueuedHistoricData.objects(subscription=subscription) ] LOGGER.debug('QUEUED_POSTS, len: %s', len(queued_data)) self.assertTrue(len(queued_data) == FakeTwitterApi.ALL_DATA_LENGTH, msg="len=%d %s" % (len(queued_data), queued_data)) self.assertEqual(set(queued_data), set(fetched_data), msg=u"\nqueued =%s\nfetched=%s" % (queued_data, fetched_data)) return method(*args, **kwargs)
def __opensocket(self, family, socktype, protocol, *args, **kwargs): " set a timer, 90 second TCP level socket timeout " LOGGER.debug('__opensocket(%s, %s, %s)' % (family, socktype, protocol)) LOGGER.debug('__opensocket args=%s kwargs=%s' % (args, kwargs)) sock = socket.socket(family, socktype, protocol) sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPIDLE, 90) sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPINTVL, 10) sock.setsockopt(socket.SOL_TCP, socket.TCP_KEEPCNT, 2) return sock
def filters_fulfilled(self): start_date = self.filters.get('start_date') end_date = self.filters.get('end_date') predicates = [] p = predicates.append if start_date: p(utc(self.timeline_min_date) <= utc(start_date)) if end_date: p(utc(self.timeline_max_date) >= utc(end_date)) res = all(predicates) LOGGER.debug('[:::: filters_fulfilled ::::] %s', res) return res
def run(self): LOGGER.debug('[%s restored?] my_status=%s', self.cls_name, self.my_status) self.worker.start() while not self.stopped(): self.do_work() self.wait(self.DELAY) self.print_progress() self.worker.stop() self.worker.join()
def do_post(self, path, wrap_response=True, version=None, **kw): "Emulate POST request" path = get_api_url(path, version=self._get_version(version)) + '?token=%s' % self.auth_token base_url = kw.pop('base_url', 'https://localhost') data = json.dumps(kw) LOGGER.debug("Performing POST to %s with %s" % (path, data)) response = self.client.post(path, data=data, base_url=base_url, content_type='application/json') if wrap_response: return self._handle_http_response(response) else: return response
def aggregate_state(self, worker, wstate): LOGGER.debug('[aggregate state invoked] worker: %s', worker) state = self.state if worker in state: name, _, _ = state[worker] state[worker] = [name, wstate, now()] else: idx = len([1 for w in state if type(w) == type(worker)]) name = '%s_%s' % (type(worker).__name__, idx) state[worker] = [name, wstate, now()] # res = {name: [wstate, dt] for name, wstate, dt in state.values()} return {name: wstate}
def extract_upsert_data(self, ProfileCls, data): logger.debug('extract_upsert_data(%s, %s)', ProfileCls, data) if data.get('id'): query = {'id': data.get('id')} else: query = {'id': str(ObjectId())} update = { 'native_id': ProfileCls.extract_native_id(data), 'platform_data': data, 'updated_at': now() } return query, update
def do_post(self, path, **kw): "Emulate POST request" path = restapi.get_angel_url(path) kw['api_token'] = self.api_token data = json.dumps(kw) LOGGER.debug("Performing POST to %s with %s" % (path, data)) response = self.client.post(path, data=data, content_type='application/json') self.assertEqual(response.status_code, 200) return json.loads(response.data)
def run(self): for message in self.consumer: LOGGER.debug('kafka message consumed: %s', message.value) try: object = kafka_serializer.deserialize(message.value) if 'task' in object: self.kafka_handler.handle_create_post( object['task'], object['username'], object['kwargs']) if 'event' in object: self.kafka_handler.on_event_handler( object['event'], object['stream_data']) except Exception as e: LOGGER.exception(e)
def test_user_creation(self): # user is created successfuly data = { 'account_name': 'Account-Name', 'email': '*****@*****.**', 'cxb_id': '1234-abc', } resp = self.do_post('users', **data) self.assertTrue(resp['ok']) u = resp['user'] self.assertEqual(u['email'], data['email']) self.assertEqual(u['external_id'], 'cxb_id:' + data['cxb_id']) a = resp['account'] LOGGER.debug(a) self.assertEqual(a['name'], data['account_name']) self.assertEqual(a['account_type'], 'Angel') # errors if some parameters are not specified for k in data: bad_data = copy(data) bad_data.pop(k) resp = self.do_post('users', **bad_data) self.assertFalse(resp['ok']) # a user with duplicate email is not created different_data = { 'account_name': 'Account-Name-2', 'email': '*****@*****.**', 'cxb_id': '1234-abc2', } bad_data = copy(different_data) bad_data['email'] = data['email'] resp = self.do_post('users', **bad_data) self.assertFalse(resp['ok']) # a user with duplicated external id is not created bad_data = copy(different_data) bad_data['account_name'] = data['account_name'] resp = self.do_post('users', **bad_data) self.assertFalse(resp['ok']) # a user with duplicated account name is not created bad_data = copy(different_data) bad_data['cxb_id'] = data['cxb_id'] resp = self.do_post('users', **bad_data) print resp self.assertFalse(resp['ok'])
def run(self): cmd_queue = self.cmd_queue cur_hash = None ds_client = None while not self.stopped(): # make sure we intercept all errors try: # react on commands simultaneously making a 10 sec pause try: cmd, arg = cmd_queue.get(block=True, timeout=10 if not get_var('ON_TEST') else 1) LOGGER.debug('received %s command', cmd) if cmd == 'CLIENT': ds_client = arg cur_hash = None elif cmd == 'QUIT': break except Empty: LOGGER.debug('timeout (it\'s okay)') pass if ds_client is None: continue if ds_client.terminated: LOGGER.warning('ds_client is terminated') ds_client = None continue # get current datasift hash from the db ds_hash = get_datasift_hash() if not ds_hash: continue # subscibe/unsubscribe if necessary if not cur_hash: ds_client.subscribe(ds_hash) elif cur_hash != ds_hash: ds_client.unsubscribe(cur_hash) ds_client.subscribe(ds_hash) # remember the current hash cur_hash = ds_hash except Exception, err: LOGGER.error(err, exc_info=True) pass
def test_multi_agent_scenario(self): sc = self.make_service_channel("SC1", ['brand1', 'brand2']) dispatch_channel_1 = self.make_dispatch_channel("D1", "brand1") dispatch_channel_2 = self.make_dispatch_channel("D2", "brand2") #try: # ch = sc.get_outbound_channel(self.user) # self.assertFalse(True, "This should not be possible: %s" % (ch.title if ch else "NONE")) #except AppException, exc: # pass # Now, remove access from one of the channels, and confirm it works dispatch_channel_2.del_perm(self.user) LOGGER.debug("SC IS %s", sc) self.assertEqual(sc.get_outbound_channel(self.user), dispatch_channel_1)
def share_post(self, post, user, dry_run=False): # self.sync_contacts(post.user_profile) from solariat_bottle.tasks.twitter import tw_share_post post_content = post.plaintext_content status_id = post.native_id if dry_run is False and not get_var('ON_TEST') and get_var( 'APP_MODE') == 'prod': tw_share_post.ignore(self, status_id=status_id, screen_name=post.user_profile.user_name) else: create_outbound_post(user, self, "RT: %s" % post_content, post) LOGGER.debug("Retweet '%s' using %s", post_content, self)
def wrapper(*args, **kwargs): #LOGGER.debug("API Request Args: {} | Params: {}".format(args, kwargs)) assert args assert isinstance(args[0], BaseAPIView) view = args[0] # Method decorator try: args = args[1:] except IndexError: args = () params = _get_request_data() params.update(kwargs) # Pass URL variables to the view function start_execution = datetime.utcnow() # Execute API method try: # Assert authentication LOGGER.debug("Started executing API call: %s.%s(%s, %s) " % ( view.__class__.__name__, func.__name__, args, kwargs)) if allow_basic_auth is True: user = _get_user() if user is None: user = authenticate_api_user(params) if user is None: raise api_exc.AuthorizationError("User is not authenticated. Parameters for call: " + str(params)) else: user = authenticate_api_user(params) # Set user in thread local storage set_user(user) resp = func(view, user, *args, **params) elapsed = datetime.utcnow() - start_execution if elapsed.total_seconds() >= 2: log = LOGGER.info else: log = LOGGER.debug log("API call: %s.%s(%s,%s) finished after %s Parameters: %s" % (view.__class__.__name__, func.__name__, str(args)[:500], str(kwargs)[:500], elapsed, str(params)[:500])) # auth token expiration and other auth errors except api_exc.AuthorizationError, exc: LOGGER.info(exc) return view.format_api_error_response(exc, msg=str(exc), description=exc.description)
def send_message(self, dry_run, creative, post, user, direct_message=None): # self.sync_contacts(post.user_profile) from solariat_bottle.tasks.twitter import tw_normal_reply, tw_direct_reply is_dm = False # Assume this is not a DM if direct_message is not None: # If we specifically passed the fact that we want a direct message, use DM # Otherwise decide based on post type is_dm = direct_message else: if post.message_type == 'direct': is_dm = True if not is_dm: status = "@%s %s" % (post.user_profile.user_name, creative) else: status = creative if len(status) > 140: msg = ( 'Sorry, you have exceeded your 140 character limit by %d characters. ' 'Please correct your reply and try again.') % (len(status) - 140) raise AppException(msg) status_id = post.native_id # Update the engagement history post.user_profile.update_history(self) LOGGER.debug("For current message, direct message flag is %s", is_dm) if not dry_run and not get_var('ON_TEST'): if is_dm: tw_direct_reply.ignore(self, status=status, screen_name=post.user_profile.user_name) else: tw_normal_reply.ignore(self, status=status, status_id=status_id, post=post) else: create_outbound_post(user, self, creative, post) LOGGER.debug("Sent '%s' to %s using %s", creative, post.user_profile.user_name, self.title)
def do_get(self, path, version=None, **kw): "Emulate GET request" kw['token'] = self.auth_token path = get_api_url(path, version=self._get_version(version=version)) if '?' in path: path += '&' else: path +='?' base_url = kw.pop('base_url', 'https://localhost') path += urllib.urlencode(kw) LOGGER.debug( "Performing GET with %s" % path) return self._handle_http_response( self.client.get(path, base_url=base_url))
def set_dynamic_class(self, inheritance): ''' Support dynamic events ''' if DynamicEvent.__name__ in inheritance: # from solariat_bottle.db.dynamic_event import EventType from solariat_bottle.db.events.event_type import BaseEventType event_type_name = self.data[DynamicEvent.event_type.db_field] # event_type = EventType.objects.find_one(event_type_id) # TODO: check using get_user() is correct here acc_id = get_user().account.id LOGGER.debug('Set event dynamic class, use acc: %s for find event types', acc_id) event_type = BaseEventType.objects.find_one_by_display_name(acc_id, event_type_name) if event_type: dyn_cls = event_type.get_data_class() # initialize class in Registry self.__class__ = dyn_cls else: LOGGER.error('Cannot find suitable event type for dynamic class!')
def send_email(self, attachments): from solariat_bottle.utils.mailer import \ send_mail, Message, _get_sender, render_template rcps = self.recipient_emails self.export_item.update(set__recipient_emails=rcps) msg = Message(subject=DataExportMailer.EMAIL_SUBJECT, sender=_get_sender(), recipients=rcps) params = self.get_email_parameters() msg.html = render_template("mail/export/posts.html", **params) msg.body = render_template("mail/export/posts.txt", **params) from solariat_bottle.settings import LOGGER LOGGER.debug(msg.body) for (filename, mimetype, stream) in attachments: msg.attach(filename, mimetype, stream.getvalue()) send_mail(msg)
def generate_csv(cls, iterable_data, context): header = [ 'Channel Type', 'Channel Name', 'Inbound/Outbound', 'Post/Comment', 'Initial Post', 'Date/Time (UTC)', 'Smart Tags', 'Intentions', 'Post Status', 'Topics' ] def get_post_content(p): _get = lambda p, attr: \ p.get(attr) if isinstance(p, dict) else getattr(p, attr) attr = 'url' if context.get('platform') == 'Twitter' else 'content' return unwrap_hidden(_get(p, attr)) def get_initial_post(p): if p['parent']: return get_post_content(p['parent']) return '' value_getters = [ cls.PrefetchedValue(context.get('platform')), cls.PrefetchedValue(context.get('channel_name')), cls.PrefetchedValue(context.get('channel_direction')), get_post_content, get_initial_post, 'created_at', 'smart_tags', 'intentions', 'filter_status', 'topics' ] stream = StringIO() writer = UnicodeWriter(stream) writer.writerow(header) from solariat_bottle.settings import LOGGER for item in iterable_data: try: row = cls.csv_row(value_getters, item) except Exception: LOGGER.exception(u"Cannot generate csv tuple for %s" % item) else: LOGGER.debug(row) writer.writerow(row) stream.seek(0) return stream
def sync_streams(self): """ In case new channels are added, or old ones are suspended / logged out we need to also reflect those changes in the UserStreams we are tracking. """ LOGGER.debug('synchronizing streams') _channels = {} for ustream in self.handles: _channels[ustream.channel] = ustream new_channels = {} # Get a list of all the EnterpriseTwitterChannel channels that are currently active for channel in EnterpriseTwitterChannel.objects(status='Active'): try: if (channel.access_token_key and channel.access_token_secret): options = self._get_stream_opts(channel) new_channels[options.channel] = options except Exception, ex: LOGGER.error("'%s' channel failed: %s", channel, ex)