def __init__(self, current_task): # first set logger self.logger = GatewayLogger() # define API endpoint self.gateway_ip = CONFIG['ip-address']['gateway'] self.task_url = 'http://{}/hidden-api/task/?type='.format( self.gateway_ip) # needs current_task for loggin purposes self.current_task = current_task
def test(): task_name = 'TEST' logger = GatewayLogger() logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) # test redis can set/get data r = RedisClient() r.set_key('TEST', 'testing gateway API data flow') res = r.get_key('TEST') logger.set_log( task_name, 'P', 'TEST 1: cache has key: {0}, with value: {1}'.format('TEST', res)) cache.set('TEST2', 'testing Django cache set/get') res2 = cache.get('TEST2') logger.set_log( task_name, 'P', 'TEST 2: django cache has key: {0}, with value: {1}'.format( 'TEST2', res2)) logger.set_log(task_name, 'P', 'COMPLETE') return True
class TaskSender(object): def __init__(self, current_task): # first set logger self.logger = GatewayLogger() # define API endpoint self.gateway_ip = CONFIG['ip-address']['gateway'] self.task_url = 'http://{}/hidden-api/task/?type='.format( self.gateway_ip) # needs current_task for loggin purposes self.current_task = current_task def send_task(self, next_task): # define API endpoint with new task name task_url = self.task_url + next_task # make API get request r = requests.get(task_url) reply = r.json( )['status'] # status can return either of: DONE, FAIL, NO ACTION: ... if reply == 'DONE': self.logger.set_log(self.current_task, 'P', 'new task sent: {}'.format(next_task)) elif reply == 'FAIL': self.logger.set_log( self.current_task, 'F', 'failed to send new task: {}'.format(next_task)) elif 'NO ACTION' in reply: self.logger.set_log(self.current_task, 'F', 'no action called: {}'.format(next_task))
def sentry_enabled_reducer_function(*args, **kwargs): reducer_name = reducer_function.__qualname__ task_name = reducer_name.upper() c = Controller(task_name) c.transform_action_definition_to_variables() kwargs = {} # change all dictionary key values to variables for key, val in c.__dict__.items(): if key != 'action': kwargs[key] = val reduce_start_time = time.time() try: reducer_function(**kwargs) except: client.captureException() # capture error with Sentry reduce_end_time = time.time() took_time = reduce_end_time - reduce_start_time logger = GatewayLogger() logger.set_log(task_name, 'P', 'reducer {0} ran in {1} s'.format(reducer_name, took_time))
def mass_sd_crawl(): task_name = 'MASS_SD_CRAWL' logger = GatewayLogger() task_sender = TaskSender(task_name) # log to gateway server logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) local('node /home/arbiter/js-gobble/{}.js'.format(task_name)) logger.set_log( task_name, 'P', 'running "node /home/arbiter/js-gobble/{}.js"'.format(task_name)) return True
def get_wonseok_test(): try: task_name = 'GET_WONSEOK_TEST' logger = GatewayLogger() task_sender = TaskSender(task_name) logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) r = RedisClient() r.set_key('wonseok_test_from_local', 'this is a test value') logger.set_log(task_name, 'P', 'key value set to cache') task_sender.send_task('SAVE_WONSEOK_TEST') return True except: client.captureException()
def mass_date_crawl(): try: # always capture exceptions with Sentry task_name = 'MASS_DATE_CRAWL' logger = GatewayLogger() task_sender = TaskSender(task_name) # log to gateway server logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) local('node /home/arbiter/js-gobble/{}.js'.format(task_name)) logger.set_log( task_name, 'P', 'running "node /home/arbiter/js-gobble/{}.js"'.format(task_name)) return True except: client.captureException()
def save_wonseok_test(): try: task_name = 'SAVE_WONSEOK_TEST' logger = GatewayLogger() task_sender = TaskSender(task_name) logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) r = RedisClient() value = r.get_key('wonseok_test_from_local') soul_log_data = SoulLog(log=value) soul_log_data.save() logger.set_log(task_name, 'P', 'key value saved to DB') return True except: client.captureException()
def restart_test(): task_name = 'RESTART_TEST' logger = GatewayLogger() task_sender = TaskSender(task_name) logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) # delete all TEST action related gateway logs GatewayState.objects.filter(task_name='TEST').delete() test_logs_num = GatewayState.objects.filter(task_name='TEST').count() if test_logs_num == 0: logger.set_log(task_name, 'P', 'deleted all TEST action related logs') # test redis can delete data r = RedisClient() res = r.del_key('TEST') if res == 1: logger.set_log(task_name, 'P', 'deleted key: {}'.format('TEST')) cache.delete('TEST2') logger.set_log(task_name, 'P', 'deleted key: {}'.format('TEST2')) # as a finishing touch retest TEST action by sending action task_sender.send_task('TEST') logger.set_log(task_name, 'P', 'COMPLETE') return True
def mass_date_save(cache_key): try: # always capture exceptions with Sentry task_name = 'MASS_DATE_SAVE' logger = GatewayLogger() task_sender = TaskSender(task_name) # log to gateway server logger.set_log(task_name, 'P', 'gobble server received task: {}'.format(task_name)) r = RedisClient() res = r.get_list(cache_key) date_list = [] for date in res: date_inst = Date(date=date) date_list.append(date_inst) Date.objects.bulk_create(date_list) logger.set_log(task_name, 'P', 'saved mass_date data to Django DB') # delete duplicates if existent unique_dates = Date.objects.distinct('date') deleted = Date.objects.exclude(pk__in=unique_dates).delete() if deleted != 0: logger.set_log(task_name, 'P', 'deleted duplicate dates in DB') # delete cache key after done to prevent duplicates in cache server cache_deleted = r.del_key(cache_key) if cache_deleted: logger.set_log(task_name, 'P', 'deleted cache key in cache server') else: logger.set_log(task_name, 'F', 'failed to delete cache key') return True except: client.captureException()
def get(self, request): # receive a type value through URL action_type = request.GET.get('type') # create the logger logger = GatewayLogger() # forward to gateway server when requested to a different server if THIS_SYSTEM != 'gateway': logger.set_log( action_type, 'P', 'requested task server not gateway, forwarding request to gateway server' ) task_sender = TaskSender(action_type) task_sender = task_sender.send_task(action_type) return JsonResponse({'status': 'FORWARDED'}, json_dumps_params={'ensure_ascii': True}) else: # check for action_type availability action_inst = GatewayActionOBJ(action_type) if action_inst.action_exists(): logger.set_log(action_type, 'P', 'store received action type') # initialize action class by passing in the action type retrieved from URL reducer_inst = GatewayReducer( action_inst) # pass in the action object to reducer reducer_status = reducer_inst.reduce() if reducer_status == True: logger.set_log(action_type, 'P', 'action successfully reduced') return JsonResponse( {'status': 'DONE'}, json_dumps_params={'ensure_ascii': True}) else: logger.set_log(action_type, 'F', 'action failed to reduce') return JsonResponse( {'status': 'FAIL'}, json_dumps_params={'ensure_ascii': True}) elif not action_inst.action_exists(): logger.set_log(action_type, 'F', 'no such action exists error') return JsonResponse( {'status': 'NO ACTION: {}'.format(action_type)}, json_dumps_params={'ensure_ascii': True})