def mass_date_save(cache_key):
try: # always capture exceptions with Sentry
task_name = 'MASS_DATE_SAVE'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
# log to gateway server
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
r = RedisClient()
res = r.get_list(cache_key)
date_list = []
for date in res:
date_inst = Date(date=date)
date_list.append(date_inst)
Date.objects.bulk_create(date_list)
logger.set_log(task_name, 'P', 'saved mass_date data to Django DB')
# delete duplicates if existent
unique_dates = Date.objects.distinct('date')
deleted = Date.objects.exclude(pk__in=unique_dates).delete()
if deleted != 0:
logger.set_log(task_name, 'P', 'deleted duplicate dates in DB')
# delete cache key after done to prevent duplicates in cache server
cache_deleted = r.del_key(cache_key)
if cache_deleted:
logger.set_log(task_name, 'P', 'deleted cache key in cache server')
else:
logger.set_log(task_name, 'F', 'failed to delete cache key')
return True
except:
client.captureException()
def restart_test():
task_name = 'RESTART_TEST'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
# delete all TEST action related gateway logs
GatewayState.objects.filter(task_name='TEST').delete()
test_logs_num = GatewayState.objects.filter(task_name='TEST').count()
if test_logs_num == 0:
logger.set_log(task_name, 'P', 'deleted all TEST action related logs')
# test redis can delete data
r = RedisClient()
res = r.del_key('TEST')
if res == 1:
logger.set_log(task_name, 'P', 'deleted key: {}'.format('TEST'))
cache.delete('TEST2')
logger.set_log(task_name, 'P', 'deleted key: {}'.format('TEST2'))
# as a finishing touch retest TEST action by sending action
task_sender.send_task('TEST')
logger.set_log(task_name, 'P', 'COMPLETE')
return True
def __init__(self, current_task):
# first set logger
self.logger = GatewayLogger()
# define API endpoint
self.gateway_ip = CONFIG['ip-address']['gateway']
self.task_url = 'http://{}/hidden-api/task/?type='.format(
self.gateway_ip)
# needs current_task for loggin purposes
self.current_task = current_task
def mass_sd_crawl():
task_name = 'MASS_SD_CRAWL'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
# log to gateway server
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
local('node /home/arbiter/js-gobble/{}.js'.format(task_name))
logger.set_log(
task_name, 'P',
'running "node /home/arbiter/js-gobble/{}.js"'.format(task_name))
return True
def get_wonseok_test():
try:
task_name = 'GET_WONSEOK_TEST'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
r = RedisClient()
r.set_key('wonseok_test_from_local', 'this is a test value')
logger.set_log(task_name, 'P', 'key value set to cache')
task_sender.send_task('SAVE_WONSEOK_TEST')
return True
except:
client.captureException()
def mass_date_crawl():
try: # always capture exceptions with Sentry
task_name = 'MASS_DATE_CRAWL'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
# log to gateway server
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
local('node /home/arbiter/js-gobble/{}.js'.format(task_name))
logger.set_log(
task_name, 'P',
'running "node /home/arbiter/js-gobble/{}.js"'.format(task_name))
return True
except:
client.captureException()
def save_wonseok_test():
try:
task_name = 'SAVE_WONSEOK_TEST'
logger = GatewayLogger()
task_sender = TaskSender(task_name)
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
r = RedisClient()
value = r.get_key('wonseok_test_from_local')
soul_log_data = SoulLog(log=value)
soul_log_data.save()
logger.set_log(task_name, 'P', 'key value saved to DB')
return True
except:
client.captureException()
def get(self, request):
# receive a type value through URL
action_type = request.GET.get('type')
# create the logger
logger = GatewayLogger()
# forward to gateway server when requested to a different server
if THIS_SYSTEM != 'gateway':
logger.set_log(
action_type, 'P',
'requested task server not gateway, forwarding request to gateway server'
)
task_sender = TaskSender(action_type)
task_sender = task_sender.send_task(action_type)
return JsonResponse({'status': 'FORWARDED'},
json_dumps_params={'ensure_ascii': True})
else:
# check for action_type availability
action_inst = GatewayActionOBJ(action_type)
if action_inst.action_exists():
logger.set_log(action_type, 'P', 'store received action type')
# initialize action class by passing in the action type retrieved from URL
reducer_inst = GatewayReducer(
action_inst) # pass in the action object to reducer
reducer_status = reducer_inst.reduce()
if reducer_status == True:
logger.set_log(action_type, 'P',
'action successfully reduced')
return JsonResponse(
{'status': 'DONE'},
json_dumps_params={'ensure_ascii': True})
else:
logger.set_log(action_type, 'F', 'action failed to reduce')
return JsonResponse(
{'status': 'FAIL'},
json_dumps_params={'ensure_ascii': True})
elif not action_inst.action_exists():
logger.set_log(action_type, 'F', 'no such action exists error')
return JsonResponse(
{'status': 'NO ACTION: {}'.format(action_type)},
json_dumps_params={'ensure_ascii': True})
def test():
task_name = 'TEST'
logger = GatewayLogger()
logger.set_log(task_name, 'P',
'gobble server received task: {}'.format(task_name))
# test redis can set/get data
r = RedisClient()
r.set_key('TEST', 'testing gateway API data flow')
res = r.get_key('TEST')
logger.set_log(
task_name, 'P',
'TEST 1: cache has key: {0}, with value: {1}'.format('TEST', res))
cache.set('TEST2', 'testing Django cache set/get')
res2 = cache.get('TEST2')
logger.set_log(
task_name, 'P',
'TEST 2: django cache has key: {0}, with value: {1}'.format(
'TEST2', res2))
logger.set_log(task_name, 'P', 'COMPLETE')
return True
def sentry_enabled_reducer_function(*args, **kwargs):
reducer_name = reducer_function.__qualname__
task_name = reducer_name.upper()
c = Controller(task_name)
c.transform_action_definition_to_variables()
kwargs = {}
# change all dictionary key values to variables
for key, val in c.__dict__.items():
if key != 'action':
kwargs[key] = val
reduce_start_time = time.time()
try:
reducer_function(**kwargs)
except:
client.captureException() # capture error with Sentry
reduce_end_time = time.time()
took_time = reduce_end_time - reduce_start_time
logger = GatewayLogger()
logger.set_log(task_name, 'P', 'reducer {0} ran in {1} s'.format(reducer_name, took_time))