def _create_unique_file(temp):
temp._check_open()
with lockfile.LockFile(join(temp.dpath, 'tempfile.lock')):
flag = True
while flag or exists(temp.fpath):
temp.fname = six.text_type(uuid.uuid4()) + '.temp'
temp.fpath = join(temp.dpath, temp.fname)
flag = False
ut.touch(temp.fpath, verbose=temp.verbose)
def fix_importlib_hook():
""" IMPORTLIB FIX
References:
http://stackoverflow.com/questions/18596410/importerror-no-module-named-mpl-toolkits-with-maptlotlib-1-3-0-and-py2exe
"""
try:
dpath_ = importlib.import_module('mpl_toolkits').__path__
if isinstance(dpath_, (list, tuple)):
for dpath in dpath_:
fpath = join(dpath, '__init__.py')
break
else:
dpath = dpath_
if not ut.checkpath(dpath, verbose=True, info=True):
ut.touch(fpath)
except ImportError as ex:
ut.printex(ex, 'pip install mpl_toolkits?')
def fix_importlib_hook():
""" IMPORTLIB FIX
References:
http://stackoverflow.com/questions/18596410/importerror-no-module-named-mpl-toolkits-with-maptlotlib-1-3-0-and-py2exe
"""
try:
dpath_ = importlib.import_module('mpl_toolkits').__path__
if isinstance(dpath_, (list, tuple)):
for dpath in dpath_:
fpath = join(dpath, '__init__.py')
break
else:
dpath = dpath_
if not ut.checkpath(dpath, verbose=True, info=True):
ut.touch(fpath)
except ImportError as ex:
ut.printex(ex, 'pip install mpl_toolkits?')
def on_collect_request(collect_request,
collecter_data,
awaiting_data,
shelve_path,
containerized=False):
""" Run whenever the collector recieves a message """
import requests
reply = {}
action = collect_request['action']
if VERBOSE_JOBS:
print('...building action=%r response' % (action, ))
if action == 'notification':
# From the Queue
jobid = collect_request['jobid']
awaiting_data[jobid] = collect_request['text']
# Make waiting lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.touch(lock_filepath)
elif action == 'store':
# From the Engine
engine_result = collect_request['engine_result']
callback_url = collect_request['callback_url']
callback_method = collect_request['callback_method']
jobid = engine_result['jobid']
if containerized:
callback_url = callback_url.replace('://localhost/',
'://wildbook:8080/')
# OLD METHOD
# collecter_data[jobid] = engine_result
collecter_data[jobid] = engine_result['exec_status']
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath, writeback=True)
try:
shelf[str('result')] = engine_result
finally:
shelf.close()
# Delete the lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.delete(lock_filepath)
if callback_url is not None:
if callback_method is None:
callback_method = 'post'
else:
callback_method = callback_method.lower()
if VERBOSE_JOBS:
print('calling callback_url using callback_method')
try:
# requests.get(callback_url)
data_dict = {'jobid': jobid}
if callback_method == 'post':
response = requests.post(callback_url, data=data_dict)
elif callback_method == 'get':
response = requests.get(callback_url, params=data_dict)
elif callback_method == 'put':
response = requests.put(callback_url, data=data_dict)
else:
raise ValueError('callback_method %r unsupported' %
(callback_method, ))
try:
text = response.text
except:
text = None
args = (
callback_url,
callback_method,
data_dict,
response,
text,
)
print(
'WILDBOOK CALLBACK TO %r\n\tMETHOD: %r\n\tDATA: %r\n\tRESPONSE: %r\n\tTEXT: %r'
% args)
except Exception as ex:
msg = (
('ERROR in collector. '
'Tried to call callback_url=%r with callback_method=%r') %
(
callback_url,
callback_method,
))
print(msg)
ut.printex(ex, msg)
#requests.post(callback_url)
if VERBOSE_JOBS:
print('stored result')
elif action == 'job_status':
# From a Client
jobid = collect_request['jobid']
if jobid in collecter_data:
reply['jobstatus'] = 'completed'
reply['exec_status'] = collecter_data[jobid]
elif jobid in awaiting_data:
reply['jobstatus'] = 'working'
else:
reply['jobstatus'] = 'unknown'
reply['status'] = 'ok'
reply['jobid'] = jobid
elif action == 'job_id_list':
reply['status'] = 'ok'
reply['jobid_list'] = list(collecter_data.keys())
elif action == 'job_result':
# From a Client
jobid = collect_request['jobid']
try:
# OLD METHOD
# engine_result = collecter_data[jobid]
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath)
try:
engine_result = shelf[str('result')]
finally:
shelf.close()
json_result = engine_result['json_result']
reply['jobid'] = jobid
reply['status'] = 'ok'
# reply['json_result'] = json_result
# We want to parse the JSON result here, since we need to live in
# Python land for the rest of the call until the API wrapper
# converts the Python objcets to JSON before the response is
# generated. This prevents the API from converting a Python string
# of JSON to a JSON string of JSON, which is bad.
reply['json_result'] = ut.from_json(json_result)
except KeyError:
reply['jobid'] = jobid
reply['status'] = 'invalid'
reply['json_result'] = None
else:
# Other
print('...error unknown action=%r' % (action, ))
reply['status'] = 'error'
reply['text'] = 'unknown action'
return reply
def on_collect_request(collect_request, collecter_data, awaiting_data, shelve_path):
""" Run whenever the collector recieves a message """
import requests
reply = {}
action = collect_request['action']
if VERBOSE_JOBS:
print('...building action=%r response' % (action,))
if action == 'notification':
# From the Queue
jobid = collect_request['jobid']
awaiting_data[jobid] = collect_request['text']
# Make waiting lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.touch(lock_filepath)
elif action == 'store':
# From the Engine
engine_result = collect_request['engine_result']
callback_url = collect_request['callback_url']
callback_method = collect_request['callback_method']
jobid = engine_result['jobid']
# OLD METHOD
# collecter_data[jobid] = engine_result
collecter_data[jobid] = engine_result['exec_status']
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath, writeback=True)
try:
shelf[str('result')] = engine_result
finally:
shelf.close()
# Delete the lock
lock_filepath = join(shelve_path, '%s.lock' % (jobid, ))
ut.delete(lock_filepath)
if callback_url is not None:
if callback_method is None:
callback_method = 'post'
else:
callback_method = callback_method.lower()
if VERBOSE_JOBS:
print('calling callback_url using callback_method')
try:
# requests.get(callback_url)
if callback_method == 'post':
requests.post(callback_url, data={'jobid': jobid})
elif callback_method == 'get':
requests.get(callback_url, params={'jobid': jobid})
elif callback_method == 'put':
requests.put(callback_url, data={'jobid': jobid})
else:
raise ValueError('callback_method %r unsupported' %
(callback_method, ))
except Exception as ex:
msg = (('ERROR in collector. '
'Tried to call callback_url=%r with callback_method=%r')
% (callback_url, callback_method, ))
print(msg)
ut.printex(ex, msg)
#requests.post(callback_url)
if VERBOSE_JOBS:
print('stored result')
elif action == 'job_status':
# From a Client
jobid = collect_request['jobid']
if jobid in collecter_data:
reply['jobstatus'] = 'completed'
reply['exec_status'] = collecter_data[jobid]
elif jobid in awaiting_data:
reply['jobstatus'] = 'working'
else:
reply['jobstatus'] = 'unknown'
reply['status'] = 'ok'
reply['jobid'] = jobid
elif action == 'job_result':
# From a Client
jobid = collect_request['jobid']
try:
# OLD METHOD
# engine_result = collecter_data[jobid]
# NEW METHOD
shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, ))
shelf = shelve.open(shelve_filepath)
try:
engine_result = shelf[str('result')]
finally:
shelf.close()
json_result = engine_result['json_result']
reply['jobid'] = jobid
reply['status'] = 'ok'
# reply['json_result'] = json_result
# We want to parse the JSON result here, since we need to live in
# Python land for the rest of the call until the API wrapper
# converts the Python objcets to JSON before the response is
# generated. This prevents the API from converting a Python string
# of JSON to a JSON string of JSON, which is bad.
reply['json_result'] = ut.from_json(json_result)
except KeyError:
reply['jobid'] = jobid
reply['status'] = 'invalid'
reply['json_result'] = None
else:
# Other
print('...error unknown action=%r' % (action,))
reply['status'] = 'error'
reply['text'] = 'unknown action'
return reply
