def _create_unique_file(temp): temp._check_open() with lockfile.LockFile(join(temp.dpath, 'tempfile.lock')): flag = True while flag or exists(temp.fpath): temp.fname = six.text_type(uuid.uuid4()) + '.temp' temp.fpath = join(temp.dpath, temp.fname) flag = False ut.touch(temp.fpath, verbose=temp.verbose)
def fix_importlib_hook(): """ IMPORTLIB FIX References: http://stackoverflow.com/questions/18596410/importerror-no-module-named-mpl-toolkits-with-maptlotlib-1-3-0-and-py2exe """ try: dpath_ = importlib.import_module('mpl_toolkits').__path__ if isinstance(dpath_, (list, tuple)): for dpath in dpath_: fpath = join(dpath, '__init__.py') break else: dpath = dpath_ if not ut.checkpath(dpath, verbose=True, info=True): ut.touch(fpath) except ImportError as ex: ut.printex(ex, 'pip install mpl_toolkits?')
def on_collect_request(collect_request, collecter_data, awaiting_data, shelve_path, containerized=False): """ Run whenever the collector recieves a message """ import requests reply = {} action = collect_request['action'] if VERBOSE_JOBS: print('...building action=%r response' % (action, )) if action == 'notification': # From the Queue jobid = collect_request['jobid'] awaiting_data[jobid] = collect_request['text'] # Make waiting lock lock_filepath = join(shelve_path, '%s.lock' % (jobid, )) ut.touch(lock_filepath) elif action == 'store': # From the Engine engine_result = collect_request['engine_result'] callback_url = collect_request['callback_url'] callback_method = collect_request['callback_method'] jobid = engine_result['jobid'] if containerized: callback_url = callback_url.replace('://localhost/', '://wildbook:8080/') # OLD METHOD # collecter_data[jobid] = engine_result collecter_data[jobid] = engine_result['exec_status'] # NEW METHOD shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, )) shelf = shelve.open(shelve_filepath, writeback=True) try: shelf[str('result')] = engine_result finally: shelf.close() # Delete the lock lock_filepath = join(shelve_path, '%s.lock' % (jobid, )) ut.delete(lock_filepath) if callback_url is not None: if callback_method is None: callback_method = 'post' else: callback_method = callback_method.lower() if VERBOSE_JOBS: print('calling callback_url using callback_method') try: # requests.get(callback_url) data_dict = {'jobid': jobid} if callback_method == 'post': response = requests.post(callback_url, data=data_dict) elif callback_method == 'get': response = requests.get(callback_url, params=data_dict) elif callback_method == 'put': response = requests.put(callback_url, data=data_dict) else: raise ValueError('callback_method %r unsupported' % (callback_method, )) try: text = response.text except: text = None args = ( callback_url, callback_method, data_dict, response, text, ) print( 'WILDBOOK CALLBACK TO %r\n\tMETHOD: %r\n\tDATA: %r\n\tRESPONSE: %r\n\tTEXT: %r' % args) except Exception as ex: msg = ( ('ERROR in collector. ' 'Tried to call callback_url=%r with callback_method=%r') % ( callback_url, callback_method, )) print(msg) ut.printex(ex, msg) #requests.post(callback_url) if VERBOSE_JOBS: print('stored result') elif action == 'job_status': # From a Client jobid = collect_request['jobid'] if jobid in collecter_data: reply['jobstatus'] = 'completed' reply['exec_status'] = collecter_data[jobid] elif jobid in awaiting_data: reply['jobstatus'] = 'working' else: reply['jobstatus'] = 'unknown' reply['status'] = 'ok' reply['jobid'] = jobid elif action == 'job_id_list': reply['status'] = 'ok' reply['jobid_list'] = list(collecter_data.keys()) elif action == 'job_result': # From a Client jobid = collect_request['jobid'] try: # OLD METHOD # engine_result = collecter_data[jobid] # NEW METHOD shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, )) shelf = shelve.open(shelve_filepath) try: engine_result = shelf[str('result')] finally: shelf.close() json_result = engine_result['json_result'] reply['jobid'] = jobid reply['status'] = 'ok' # reply['json_result'] = json_result # We want to parse the JSON result here, since we need to live in # Python land for the rest of the call until the API wrapper # converts the Python objcets to JSON before the response is # generated. This prevents the API from converting a Python string # of JSON to a JSON string of JSON, which is bad. reply['json_result'] = ut.from_json(json_result) except KeyError: reply['jobid'] = jobid reply['status'] = 'invalid' reply['json_result'] = None else: # Other print('...error unknown action=%r' % (action, )) reply['status'] = 'error' reply['text'] = 'unknown action' return reply
def on_collect_request(collect_request, collecter_data, awaiting_data, shelve_path): """ Run whenever the collector recieves a message """ import requests reply = {} action = collect_request['action'] if VERBOSE_JOBS: print('...building action=%r response' % (action,)) if action == 'notification': # From the Queue jobid = collect_request['jobid'] awaiting_data[jobid] = collect_request['text'] # Make waiting lock lock_filepath = join(shelve_path, '%s.lock' % (jobid, )) ut.touch(lock_filepath) elif action == 'store': # From the Engine engine_result = collect_request['engine_result'] callback_url = collect_request['callback_url'] callback_method = collect_request['callback_method'] jobid = engine_result['jobid'] # OLD METHOD # collecter_data[jobid] = engine_result collecter_data[jobid] = engine_result['exec_status'] # NEW METHOD shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, )) shelf = shelve.open(shelve_filepath, writeback=True) try: shelf[str('result')] = engine_result finally: shelf.close() # Delete the lock lock_filepath = join(shelve_path, '%s.lock' % (jobid, )) ut.delete(lock_filepath) if callback_url is not None: if callback_method is None: callback_method = 'post' else: callback_method = callback_method.lower() if VERBOSE_JOBS: print('calling callback_url using callback_method') try: # requests.get(callback_url) if callback_method == 'post': requests.post(callback_url, data={'jobid': jobid}) elif callback_method == 'get': requests.get(callback_url, params={'jobid': jobid}) elif callback_method == 'put': requests.put(callback_url, data={'jobid': jobid}) else: raise ValueError('callback_method %r unsupported' % (callback_method, )) except Exception as ex: msg = (('ERROR in collector. ' 'Tried to call callback_url=%r with callback_method=%r') % (callback_url, callback_method, )) print(msg) ut.printex(ex, msg) #requests.post(callback_url) if VERBOSE_JOBS: print('stored result') elif action == 'job_status': # From a Client jobid = collect_request['jobid'] if jobid in collecter_data: reply['jobstatus'] = 'completed' reply['exec_status'] = collecter_data[jobid] elif jobid in awaiting_data: reply['jobstatus'] = 'working' else: reply['jobstatus'] = 'unknown' reply['status'] = 'ok' reply['jobid'] = jobid elif action == 'job_result': # From a Client jobid = collect_request['jobid'] try: # OLD METHOD # engine_result = collecter_data[jobid] # NEW METHOD shelve_filepath = join(shelve_path, '%s.shelve' % (jobid, )) shelf = shelve.open(shelve_filepath) try: engine_result = shelf[str('result')] finally: shelf.close() json_result = engine_result['json_result'] reply['jobid'] = jobid reply['status'] = 'ok' # reply['json_result'] = json_result # We want to parse the JSON result here, since we need to live in # Python land for the rest of the call until the API wrapper # converts the Python objcets to JSON before the response is # generated. This prevents the API from converting a Python string # of JSON to a JSON string of JSON, which is bad. reply['json_result'] = ut.from_json(json_result) except KeyError: reply['jobid'] = jobid reply['status'] = 'invalid' reply['json_result'] = None else: # Other print('...error unknown action=%r' % (action,)) reply['status'] = 'error' reply['text'] = 'unknown action' return reply