def methods(): gbix_methods_cached = cache_get('/api/methods', cache_options['default']['name']) if not gbix_methods_cached: try: arq = os.path.join( os.path.split(os.path.split(os.path.realpath(__file__))[0])[0], 'doc/api_data.json') f = open(arq) except IOError: return 'Methods file not found.' methods_json = str() pattern = re.compile('\s+"filename') for line in f.readlines(): if re.match(pattern, line) is None: methods_json += line.strip().replace('<p>', '').replace( '</p>', '').replace(' "', '"').replace('\\"', '') f.close() cache_set('/api/methods', methods_json, 0, cache_options['default']['name']) return methods_json else: return gbix_methods_cached
def index(): try: url = request.params['url'] except KeyError: abort(400, 'Please provide a url') return # only for pylint url_hash = hash(url) cache_key = str(url_hash) lock_idx = url_hash % LOCKS with lock(lock_idx): cache = uwsgi.cache_get(cache_key) if cache == 'd': return invalid() if cache == 'u': return url try: req = urllib2.Request( url, headers={'User-Agent': USERAGENT} ) urllib2.urlopen(req, timeout=10) except: uwsgi.cache_set(cache_key, 'd', EXPIRE) return invalid() else: uwsgi.cache_set(cache_key, 'u', EXPIRE) return url
def wrapper(*args, **kwargs): wrapper.cfg = config_method(method) # __wrapped__ cache_id = wrapper.cfg.get(METHOD_CACHE_ID, None) cache_on = bool(cache_id) # not not cache_id cache_expires = int(wrapper.cfg.get(METHOD_CACHE_EXPIRES, "0")) args_spec = inspect.getfullargspec(func).args if cache_key_name in kwargs: key_val = kwargs[ cache_key_name] if cache_key_name in args_spec else kwargs.pop( cache_key_name) else: key_val = args[args_spec.index( cache_key_name)] # if not exists KeyError will be raised if isinstance(key_val, dict): key_val = hash( tuple(sorted(key_val.items(), key=lambda item: item[0]))) key_val = f'{method}:{str(key_val)}' if cache_on and uwsgi.cache_exists(key_val, cache_id): value = uwsgi.cache_get(key_val, cache_id) value = pickle.loads(value) logging.debug('cache [%s] -> %s : %s', cache_id, key_val, value) return value result = func(*args, **kwargs) if cache_on: value = pickle.dumps(result) uwsgi.cache_set(key_val, value, cache_expires, cache_id) logging.debug('cache [%s] <- %s expires %d : %s', cache_id, key_val, cache_expires, value) return result
def memrefresh(feedurl): value = jsonld_from_feedparser(loaded_feed(unquote(feedurl),repo,True).feed) if cache_exists(feedurl): cache_update(feedurl,value,3600*24) else: cache_set(feedurl,value,3600*24) return value
def memfeed(feedurl): if cache_exists(feedurl): return cache_get(feedurl) else: value = jsonld_from_feedparser(loaded_feed(unquote(feedurl),repo).feed) cache_set(feedurl,value,3600*24) return value
def log_event(self, event): """Adds an impression to the log if it is enabled, otherwise the impression is dropped. :param impression: The impression tuple :type impression: Impression """ cache_event = dict(event._asdict()) try: self.__lock_events() if self._adapter.cache_exists(self._EVENTS_KEY, _SPLITIO_STATS_CACHE_NAMESPACE): events = decode( self._adapter.cache_get(self._EVENTS_KEY, _SPLITIO_STATS_CACHE_NAMESPACE)) else: events = [] if len(events) < self._events_queue_size: events.append(cache_event) _logger.debug('Adding event to cache: {}.'.format(event)) self._adapter.cache_update(self._EVENTS_KEY, encode(events), 0, _SPLITIO_STATS_CACHE_NAMESPACE) return True # Set a key to force an events flush uwsgi.cache_set(self._EVENTS_FLUSH, '1', 0, _SPLITIO_STATS_CACHE_NAMESPACE) return False finally: self.__unlock_events()
def demo_set(): # demo = TestDemo() # print(str(demo)) # print(sys.getsizeof(demo)) demo = 'a' * 2000 print('++++++++++++++++++++++++++') print('Size:', sys.getsizeof(demo)) print('++++++++++++++++++++++++++') try: # for k, v in uwsgi.__dict__.iteritems(): # print('{}: {}'.format(k, v)) # from remote_pdb import RemotePdb # RemotePdb('127.0.0.1', 4444).set_trace() # demo = uwsgi.cache_exists('demo') global num if num > 10: num = 0 key = 'cache_{}'.format(num) demo = pickle.dumps(demo) uwsgi.cache_set(key, demo, 0, 'mycache') print(len(uwsgi.cache_get(key, 'mycache'))) num += 1 return str(num) except: import traceback print(traceback.print_exc()) return 'not set'
def memopml(opmlurl): if cache_exists(opmlurl): return cache_get(opmlurl) else: value = jsonld_from_opml(loaded_opml(unquote(opmlurl),repo).opml) cache_set(opmlurl,value,3600*24) return value
def set_value(key, value): if UWSGI: if uwsgi.cache_exists(key): uwsgi.cache_update(key, pickle.dumps(value)) else: uwsgi.cache_set(key, pickle.dumps(value)) else: _cache[key] = value
def test_non_bitmap(self): self.assertTrue(uwsgi.cache_set('KEY', 'X' * 20, 0, 'items_non_bitmap')) self.assertTrue(uwsgi.cache_del('KEY', 'items_non_bitmap')) self.assertIsNone( uwsgi.cache_set('KEY', 'X' * 21, 0, 'items_non_bitmap')) self.assertTrue(uwsgi.cache_set('KEY', 'X' * 20, 0, 'items_non_bitmap'))
def set(self, key, value, expires=0): key = str(key) value = pickle.dumps(value) if uwsgi.cache_exists(key, self.cachename): uwsgi.cache_update(key, value, expires, self.cachename) else: uwsgi.cache_set(key, value, expires, self.cachename) self._keys.add(key)
def test_big_random(self): blob = self.rand_blob(100000) self.assertTrue(uwsgi.cache_set('KEY', blob, 0, 'items_1_100000')) get_blob = uwsgi.cache_get('KEY', 'items_1_100000') self.assertEqual(blob, get_blob) self.assertTrue(uwsgi.cache_del('KEY', 'items_1_100000')) self.assertIsNone(uwsgi.cache_set('KEY', 'X' * 100001, 0, 'items_1_100000')) self.assertTrue(uwsgi.cache_set('KEY', 'X' * 10000, 0, 'items_1_100000'))
def test_lru(self): self.assertTrue(uwsgi.cache_set('KEY1', 'X' * 20, 0, 'items_lru')) self.assertTrue(uwsgi.cache_set('KEY2', 'X' * 20, 0, 'items_lru')) self.assertTrue(uwsgi.cache_set('KEY3', 'Y' * 20, 0, 'items_lru')) self.assertIsNone(uwsgi.cache_get('KEY1', 'items_lru')) uwsgi.cache_get('KEY3', 'items_lru') for i in range(4, 100): self.assertTrue(uwsgi.cache_set('KEY%d' % i, 'Y' * 20, 0, 'items_lru')) self.assertIsNone(uwsgi.cache_get('KEY%d' % (i-2), 'items_lru'))
def reset_debug_level(level): import uwsgi if level == '-1': uwsgi.cache_del("DEBUG") else: if uwsgi.cache_exists("DEBUG"): uwsgi.cache_update("DEBUG", level) else: uwsgi.cache_set("DEBUG", level) return redirect(url_for('tell_me_if_im_going_to_die', lat=39.9708657, lon=-75.1427425, meters=1000))
def increment_request_count(user_id): worker_id = str(uwsgi.worker_id()) if uwsgi.cache_get(worker_id): c = int(uwsgi.cache_get(worker_id)) c += 1 uwsgi.cache_update(worker_id, str(c)) else: uwsgi.cache_set(worker_id, '0') return f"user_id:{user_id}:workder_id:{worker_id}:request_number:{uwsgi.cache_get(worker_id).decode()}"
def __init__(self): ServiceBase.__init__(self) ProfileMixin.__init__(self) DispatcherMixin_CRR.__init__(self) if not uwsgi.cache_exists('Service2Counter'): uwsgi.cache_set('Service2Counter', '0') if not uwsgi.cache_exists('Service2Timer'): uwsgi.cache_set('Service2Timer', '0') print uwsgi.queue_size gevent.spawn(microtask, uwsgi.worker_id()) print 'after gevent.spawn'
def test_multi_delete(self): for i in range(0, 100): self.assertTrue(uwsgi.cache_set('key1', 'X' * 50 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_del('key1', 'items_4_10')) for i in range(0, 100): self.assertIsNone(uwsgi.cache_set('key1', 'X' * 51 , 0, 'items_4_10')) self.assertIsNone(uwsgi.cache_del('key1', 'items_4_10')) for i in range(0, 100): self.assertTrue(uwsgi.cache_set('key1', 'X' * 50 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_del('key1', 'items_4_10'))
def getAllTriggersAlarming(): triggerCached = cache_get('triggerTelao', cache_options['triggerGet']['name']) if triggerCached: return json.loads(triggerCached) elif cache_get('updatingCache', cache_options['updates']['name']) == 'True': while cache_get('updatingCache', cache_options['updates']['name']) == 'True': time.sleep(0.3) else: return json.loads( cache_get('triggerTelao', cache_options['updates']['name'])) else: if cache_exists('updatingCache', cache_options['updates']['name']): cache_update('updatingCache', 'True', cache_options['updates']['expiration_time'], cache_options['updates']['name']) else: cache_set('updatingCache', 'True', cache_options['updates']['expiration_time'], cache_options['updates']['name']) admin = Admin() zbx_admin_token = admin.auth() triggers = fowardZ.sendToZabbix(method='trigger.get', params={ 'selectHosts': ["name"], 'selectGroups': ['groups'], 'selectLastEvent': ['lastEvent', 'acknowledged'], 'expandComment': 1, 'expandDescription': 1, 'only_true': 1, 'output': 'extend' }, auth=zbx_admin_token) cache_set('triggerTelao', json.dumps(triggers), cache_options['triggerGet']['expiration_time'], cache_options['triggerGet']['name']) cache_update('updatingCache', 'False', cache_options['updates']['expiration_time'], cache_options['updates']['name']) return triggers
def himawari8(target): last_updatetime = bottle.request.query.get("updatetime") getLayers = bottle.request.query.get("getLayers") https_verify = (bottle.request.query.get("https_verify") or "true").lower() == "true" baseUrl = bottle.request.url[0:bottle.request.url.find("/hi8")] if not getLayers.startswith("http"): getLayers = "{}{}".format(baseUrl, getLayers) key = "himawari8.{}".format(target) result = None getcaps = None if uwsgi.cache_exists("himawari8"): if uwsgi.cache_exists(key): result = json.loads(uwsgi.cache_get(key)) else: getcaps = uwsgi.cache_get("himawari8").decode("utf-8") else: res = requests.get(getLayers, verify=https_verify) res.raise_for_status() getcaps = res.content uwsgi.cache_set("himawari8", getcaps, 60 * 10) # cache for 10 mins getcaps = getcaps.decode("utf-8") if not result: layernames = re.findall("\w+HI8\w+{}\.\w+".format(target), getcaps) layers = [] for layer in layernames: layers.append([ settings.PERTH_TIMEZONE.localize( datetime.datetime.strptime( re.findall("\w+_(\d+)_\w+", layer)[0], "%Y%m%d%H%M")), layer ]) layers = sorted(layers, key=lambda layer: layer[0]) for layer in layers: layer[0] = (layer[0]).strftime("%a %b %d %Y %H:%M:%S AWST") result = { "servers": [baseUrl + FIREWATCH_SERVICE], "layers": layers, "updatetime": layers[len(layers) - 1][0] } uwsgi.cache_set(key, json.dumps(result), 60 * 10) # cache for 10 mins if len(result["layers"]) == 0: return bottle.HTTPResponse(status=404) elif last_updatetime and last_updatetime == result["updatetime"]: bottle.response.status = 290 return "{}" else: return result
def get_picklist(listname): if uwsgi.cache_exists(listname): return pickle.loads(uwsgi.cache_get(listname)) else: # three lists have special GETs but eventually they will be a picklist if listname == "tumorType": # picklist_values['tumorType']={ "cache_date": time.time(), "values":cache_oncotree()} uwsgi.cache_set(listname, pickle.dumps(cache_oncotree()), 900) elif listname == "Tag": # picklist_values['Tag']={ "cache_date": time.time(), "values": cache_barcodes()} uwsgi.cache_set(listname, pickle.dumps(cache_barcodes()), 900) if uwsgi.cache_get(listname) == None: return cache_barcodes() elif listname == "Reads+Coverage": uwsgi.cache_set("Reads+Coverage", pickle.dumps(cache_reads_coverage()), 900) else: r = s.get( LIMS_API_ROOT + "/getPickListValues?list=%s" % listname, auth=(LIMS_USER, LIMS_PW), verify=False, ) log_lims(r) picklist = [] for value in json.loads(r.content.decode('utf-8')): picklist.append({"id": value, "value": value}) uwsgi.cache_set(listname, pickle.dumps(picklist), 900) return pickle.loads(uwsgi.cache_get(listname))
def set(self, key, value, timeout=None): ''' Adds a new key/value pair overwriting any existing value for the key. If timeout is given, sets the timeout on the key to be the given timeout, otherwise it does not set one Returns True if the key/value pair was set successfully, False otherwise ''' try: if timeout is None: return uwsgi.cache_set(key, value) else: return uwsgi.cache_set(key, value, timeout) except Exception as e: debug_error(e) return False
def register(source_hash, method_name, packed_data): logging.info("{}: {}".format(request.method, request.url)) data = rpc_data_unpack(request.get_data()) logging.debug(str(data)) fn_data = {method_name: data} if cache_man.cache_exists(source_hash): reg_dump = pickle.loads(cache_man.cache_get(source_hash)) reg_dump.update(fn_data) cache_man.cache_update(source_hash, pickle.dumps(reg_dump), settings.DEFAULT_CACHE_TTL) else: cache_man.cache_set(source_hash, pickle.dumps(fn_data), settings.DEFAULT_CACHE_TTL)
def objects(cls): if not uwsgi.cache_exists("raw_aws"): if hasattr(cls, "_objects"): del cls._objects uwsgi.cache_set("raw_aws", subprocess.check_output(["aws", "ec2", "describe-instances", "--no-paginate"]), 60*15) raw = json.loads(uwsgi.cache_get("raw_aws").decode("utf-8")) if hasattr(cls, "_objects"): return cls._objects objects = {} for data in raw["Reservations"]: for instance_data in data["Instances"]: instance = Instance(instance_data=instance_data) objects[instance.instance_id] = instance cls._objects = objects return objects # A dict
def cache_set(self, key, value, expire=None, cache_server=None): ''' Set a value in the cache. key : The cache key to write. write : The cache value to write. expire : Expiry time of the value, in seconds. cache_server : The UNIX/TCP socket where the cache portal2 is listening. Optional. ''' if expire != None: return uwsgi.cache_set(key, value, expire) elif cache_server != None: return uwsgi.cache_set(key, value, cache_server) elif expire != None and cache_server != None: return uwsgi.cache_set(key, value, expire, cache_server) else: return uwsgi.cache_set(key, value)
def test_big_update(self): self.assertTrue(uwsgi.cache_set('key1', 'X' * 40, 0, 'items_4_10')) self.assertTrue(uwsgi.cache_update('key1', 'X' * 10, 0, 'items_4_10')) self.assertTrue(uwsgi.cache_del('key1', 'items_4_10')) self.assertIsNone(uwsgi.cache_update('key1', 'X' * 51, 0, 'items_4_10')) self.assertTrue(uwsgi.cache_update('key1', 'X' * 50, 0, 'items_4_10'))
def loop(): while True: key = uwsgi.mule_get_msg() key = key.decode('utf-8') ids = key.split('_') uwsgi.cache_set(key, 'inprogress') try: result = bfs(ids[0], ids[1]) except: uwsgi.cache_update(key, 'fail') else: if result: uwsgi.cache_update(key, 'found') print(key) else: uwsgi.cache_update(key, 'notfound')
def himawari8(target): if uwsgi.cache_exists("himawari8"): getcaps = uwsgi.cache_get("himawari8") else: getcaps = requests.get(FIREWATCH_GETCAPS).content uwsgi.cache_set("himawari8", getcaps, 60*10) # cache for 10 mins getcaps = getcaps.decode("utf-8") layernames = re.findall("\w+HI8\w+{}\.\w+".format(target), getcaps) layers = [] for layer in layernames: layers.append([FIREWATCH_TZ.localize(datetime.strptime(re.findall("\w+_(\d+)_\w+", layer)[0], "%Y%m%d%H%M")).isoformat(), layer]) result = { "servers": [FIREWATCH_SERVICE], "layers": layers } return result
def bom(target): last_updatetime = bottle.request.query.get("updatetime") layeridpattern = bottle.request.query.get("layeridpattern") if layeridpattern: layeridpattern = "bom:{}".format(layeridpattern) else: layeridpattern = "bom:{}{{:0>3}}".format(target) current_timeline = None try: current_timeline = json.loads(uwsgi.cache_get(target)) except: current_timeline = None bottle.response.set_header("Content-Type", "application/json") bottle.response.status = 200 if current_timeline and datetime.datetime.now( ) - datetime.datetime.strptime( current_timeline["refreshtime"], "%a %b %d %Y %H:%M:%S") < datetime.timedelta(minutes=5): # data is refreshed within 5 minutes, use the result directly if current_timeline["updatetime"] == last_updatetime: # return 304 cause "No element found" error, so return a customized code to represent the same meaning as 304 bottle.response.status = 290 return "{}" else: return { "layers": current_timeline["layers"], "updatetime": current_timeline["updatetime"] } timeline = getTimelineFromWmsLayer(current_timeline, bomLayerIdFunc(layeridpattern)) if not timeline: raise "Missing some of http parameters 'basetimelayer', 'timelinesize', 'layertimespan'." if not current_timeline or id(timeline) != id(current_timeline): uwsgi.cache_set(target, json.dumps(timeline), 0) if timeline["updatetime"] == last_updatetime: bottle.response.status = 290 return "{}" else: return { "layers": timeline["layers"], "updatetime": timeline["updatetime"] }
def callback(): startup.getUserToken(request.args.get('code')) if not uwsgi.cache_exists('isRunning'): app.logger.info( "Creating new thread for refreshing spotify token and user stats.") uwsgi.cache_set('isRunning', 'True') uwsgi.cache_set('stop_threads', 'False') sp_t = spotify_thread(2500, "Thread-spotify") sp_t.start() try: if uwsgi.cache_get('isRunning').decode( 'utf-8') == 'True' and uwsgi.cache_get('stop_threads').decode( 'utf-8') == 'True': app.logger.info("Relancement de l'application spotify") uwsgi.cache_update('stop_threads', 'False') except AttributeError: app.logger.error( f"La variable isRunning ou stop_threads n'est pas initialisée, valeurs : ir:{uwsgi.cache_get('isRunning')} et st:{uwsgi.cache_get('stop_threads')}" ) list_time_range = ['short_term', 'medium_term', 'long_term'] list_type = ['artists', 'tracks'] dict_index = { 'short_term_artists': 1, 'medium_term_artists': 2, 'long_term_artists': 3, 'short_term_tracks': 4, 'medium_term_tracks': 5, 'long_term_tracks': 6 } for type in list_type: for time_range in list_time_range: set_analytics_data( dict_index[f"{time_range}_{type}"], json.dumps( json.loads( get_users_top( startup.getAccessToken()[1], type, time_range, ))), time_range, type) app.logger.info( f"All the threads are listed below : {[thread.name for thread in threading.enumerate()]}" ) return redirect(url_for('project_spotify.spotify'))
def objects(cls): if not uwsgi.cache_exists("raw_aws"): if hasattr(cls, "_objects"): del cls._objects uwsgi.cache_set( "raw_aws", subprocess.check_output( ["aws", "ec2", "describe-instances", "--no-paginate"]), 60 * 15) raw = json.loads(uwsgi.cache_get("raw_aws").decode("utf-8")) if hasattr(cls, "_objects"): return cls._objects objects = {} for data in raw["Reservations"]: for instance_data in data["Instances"]: instance = Instance(instance_data=instance_data) objects[instance.instance_id] = instance cls._objects = objects return objects
def methods(): gbix_methods_cached = cache_get('/api/methods', cache_options['default']['name']) if not gbix_methods_cached: try: arq = os.path.join(os.path.split(os.path.split(os.path.realpath(__file__))[0])[0], 'doc/api_data.json') f = open(arq) except IOError: return 'Methods file not found.' methods_json = str() pattern = re.compile('\s+"filename') for line in f.readlines(): if re.match(pattern, line) is None: methods_json += line.strip().replace('<p>', '').replace('</p>', '').replace(' "', '"').replace('\\"', '') f.close() cache_set('/api/methods', methods_json, 0, cache_options['default']['name']) return methods_json else: return gbix_methods_cached
def himawari8(target): last_updatetime = bottle.request.query.get("updatetime") baseUrl = bottle.request.url[0:bottle.request.url.find("/hi8")] key = "himawari8.{}".format(target) result = None getcaps = None if uwsgi.cache_exists("himawari8"): if uwsgi.cache_exists(key): result = json.loads(uwsgi.cache_get(key)) else: getcaps = uwsgi.cache_get("himawari8") else: res = requests.get("{}{}".format(baseUrl,FIREWATCH_GETCAPS),verify=FIREWATCH_HTTPS_VERIFY) res.raise_for_status() getcaps = res.content getcaps = getcaps.decode("utf-8") uwsgi.cache_set("himawari8", getcaps, 60*10) # cache for 10 mins if not result: layernames = re.findall("\w+HI8\w+{}\.\w+".format(target), getcaps) layers = [] for layer in layernames: layers.append([settings.PERTH_TIMEZONE.localize(datetime.datetime.strptime(re.findall("\w+_(\d+)_\w+", layer)[0], "%Y%m%d%H%M")), layer]) layers = sorted(layers,key=lambda layer:layer[0]) for layer in layers: layer[0] = (layer[0]).strftime("%a %b %d %Y %H:%M:%S AWST") result = { "servers": [baseUrl + FIREWATCH_SERVICE], "layers": layers, "updatetime":layers[len(layers) - 1][0] } uwsgi.cache_set(key, json.dumps(result), 60*10) # cache for 10 mins if len(result["layers"]) == 0: return bottle.HTTPResponse(status=404) elif last_updatetime and last_updatetime == result["updatetime"]: bottle.response.status = 290 return "{}" else: return result
def healthcheck(): fe_status_cached = cache_get('gbix_healthcheck', cache_options['default']['name']) if fe_status_cached: return fe_status_cached else: try: fe_status = fowardZ.sendRequest('{"jsonrpc": "2.0", \ "method": "apiinfo.version", \ "params": [], \ "id": 1 \ }') except: status_string = 'FAILED ' + socket.gethostname( ) + ' Sem conexao com o FE do Zabbix' cache_set('gbix_healthcheck', status_string, 3, cache_options['default']['name']) return status_string if 'result' in fe_status: status_string = 'WORKING ' + socket.gethostname() cache_set('gbix_healthcheck', status_string, 5, cache_options['default']['name']) return status_string else: status_string = 'FAILED ' + socket.gethostname( ) + 'Sem conexao com o FE do Zabbix' cache_set('gbix_healthcheck', status_string, 3, cache_options['default']['name']) return status_string
def healthcheck(): fe_status_cached = cache_get('gbix_healthcheck', cache_options['default']['name']) if fe_status_cached: return fe_status_cached else: try: fe_status = fowardZ.sendRequest('{"jsonrpc": "2.0", \ "method": "apiinfo.version", \ "params": [], \ "id": 1 \ }') except: status_string = 'FAILED ' + socket.gethostname() + ' Sem conexao com o FE do Zabbix' cache_set('gbix_healthcheck', status_string, 3, cache_options['default']['name']) return status_string if 'result' in fe_status: status_string = 'WORKING ' + socket.gethostname() cache_set('gbix_healthcheck', status_string, 5, cache_options['default']['name']) return status_string else: status_string = 'FAILED ' + socket.gethostname() + 'Sem conexao com o FE do Zabbix' cache_set('gbix_healthcheck', status_string, 3, cache_options['default']['name']) return status_string
def get_url(url): """Download a file from url to cache_dir.""" # set a lock to prevent multiple simultaneous downloads of the same file mypid = os.getpid() uwsgi.lock() otherpid = uwsgi.cache_get(url) if otherpid: uwsgi.unlock() while otherpid: log('D: [%d] waiting for pid %s to download %s' % (mypid, otherpid, url)) time.sleep(1) otherpid = uwsgi.cache_get(url) return 200 else: uwsgi.cache_set(url, str(mypid)) uwsgi.unlock() dest = localfile(url) log('D: [%d] downloading %s to %s' % (mypid, url, dest)) curl = pycurl.Curl() curl.setopt(curl.URL, url) curl.setopt(curl.FOLLOWLOCATION, True) path = '/'.join(dest.split('/')[:-1]) if not os.path.exists(path): # parallel download of rpms in subdir will create it right now try: os.makedirs(path) except OSError as e: # this catches duplicate creation (so just W not E) # TODO: need to bypass the open() on real errors # like permissions log('W: [%d] OS error(%d): %s' % (mypid, e.errno, e.strerror)) with open(dest, 'wb') as fil: curl.setopt(curl.WRITEFUNCTION, fil.write) curl.perform() uwsgi.cache_del(url) return curl.getinfo(curl.HTTP_CODE)
def create_tests(): """Execute tests upon successful POST.""" payload = request.get_json() if payload is None: return make_response( jsonify({"error": "Valid request payload not found."}), 400) test_count = 0 for tests in payload.values(): test_count += len(tests) if test_count > CONFIG["max_test_count"]: return make_response( jsonify({ "error": f"Provided number of tests is too high. Max: {CONFIG['max_test_count']}" }), 400, ) # Generate the client's receipt and pass the test payload to a background thread to be executed. receipt = token_hex(16) uwsgi.cache_set(receipt, "{}", 600, "receipts") execute_tests(receipt, payload, CONFIG["max_process_count"]) return jsonify({"receipt": receipt})
def test_big_delete(self): self.assertTrue(uwsgi.cache_set('key1', 'X' * 50 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_del('key1', 'items_4_10')) self.assertTrue(uwsgi.cache_set('key1', 'HELLOHELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key2', 'HELLOHELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key3', 'HELLOHELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key4', 'HELLOHELLO', 0, 'items_4_10')) self.assertIsNone(uwsgi.cache_set('key5', 'HELLOHELLO', 0, 'items_4_10'))
def get_url(url): """Download a file from url to cache_dir.""" # set a lock to prevent multiple simultaneous downloads of the same # file mypid = os.getpid() uwsgi.lock() otherpid = uwsgi.cache_get(url) if otherpid: uwsgi.unlock() while otherpid: log('D: pid %d waiting for pid %s to download %s' % (mypid, otherpid, url)) time.sleep(1) otherpid = uwsgi.cache_get(url) return 200 else: uwsgi.cache_set(url, str(mypid)) uwsgi.unlock() dest = localfile(url) log('D: pid %d downloading %s' % (mypid, url)) curl = pycurl.Curl() curl.setopt(curl.URL, url) path = '/'.join(dest.split('/')[:-1]) if not os.path.exists(path): # parallel download of rpms in subdir will create it right now try: os.makedirs(path) except OSError as e: # this catches duplicate creation (so just W not E) # TODO: need to bypass the open() on real errors # like permissions log('W: OS error(%d): %s' % (e.errno, e.strerror)) with open(dest, 'wb') as fil: curl.setopt(curl.WRITEFUNCTION, fil.write) curl.perform() uwsgi.cache_del(url) return curl.getinfo(curl.HTTP_CODE)
def bom(target): last_updatetime = bottle.request.query.get("updatetime") layeridpattern = bottle.request.query.get("layeridpattern") if layeridpattern: layeridpattern = "bom:{}".format(layeridpattern) else: layeridpattern = "bom:{}{{:0>3}}".format(target) current_timeline = None try: current_timeline = json.loads(uwsgi.cache_get(target)) except: current_timeline = None bottle.response.set_header("Content-Type", "application/json") bottle.response.status = 200 if current_timeline and datetime.datetime.now() - datetime.datetime.strptime(current_timeline["refreshtime"], "%a %b %d %Y %H:%M:%S") < datetime.timedelta(minutes=5): # data is refreshed within 5 minutes, use the result directly if current_timeline["updatetime"] == last_updatetime: # return 304 cause "No element found" error, so return a customized code to represent the same meaning as 304 bottle.response.status = 290 return "{}" else: return {"layers": current_timeline["layers"], "updatetime": current_timeline["updatetime"]} timeline = getTimelineFromWmsLayer(current_timeline, bomLayerIdFunc(layeridpattern)) if not timeline: raise "Missing some of http parameters 'basetimelayer', 'timelinesize', 'layertimespan'." if not current_timeline or id(timeline) != id(current_timeline): uwsgi.cache_set(target, json.dumps(timeline), 0) if timeline["updatetime"] == last_updatetime: bottle.response.status = 290 return "{}" else: return {"layers": timeline["layers"], "updatetime": timeline["updatetime"]}
def set(self, cache, key, value): uwsgi.cache_set(key, value, 3600, cache)
def get_layermetadata(layerids, kmiserver=settings.KMI_SERVER, results={}): multiple_layers = True if isinstance(layerids, basestring): layerids = [layerids] multiple_layers = False #group layers against layer workspace layers = {} for layerid in layerids: layerid = layerid.strip() #check whether it is cached or not key = layermetadatakey(layerid) if uwsgi.cache_exists(key): try: metadata = uwsgi.cache_get(key) if metadata: if layerid in results: results[layerid].update(json.loads(metadata)) else: results[layerid] = json.loads(metadata) #print("Retrieve the metadata from cache for layer ({})".format(layerid)) continue except: pass layer = layerid.split(":") if len(layer) == 1: #no workspace layer_ws = "" layer = layer[0] else: layer_ws = layer[0] layer = layer[1] if layer_ws not in layers: layers[layer_ws] = [layer] else: layers[layer_ws].append(layer) if layers: session_cookie = settings.get_session_cookie() kmiserver = get_kmiserver(kmiserver) #find the layer's metadata url = None for layer_ws, layers in layers.iteritems(): if layer_ws: url = "{}/{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format( kmiserver, layer_ws) else: url = "{}/wms?service=wms&version=1.1.1&request=GetCapabilities".format( kmiserver) res = requests.get(url, verify=False, cookies=session_cookie) res.raise_for_status() tree = ET.fromstring(res.content) capability = tree.find('Capability') if not len(capability): raise Exception("getCapability failed") kmi_layers = capability.findall("Layer") while kmi_layers: kmi_layer = kmi_layers.pop() name = get_child_value(kmi_layer, "Name") if name: try: index = layers.index(name) except: index = -1 if index >= 0: #this layer's metadata is requsted by the user if layer_ws: layerid = "{}:{}".format(layer_ws, name) else: layerid = name if layerid in results: result = results[layerid] else: result = {"id": layerid} results[layerid] = result del layers[index] result["title"] = get_child_value(kmi_layer, "Title") result["abstract"] = get_child_value( kmi_layer, "Abstract") result["srs"] = get_child_value(kmi_layer, "SRS") bbox = kmi_layer.find("LatLonBoundingBox") if bbox is not None: result["latlonBoundingBox"] = [ float(bbox.attrib["miny"]), float(bbox.attrib["minx"]), float(bbox.attrib["maxy"]), float(bbox.attrib["maxx"]) ] else: result["latlonBoundingBox"] = None for bbox in kmi_layer.findall("BoundingBox"): result["latlonBoundingBox_{}".format( bbox.attrib["SRS"].upper())] = [ float(bbox.attrib["miny"]), float(bbox.attrib["minx"]), float(bbox.attrib["maxy"]), float(bbox.attrib["maxx"]) ] #cache it for 6 hours key = layermetadatakey(result["id"]) try: if uwsgi.cache_exists(key): uwsgi.cache_update(key, json.dumps(result), 6 * 3600) else: uwsgi.cache_set(key, json.dumps(result), 6 * 3600) except: pass #print("Retrieve the metadata from kmi for layer ({})".format(result["id"])) if len(layers): continue else: #already find metadata for all required layers break sub_layers = kmi_layer.findall("Layer") if sub_layers: kmi_layers += sub_layers if len(layers) == 1: if layer_ws: raise Exception("The layer({}:{}) Not Found".format( layer_ws, layers[0])) else: raise Exception("The layer({}) Not Found".format( layers[0])) elif len(layers) > 1: if layer_ws: raise Exception("The layers({}) Not Found".format(",".join( ["{}:{}".format(layer_ws, l) for l in layers]))) else: raise Exception("The layers({}) Not Found".format( ",".join(layers))) if multiple_layers: return results else: return results[layerids[0]]
def _update_container_ip(client, container_name, cache_key): info = client.inspect_container(container_name) ip_address = info['NetworkSettings']['IPAddress'] uwsgi.cache_set(cache_key, ip_address) return ip_address
def gen_rand_n(max_n): return random.randint(8, max_n) def gen_rand_s(size): return ''.join([random.choice(string.letters) for i in range(size)]) print 'filling cache...' for i in range(0, 1000): kl = gen_rand_n(200) key = gen_rand_s(kl) vl = gen_rand_n(10000) val = gen_rand_s(vl) items[key] = val uwsgi.cache_set(key, val) print 'checking cache...' count = 0 for key in items.keys(): val = uwsgi.cache_get(key) count += 1 if val != items[key]: print len(val), val print len(items[key]), items[key] raise Exception('CACHE TEST FAILED AFTER %d ITERATIONS !!!' % count) print "TEST PASSED"
def get_layerdefinition(layerids, kmiserver=settings.KMI_SERVER, results={}): kmiserver = get_kmiserver(kmiserver) multiple_layers = True if isinstance(layerids, basestring): layerids = [layerids] multiple_layers = False #group layers against layer workspace layers = {} for layerid in layerids: layerid = layerid.strip() #check whether it is cached or not key = layerdefinitionkey(layerid) if uwsgi.cache_exists(key): try: definitiondata = uwsgi.cache_get(key) if definitiondata: if layerid in results: results[layerid].update(json.loads(definitiondata)) else: results[layerid] = json.loads(definitiondata) continue except: pass layer = layerid.split(":") if len(layer) == 1: #no workspace layer_ws = "" layer = layer[0] else: layer_ws = layer[0] layer = layer[1] if layer_ws not in layers: layers[layer_ws] = [layerid] else: layers[layer_ws].append(layerid) if layers: kmiserver = get_kmiserver(kmiserver) session_cookie = settings.get_session_cookie() url = None for layer_ws, layers in layers.iteritems(): if layer_ws: url = "{}/{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format( kmiserver, layer_ws, ",".join(layers)) else: url = "{}/wfs?request=DescribeFeatureType&version=2.0.0&service=WFS&outputFormat=application%2Fjson&typeName=".format( kmiserver, ",".join(layers)) res = requests.get(url, verify=False, cookies=session_cookie) res.raise_for_status() layersdata = res.json() for layer in layersdata.get("featureTypes") or []: if layer_ws: layerid = "{}:{}".format(layer_ws, layer["typeName"]) else: layerid = layer["typeName"] try: index = layers.index(layerid) except: index = -1 if index >= 0: #this layer's metadata is requsted by the user if layerid in results: result = results[layerid] else: result = {"id": layerid} results[layerid] = result result["properties"] = layer["properties"] result["geometry_property"] = None result["geometry_properties"] = [] result["geometry_type"] = None result["geometry_property_msg"] = None del layers[index] #find spatial columns for prop in layer["properties"]: if prop["type"].startswith("gml:"): #spatial column result["geometry_properties"].append(prop) if len(result["geometry_properties"]) == 1: result["geometry_property"] = result[ "geometry_properties"][0] result["geometry_type"] = result[ "geometry_properties"][0]["localType"].lower() elif len(result["geometry_properties"]) > 1: #have more than one geometry properties, try to find the right one if layer_ws: url = "{}/{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format( kmiserver, layer_ws, layerid) else: url = "{}/ows?service=WFS&version=2.0.0&request=GetFeature&typeName={}&count=1&outputFormat=application%2Fjson".format( kmiserver, layerid) res = requests.get(url, verify=False, cookies=session_cookie) res.raise_for_status() featuresdata = res.json() if len(featuresdata["features"]) > 0: feat = featuresdata["features"][0] for prop in result["geometry_properties"]: if prop["name"] == feat["geometry_name"]: result["geometry_property"] = prop result["geometry_type"] = prop[ "localType"].lower() break if not result["geometry_property"]: result[ "geometry_property_msg"] = "Layer '{}' has more than one geometry columns, can't identity which column is used as the geometry column.".format( layerid) else: result[ "geometry_property_msg"] = "Layer '{}' is not a spatial layer".format( layerid) if result["geometry_property"]: #found the geometry property, remove it from properties index = len(result["properties"]) - 1 while index >= 0: if result["properties"][index] == result[ "geometry_property"]: #this is the geometry property,remove it from properties del result["properties"][index] break index -= 1 #cache it for 1 day key = layerdefinitionkey(layerid) try: if uwsgi.cache_exists(key): uwsgi.cache_update(key, json.dumps(result), 24 * 3600) else: uwsgi.cache_set(key, json.dumps(result), 24 * 3600) except: pass if len(layers) == 1: if layer_ws: raise Exception("The layer({}:{}) Not Found".format( layer_ws, layers[0])) else: raise Exception("The layer({}) Not Found".format(layers[0])) elif len(layers) > 1: if layer_ws: raise Exception("The layers({}) Not Found".format(",".join( ["{}:{}".format(layer_ws, l) for l in layers]))) else: raise Exception("The layers({}) Not Found".format( ",".join(layers))) if multiple_layers: return results else: return results[layerids[0]]
import uwsgi if uwsgi.loop == 'gevent': import gevent print uwsgi.version print uwsgi.workers() try: uwsgi.cache_set('foo', "Hello World from cache") except: pass def application(env, start_response): if uwsgi.loop == 'gevent': gevent.sleep() start_response('200 OK', [('Content-Type', 'text/html')]) yield "foobar<br/>" if uwsgi.loop == 'gevent': gevent.sleep(10) yield str(env['wsgi.input'].fileno()) yield "<h1>Hello World</h1>" try: yield uwsgi.cache_get('foo') except: pass
def test_non_bitmap(self): self.assertTrue(uwsgi.cache_set('KEY', 'X' * 20, 0, 'items_non_bitmap')) self.assertTrue(uwsgi.cache_del('KEY', 'items_non_bitmap')) self.assertIsNone(uwsgi.cache_set('KEY', 'X' * 21, 0, 'items_non_bitmap')) self.assertTrue(uwsgi.cache_set('KEY', 'X' * 20, 0, 'items_non_bitmap'))
def test_big_key(self): self.assertTrue(uwsgi.cache_set('K' * 2048, 'X' * 50 , 0, 'items_4_10')) self.assertIsNone(uwsgi.cache_set('K' * 2049, 'X' * 50 , 0, 'items_4_10'))
def _cache_set_token(token, tenant_id): uwsgi.cache_set(token, tenant_id, _CONFIG.cache.ttl, _CONFIG.cache.cache_name)
def _get_profile(app): #get app profile profile = None appPath = os.path.join(DIST_PATH,"{}.js".format(app)) if not os.path.exists(appPath): appPath = os.path.join(DIST_PATH,"sss.js") key = "{}_profile".format(app) profileChanged = False if uwsgi.cache_exists(key): profile = uwsgi.cache_get(key) if profile: profile = json.loads(profile) if repr(os.path.getmtime(appPath)) != profile["mtime"] or os.path.getsize(appPath) != profile["size"]: profileChanged = True profile = None if not profile: file_data = None with open(appPath,"rb") as f: file_data = f.read() m = profile_re.search(file_data) profile = m.group("profile") if m else "{}" profile = { 'mtime':repr(os.path.getmtime(appPath)), 'size':os.path.getsize(appPath), 'profile':demjson.decode(profile) } m = hashlib.md5() m.update(file_data) profile['profile']['build']['md5'] = base64.urlsafe_b64encode(m.digest()).rstrip("=") file_data = None if profileChanged: uwsgi.cache_update(key, json.dumps(profile)) else: uwsgi.cache_set(key, json.dumps(profile)) profile["profile"]["dependents"] = {} #get vendor md5 vendorPath = os.path.join(DIST_PATH,"vendor.js") if not os.path.exists(vendorPath): raise Exception("Vendor library not found") key = "{}_profile".format("vendor") profileChanged = False vendorProfile = None if uwsgi.cache_exists(key): vendorProfile = uwsgi.cache_get(key) if vendorProfile: vendorProfile = json.loads(vendorProfile) if repr(os.path.getmtime(vendorPath)) != vendorProfile["mtime"] or os.path.getsize(vendorPath) != vendorProfile["size"]: profileChanged = True vendorProfile = None if not vendorProfile: m = hashlib.md5() with open(vendorPath,"rb") as f: m.update(f.read()) vendorProfile = { 'mtime':repr(os.path.getmtime(vendorPath)), 'size':os.path.getsize(vendorPath), 'vendorMD5':base64.urlsafe_b64encode(m.digest()).rstrip("=") } if profileChanged: uwsgi.cache_update(key, json.dumps(vendorProfile)) else: uwsgi.cache_set(key, json.dumps(vendorProfile)) profile["profile"]["dependents"]["vendorMD5"] = vendorProfile["vendorMD5"] #get env profile envPath = os.path.join(BASE_DIST_PATH,'release','static','js',"{}-{}.env.js".format(app,ENV_TYPE)) if not os.path.exists(envPath): raise Exception("'{}-{}.env.js' is missing.".format(app,ENV_TYPE)) else: key = "{}_{}_profile".format("env",ENV_TYPE) profileChanged = False envProfile = None if uwsgi.cache_exists(key): envProfile = uwsgi.cache_get(key) if envProfile: envProfile = json.loads(envProfile) if repr(os.path.getmtime(envPath)) != envProfile["mtime"] or os.path.getsize(envPath) != envProfile["size"]: profileChanged = True envProfile = None if not envProfile: m = hashlib.md5() with open(envPath,"rb") as f: m.update(f.read()) envProfile = { 'mtime':repr(os.path.getmtime(envPath)), 'size':os.path.getsize(envPath), 'envMD5':base64.urlsafe_b64encode(m.digest()).rstrip("=") } if profileChanged: uwsgi.cache_update(key, json.dumps(envProfile)) else: uwsgi.cache_set(key, json.dumps(envProfile)) profile["profile"]["dependents"]["envMD5"] = envProfile["envMD5"] profile["profile"]["envType"] = ENV_TYPE #get style profile stylePath = os.path.join(BASE_DIST_PATH,'release','static','css',"style.css") if not os.path.exists(stylePath): raise Exception("'style.css' is missing.") else: key = "style_profile" profileChanged = False styleProfile = None if uwsgi.cache_exists(key): styleProfile = uwsgi.cache_get(key) if styleProfile: styleProfile = json.loads(styleProfile) if repr(os.path.getmtime(stylePath)) != styleProfile["mtime"] or os.path.getsize(stylePath) != styleProfile["size"]: profileChanged = True styleProfile = None if not styleProfile: m = hashlib.md5() with open(stylePath,"rb") as f: m.update(f.read()) styleProfile = { 'mtime':repr(os.path.getmtime(stylePath)), 'size':os.path.getsize(stylePath), 'styleMD5':base64.urlsafe_b64encode(m.digest()).rstrip("=") } if profileChanged: uwsgi.cache_update(key, json.dumps(styleProfile)) else: uwsgi.cache_set(key, json.dumps(styleProfile)) profile["profile"]["dependents"]["styleMD5"] = styleProfile["styleMD5"] return profile["profile"]
import uwsgi import werkzeug.testapp uwsgi.cache_set("/cache/get", "HTTP 1.1 200 OK\r\nContent-Type: text/html\r\n\r\n<h1>I am the uWSGI cache</h1>") def app001(env, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) return "PATH_INFO=%s" % env['PATH_INFO'] def app002(env, start_response): start_response('200 OK', [('Content-Type', 'text/html')]) return "requests: %d" % uwsgi.total_requests() uwsgi.applications = { '': werkzeug.testapp.test_app, '/app001': app001, '/app002': app002 }
def test_too_much_items(self): self.assertTrue(uwsgi.cache_set('key1', 'HELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key2', 'HELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key3', 'HELLO', 0, 'items_4_10')) self.assertTrue(uwsgi.cache_set('key4', 'HELLO', 0, 'items_4_10')) self.assertIsNone(uwsgi.cache_set('key5', 'HELLO', 0, 'items_4_10'))
def test_set(self): self.assertTrue(uwsgi.cache_set('key1', 'HELLO', 0, 'items_17')) self.assertIsNone(uwsgi.cache_set('key1', 'HELLO', 0, 'items_17')) self.assertTrue(uwsgi.cache_del('key1', 'items_17')) self.assertTrue(uwsgi.cache_set('key1', 'HELLO', 0, 'items_17')) self.assertIsNone(uwsgi.cache_set('key1', 'HELLO', 0, 'items_17'))
def test_big_update(self): self.assertTrue(uwsgi.cache_set('key1', 'X' * 40 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_update('key1', 'X' * 10 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_del('key1', 'items_4_10')) self.assertIsNone(uwsgi.cache_update('key1', 'X' * 51 , 0, 'items_4_10')) self.assertTrue(uwsgi.cache_update('key1', 'X' * 50 , 0, 'items_4_10'))