def sync_full(cls, **kwargs): """ Sync Cisco DNA Center as RQ job """ data = {} # Get RQ queue queue = get_queue("default") # Get RQ Job ID and display results if "id" in kwargs: data = queue.fetch_job(str(kwargs["id"])) if data is None: return None return data.result # Check if ongoing RQ Job is ongoing try: job = cache.get("ciscodnacnetbox_bg") except CacheMiss: # If not, start full sync task job = full_sync.delay(**kwargs) cache.set("ciscodnacnetbox_bg", job.id, timeout=600) # Get Job Status j = queue.fetch_job(cache.get("ciscodnacnetbox_bg")) if "finished" == j.get_status(): # Start again, if cache expired job = full_sync.delay(**kwargs) cache.set("ciscodnacnetbox_bg", job.id, timeout=600) data["id"] = str(j.id) data["task"] = str(j.func_name) return data
def get_releases(pre_releases=False): url = settings.RELEASE_CHECK_URL releases = [] try: logger.debug(f"fetching new releases from {url}") response = requests.get( url, headers={"Accept": "application/vnd.github.v3+json"}) response.raise_for_status() total_releases = len(response.json()) for release in response.json(): if "tag_name" not in release: continue if not pre_releases and (release.get("devrelease") or release.get("prerelease")): continue releases.append( (version.parse(release["tag_name"]), release.get("html_url"))) logger.debug( f"found {total_releases} releases; {len(releases)} usable") except requests.exceptions.RequestException: logger.exception(f"error while fetching {url}") return [] # Cache the most recent release cache.set("latest_release", max(releases), settings.RELEASE_CHECK_TIMEOUT) return releases
def activate_super_powers(request): cache.set('%s:%s' % (request.user.pk, settings.SUPER_POWERS_KEY), data={ 'started': timezone.now(), 'status': PENDING_VALIDATION }, timeout=settings.SUPER_POWERS_DURATION) if get_super_powers_details(request) is None: return False else: return True
def save_prediction_server_status_to_cache_job(cache_key, cvat_project_id, timeout=60): cvat_project = Project.objects.get(pk=cvat_project_id) api = TrainingServerAPI(host=cvat_project.training_project.host, username=cvat_project.training_project.username, password=cvat_project.training_project.password) status = api.get_project_status( project_id=cvat_project.training_project.training_id) resp = {**status, 'status': 'done'} cache.set(cache_key=cache_key, data=resp, timeout=timeout)
def profile_to_location(handle): timeout = 60 * 20 key_salt = '1' key = f'profile_to_location{handle}_{key_salt}' try: results = cache.get(key) except CacheMiss: results = None if not results: results = profile_to_location_helper(handle) cache.set(key, results, timeout) return results
def build_stat_results(keyword=None): timeout = 60 * 60 * 24 key_salt = '3' key = f'build_stat_results_{keyword}_{key_salt}' try: results = cache.get(key) except CacheMiss: results = None if results and not settings.DEBUG: return results results = build_stat_results_helper(keyword) cache.set(key, results, timeout) return results
def get_history_cached(breakdown, i): timeout = 60 * 60 * 3 key_salt = '0' key = f'get_history_cached_{breakdown}_{i}_{key_salt}' try: results = cache.get(key) except CacheMiss: results = None if results: return results results = gas_history(breakdown, i) cache.set(key, results, timeout) return results
def token(self) -> str: def get_token(host: str, username: str, password: str) -> dict: url = f'{host}/v2/authentication' data = { 'username': (None, username), 'password': (None, password), } r = requests.post(url=url, files=data, verify=False) # nosec return r.json() try: token = cache.get(self.token_key) except CacheMiss: response = get_token(self.host, self.username, self.password) token = response.get('secure_token', '') expires_in = response.get('expires_in', 3600) cache.set(cache_key=self.token_key, data=token, timeout=expires_in) return token
def predict_status(self, request): project_id = self.request.query_params.get('project') if not project_id: return Response(data='query param "project" empty or not provided', status=status.HTTP_400_BAD_REQUEST) project = Project.objects.get(pk=project_id) if not project.training_project: Response({'status': 'done'}) cache_key = f'predict_status_{project_id}' try: resp = cache.get(cache_key) except CacheMiss: save_prediction_server_status_to_cache_job.delay( cache_key, cvat_project_id=project_id) resp = { 'status': 'queued', } cache.set(cache_key=cache_key, data=resp, timeout=60) return Response(resp)
def activate_super_powers(request): if get_super_powers_details(request) is None: cache.set( '%s:%s' % (request.user.pk, settings.SUPER_POWERS_KEY), data={ 'started': timezone.now(), 'status': PENDING_VALIDATION}, timeout=settings.SUPER_POWERS_DURATION) return False else: cache.set( '%s:%s' % (request.user.pk, settings.SUPER_POWERS_KEY), data={ 'started': timezone.now(), 'status': VALIDATED}, timeout=settings.SUPER_POWERS_DURATION) return True
def predict_image(self, request): frame = self.request.query_params.get('frame') task_id = self.request.query_params.get('task') if not task_id: return Response(data='query param "task" empty or not provided', status=status.HTTP_400_BAD_REQUEST) if not frame: return Response(data='query param "frame" empty or not provided', status=status.HTTP_400_BAD_REQUEST) cache_key = f'predict_image_{task_id}_{frame}' try: resp = cache.get(cache_key) except CacheMiss: save_frame_prediction_to_cache_job.delay(cache_key, task_id=task_id, frame=frame) resp = { 'status': 'queued', } cache.set(cache_key=cache_key, data=resp, timeout=60) return Response(resp)
def save_frame_prediction_to_cache_job(cache_key: str, task_id: int, frame: int, timeout: int = 60): task = Task.objects.get(pk=task_id) training_project_image = TrainingProjectImage.objects.filter( idx=frame, task=task).first() if not training_project_image: cache.set(cache_key=cache_key, data={ 'annotation': [], 'status': 'done' }, timeout=timeout) return cvat_labels = Label.objects.filter(project__id=task.project_id).all() training_project = Project.objects.get(pk=task.project_id).training_project api = TrainingServerAPI(host=training_project.host, username=training_project.username, password=training_project.password) image = Image.objects.get(frame=frame, data=task.data) labels_mapping = { TrainingProjectLabel.objects.get( cvat_label=cvat_label).training_label_id: cvat_label.id for cvat_label in cvat_labels } annotation = api.get_annotation( project_id=training_project.training_id, image_id=training_project_image.training_image_id, width=image.width, height=image.height, labels_mapping=labels_mapping, frame=frame) resp = {'annotation': annotation, 'status': 'done'} cache.set(cache_key=cache_key, data=resp, timeout=timeout)
def refresh_virtual_machines(vcenter: ClusterVCenter, force=False): config = settings.PLUGINS_CONFIG['netbox_vcenter'] vcenter_cache_key = get_cache_key(vcenter) # Check whether this server has failed recently and shouldn't be retried yet try: cached_data = cache.get(vcenter_cache_key) if not force and cached_data == 'FAILED': logger.info( "Skipping vCenter update; server {} failed recently".format( vcenter.server)) return if not force: logger.info( "Skipping vCenter update; server {} already in cache".format( vcenter.server)) return cached_data except CacheMiss: pass service_instance = None try: logger.debug("Fetching VMs from {}".format(vcenter.server)) # Connect to the vCenter server if vcenter.validate_certificate: service_instance = connect.Connect(vcenter.server, user=vcenter.username, pwd=vcenter.password) else: service_instance = connect.ConnectNoSSL(vcenter.server, user=vcenter.username, pwd=vcenter.password) content = service_instance.RetrieveContent() vms = get_objects_of_type(content, vim.VirtualMachine) all_stats = {'timestamp': time.time(), 'vms': {}} dvs_cache = {} portgroup_cache = {} for vm in vms: vm_stats = { 'power': None, 'vcpus': None, 'memory': None, 'disk': None, 'nics': [], } try: if vm.runtime.powerState: vm_stats[ 'powered_on'] = vm.runtime.powerState == 'poweredOn' if vm.config.hardware.numCPU: vm_stats['vcpus'] = vm.config.hardware.numCPU if vm.config.hardware.memoryMB: vm_stats['memory'] = vm.config.hardware.memoryMB disk_devices = [ device for device in vm.config.hardware.device if isinstance(device, vim.vm.device.VirtualDisk) ] if disk_devices: # Sum and convert from KB to GB total_capacity = 0 for device in disk_devices: total_capacity += device.capacityInKB vm_stats['disk'] = round(total_capacity / 1048576) for dev in vm.config.hardware.device: if isinstance(dev, vim.vm.device.VirtualEthernetCard): vlan = get_nic_vlan(content, dvs_cache, portgroup_cache, vm, dev) vm_stats['nics'].append({ 'label': dev.deviceInfo.label, 'mac_address': dev.macAddress, 'vlan': vlan, }) except Exception: logger.exception( "Error while fetching virtual machine {} from {}".format( vm.name, vcenter.server)) continue # Collect all stats for returning all_stats['vms'][vm.name] = vm_stats # Cache a list of all VMs cache.set(vcenter_cache_key, all_stats, config['CACHE_TIMEOUT']) return all_stats except Exception: # Set a cookie in the cache so we don't keep retrying logger.exception("Error while fetching virtual machines from {}. " "Disabling checks for 5 minutes.".format( vcenter.server)) cache.set(vcenter_cache_key, 'FAILED', config['CACHE_FAILURE_TIMEOUT']) finally: if service_instance: connect.Disconnect(service_instance)