def run_script_view(self, request, endpoint, path, **kwargs): # Use method specific metadata if defined, default to full metadata info metadata = endpoint.metadata.get(self.request.method, endpoint.metadata) socket = endpoint.socket = Cached(Socket, kwargs={ 'pk': endpoint.socket_id }).get() if socket.is_new_format: return self.run_codebox_script(request, socket, endpoint, metadata, path, **kwargs) # If it was skipped, set full data to Empty so it does get parsed and consumed. if request._empty_data: request._full_data = Empty # Old sockets have lower payload limit. if self.get_request_content_length( request) > settings.SOCKETS_MAX_PARSED_PAYLOAD: raise RequestLimitExceeded(settings.SOCKETS_MAX_PARSED_PAYLOAD) try: script = Cached(CodeBox, kwargs={ 'socket': endpoint.socket_id, 'path': path }).get() except CodeBox.DoesNotExist: raise Http404() return self.run_view(request, obj=endpoint, script=script, metadata=metadata, endpoint=endpoint)
def initialize_request(self, request, *args, **kwargs): instance = getattr(request, 'instance', kwargs.get('instance', None)) if not isinstance(instance, Instance): value = instance.lower() instance = None if INSTANCE_NAME_REGEX.match(value): try: instance = Cached(Instance, kwargs=dict(name=value)).get() except Instance.DoesNotExist: pass if instance is not None: try: self.validate_instance(instance) if getattr(request, 'instance', None) is None and request.META.get( 'HTTP_HOST_TYPE') != 'hosting': admin = Cached(Admin, kwargs={ 'id': instance.owner_id }).get() admin.update_last_access() self.kwargs['instance'] = instance set_current_instance(instance) except SyncanoException as ex: request.error = ex instance = None request.instance = instance return super().initialize_request(request, *args, **kwargs)
def create(self, request, *args, **kwargs): """Create new discount by redeeming a coupon""" try: try: coupon = Coupon.objects.get( name=get_from_request_data(request, 'coupon')) except Coupon.DoesNotExist: return Response(COUPON_DOESNT_EXIST, status=status.HTTP_400_BAD_REQUEST) try: # this way should support both name and primary key (only supported option in django rest framework) instance_identifier = get_from_request_data( request, 'instance') if isinstance(instance_identifier, int): instance = Cached( Instance, kwargs=dict(pk=instance_identifier)).get() else: instance = Cached( Instance, kwargs=dict(name=instance_identifier)).get() except Instance.DoesNotExist: return Response(INSTANCE_DOESNT_EXIST, status=status.HTTP_400_BAD_REQUEST) customer = request.user discount = coupon.redeem(instance=instance, customer=customer) serialized_discount = DiscountSerializer( discount, context={'request': request}) return Response(serialized_discount.data, status=status.HTTP_201_CREATED) except ValidationError as e: return Response(e.message_dict, status=status.HTTP_400_BAD_REQUEST)
def __call__(self, request): host = request.META.get('HTTP_HOST') if not host or request.META.get('HTTP_HOST_TYPE') != 'hosting': return self.get_response(request) host = host.split(':', 1)[0] is_custom_domain = not host.endswith(settings.HOSTING_DOMAIN) if is_custom_domain: try: instance = Cached(Instance, kwargs={'domains__contains': [host], 'location': settings.LOCATION}).get() except Instance.DoesNotExist: raise Http404() else: instance = host.split('.')[0] # Check if we're dealing with: <prefix>--<instance_name> domain_data = instance.rsplit('--', 1) if len(domain_data) == 2: host, instance = domain_data else: host = '_default' kwargs = { 'domain': host, 'instance': instance } return HostingView.as_view()(request, **kwargs)
def run(self, instance_pks): for instance_pk in instance_pks: instance = _get_instance(instance_pk) if instance is None: continue if not OwnerInGoodStanding.is_admin_in_good_standing( instance.owner_id): continue set_current_instance(instance) schedules = CodeBoxSchedule.objects.get_for_process() for schedule in schedules: key = PERIODIC_SCHEDULE_TEMPLATE.format( instance_pk=instance.pk, schedule_pk=schedule.pk, scheduled_at=schedule.scheduled_next.isoformat()) can_run = redis.set(name=key, value=1, nx=True, ex=SCHEDULE_TRACE_TIMEOUT) if not can_run: time_in_queue = timezone.now() - schedule.scheduled_next if time_in_queue > timedelta(minutes=2): self.get_logger().warning( "%s in %s cannot be run yet, because it was already scheduled for %s, time in queue: %s.", schedule, instance, schedule.scheduled_next.isoformat(), time_in_queue) else: ScheduleTask.delay(schedule.id, instance_pk) if schedules: admin = Cached(Admin, kwargs={'id': instance.owner_id}).get() admin.update_last_access()
def test_automatic_invalidation_on_update(self): obj = G(CacheableModel, value='initial') obj_cached = Cached(CacheableModel, kwargs={'pk': obj.pk}).get() self.assertEqual(obj_cached.value, obj.value) obj.value = 'new' obj.save() obj_cached = Cached(CacheableModel, kwargs={'pk': obj.pk}).get() self.assertEqual(obj_cached.value, obj.value)
def test_automatic_invalidation_on_delete(self): self.assertRaises(CacheableModel.DoesNotExist, Cached(CacheableModel, kwargs={'pk': 1}).get) obj = G(CacheableModel, value='initial') obj_pk = obj.pk obj_cached = Cached(CacheableModel, kwargs={'pk': obj_pk}).get() self.assertEqual(obj_cached.value, obj.value) obj.delete() self.assertRaises(CacheableModel.DoesNotExist, Cached(CacheableModel, kwargs={'pk': obj_pk}).get)
def invalidate(self, request, *args, **kwargs): endpoint = self.get_object() socket = Cached(Socket, kwargs={'pk': endpoint.socket_id}).get() cache_key = ENDPOINT_CACHE_KEY_TEMPLATE.format( schema=request.instance.pk, name=endpoint.name, hash=socket.get_hash(), ) redis.delete(cache_key) return HttpResponse(status=status.HTTP_204_NO_CONTENT)
def test_sync_invalidation(self, task_mock): obj = G(CacheableSyncModel, value='initial') obj_cached = Cached(CacheableSyncModel, kwargs={'pk': obj.pk}).get() self.assertEqual(obj_cached.value, obj.value) self.assertFalse(task_mock.delay.called) obj.value = 'new' obj.save() self.assertTrue(task_mock.delay.called) version_key = Cached(CacheableSyncModel).get_version_key(obj) task_mock.delay.assert_called_with(version_key)
def validate_value(self, view, field, value, expected_value_type=None): target = field.target query_fields = None query_fields_extra = None id_field = 'id' if target == 'user': # For target=user, use query_fields from v2 UserViewSet from apps.users.v2.views import UserViewSet target = 'user_profile' query_fields = UserViewSet.query_fields query_fields_extra = UserViewSet.query_fields_extra id_field = 'owner_id' if target == 'self': klass = view.klass else: try: klass = Cached(Klass, kwargs=dict(name=target)).get() except Klass.DoesNotExist: raise InvalidQuery('Referenced class "{klass}" does not exist.'.format(klass=target)) with loaded_klass(klass): queryset = DataObject.objects.values(id_field).filter(_klass=klass) for filter_backend in view.filter_backends: queryset = filter_backend().process_query(view, queryset, value, query_fields=query_fields, query_fields_extra=query_fields_extra) queryset = queryset[:settings.DATA_OBJECT_NESTED_QUERY_LIMIT] return RawSQL(*queryset.query.get_compiler(using=queryset.db).as_sql())
def _validate_limit(self, limit, adjustable_limits): period = Invoice.current_period() value = getattr(self, limit) if self.has_changed(limit): if not adjustable_limits: raise serializers.ValidationError({ limit: 'Limits are not adjustable on your current pricing plan.' }) # Check if new limit doesn't make us hit the limit or should it be cleared if Invoice.objects.filter(admin=self.admin_id, period=period, overage_amount__gt=value).exists(): reached_time = period else: # Otherwise clear the limit if it was reached before reached_time = Profile.LIMIT_NOT_REACHED invalidate = False if getattr(self, '%s_reached' % limit) != reached_time: setattr(self, '%s_reached' % limit, reached_time) invalidate = True if invalidate: # Invalidate if needed Cached(Profile).invalidate(self) if value < 0: raise serializers.ValidationError( {limit: 'Needs to be equal or greater than "0".'})
def run(self, message_pk, **kwargs): config = Cached(GCMConfig, kwargs={'id': 1}).get() message = GCMMessage.objects.get(pk=message_pk) environment = message.content.pop('environment') api_key = getattr(config, '{}_api_key'.format(environment)) status = GCMMessage.STATUSES.ERROR try: if not api_key: raise GCMException( 'GCM api key for "{}" environment is required.'.format( environment)) status, result = self.make_request(api_key, message) except GCMException as ex: result = str(ex) except Exception: result = 'Internal server error.' self.get_logger().error( 'Unhandled error during processing of GCMMessage[pk=%s] in Instance[pk=%s]', message_pk, self.instance.pk, exc_info=1) GCMMessage.objects.filter(pk=message_pk).update(status=status, result=result)
def create_spec(cls, instance, obj, additional_args, result_key, meta, trace_pk, expire_at=None, template_name=None, script_pk=None): if script_pk is not None: script = Cached(CodeBox, kwargs={'pk': script_pk}).get() else: script = obj.codebox meta = json.loads(meta) meta.update({ 'executed_by': cls.trace_type, 'executor': obj.name, 'instance': instance.name }) trace_spec = cls.create_trace_spec(instance, obj=obj, trace_pk=trace_pk) return cls.create_script_spec(instance, script, additional_args, meta, result_key, trace_spec, obj.socket, expire_at, template_name)
def _validate_target(self, field_dict): from apps.data.models import Klass target = field_dict.get(self.reference_class_key) if target is None: raise serializers.ValidationError( 'Target must specify class name or "self".') if not target or not isinstance(target, str): raise serializers.ValidationError( 'Invalid value of target. Expected non-empty string.') target = target.lower() if target == 'users': raise serializers.ValidationError( 'Target cannot be set to "{}". ' 'Use "user" instead.'.format(target)) if target not in ( 'self', 'user') and target not in self.ignored_target_classes: try: Cached(Klass, kwargs=dict(name=target)).get() except Klass.DoesNotExist: raise serializers.ValidationError( 'Class specified by target is missing.')
def run(self, model_class_name, object_pk, instance_pk=None, **kwargs): logger = self.get_logger() try: model_class = apps.get_model(model_class_name) except (LookupError, ValueError): # do not retry raise if instance_pk is not None: try: instance = Cached(Instance, kwargs=dict(pk=instance_pk)).get() set_current_instance(instance) except Instance.DoesNotExist: logger.warning( 'Error occurred during deletion of %s[pk=%s] in Instance[pk=%s]. ' 'Instance no longer exists.', model_class_name, object_pk, instance_pk) return logger.info('Deleting %s[pk=%s] in %s.', model_class_name, object_pk, instance) else: logger.info('Deleting %s[pk=%s] in public schema.', model_class_name, object_pk) try: db = router.db_for_write(model_class) with transaction.atomic(db): model_class.all_objects.filter(pk=object_pk).delete() except Exception as exc: raise self.retry(exc=exc)
def get_auth_user(cls, request): user_key = cls.get_user_key(request) if user_key: try: return Cached(User, kwargs=dict(key=user_key)).get() except User.DoesNotExist: pass
def get_billing_status(admin_id): return Cached( Profile, args=('billing_status', timezone.now().month), kwargs={ 'pk': admin_id }, compute_func=lambda: Profile._get_billing_status(admin_id)).get()
def get_active_subscription(admin_id): return Cached( Profile, args=('active_subscription', timezone.now().month), kwargs={ 'pk': admin_id }, compute_func=lambda: Subscription.objects.select_related( 'plan').active_for_admin(admin_id=admin_id).get()).get()
def match_cached(cls, instance_pk, event, signal): def _match_cached(signal): return cls.objects.match(event, signal) return Cached(_match_cached, args=(signal, ), key='Trigger.Match', version_key='i=%d;e=%s' % (instance_pk, json.dumps(event, sort_keys=True)))
def get_auth(cls, api_key, instance): lookup_kwargs = {'key': api_key} if instance: lookup_kwargs['instance'] = instance try: return Cached(ApiKey, kwargs=lookup_kwargs).get() except ApiKey.DoesNotExist: raise exceptions.AuthenticationFailed('No such API Key.')
def run_codebox_script(self, request, socket, endpoint, metadata, path, **kwargs): skip_payload = request._empty_data # Skip payload if we're dealing with small text/plain requests # (for backwards compatibility when depending on META.user, META.admin) uwsgi.add_var('PAYLOAD_PARSED', '0' if skip_payload else '1') script = Munch(config={ 'allow_full_access': True, 'timeout': kwargs.get('timeout', settings.SOCKETS_DEFAULT_TIMEOUT), 'async': kwargs.get('async', settings.SOCKETS_DEFAULT_ASYNC), 'mcpu': kwargs.get('mcpu', settings.SOCKETS_DEFAULT_MCPU), }, runtime_name=kwargs.get('runtime', LATEST_NODEJS_RUNTIME), source='') if path in socket.file_list: # Prepare spec. script_files = socket.get_files() entrypoint = socket.get_local_path(path) spec = { 'files': script_files, 'source_hash': socket.get_hash(), 'entrypoint': entrypoint, 'output_limit': settings.SOCKETS_MAX_RESULT_SIZE, 'name': endpoint.name, 'cache': kwargs.get('cache', 0), } # Add environment if socket.environment_id: environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get() if not environment.is_ready: if environment.status == SocketEnvironment.STATUSES.ERROR: raise SocketEnvironmentFailure() raise SocketEnvironmentNotReady() spec['environment'] = environment.get_hash() spec['environment_url'] = environment.get_url() return self.run_view(request, obj=endpoint, script=script, metadata=metadata, endpoint=endpoint, spec=spec, skip_payload=skip_payload, flat_args=True, uwsgi_handler=self.get_codebox_handler)
def get_object(self): lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field # Raise 404 also when kwargs don't match the required types try: obj = Cached(self.model, kwargs={self.lookup_field: self.kwargs[lookup_url_kwarg]}).get() except (self.model.DoesNotExist, TypeError, ValueError): raise Http404 self.check_object_permissions(self.request, obj) return obj
def get_file_cached(cls, hosting_id, path): def _get_file(): try: return cls.objects.filter(path=path, hosting=hosting_id).get() except cls.DoesNotExist: return 'DoesNotExist' return Cached(_get_file, key='Hosting.GetFile', version_key='i=%d;h=%d;p=%s' % (get_current_instance().id, hosting_id, path))
def run(self, environment, **kwargs): logger = self.get_logger() config = Cached(APNSConfig, kwargs={'id': 1}).get() certificate = getattr(config, '{}_certificate'.format(environment)) bundle_identifier = getattr(config, '{}_bundle_identifier'.format(environment)) certfile = NamedTemporaryFile(mode='w', delete=False) keyfile = NamedTemporaryFile(mode='w', delete=False) try: if not certificate: raise APNSException( 'APNS certificate for "{}" environment is required.'. format(environment)) if not bundle_identifier: raise APNSException( 'APNS bundle identifier for "{}" environment is required.'. format(environment)) p12 = crypto.load_pkcs12(str(certificate), '') # Looks like Python 2.5 :( certfile.write( crypto.dump_certificate(crypto.FILETYPE_PEM, p12.get_certificate())) certfile.close() keyfile.write( crypto.dump_privatekey(crypto.FILETYPE_PEM, p12.get_privatekey())) keyfile.close() socket = APNSFeedbackSocket(environment, certfile=certfile.name, keyfile=keyfile.name) for timestamp, token in socket.read(): created_at = datetime.fromtimestamp(timestamp, tz=pytz.UTC) registration_id = codecs.encode(token, 'hex_codec').decode() logger.debug('Updating %s %s', timestamp, registration_id) APNSDevice.objects.filter( registration_id=registration_id, created_at__lte=created_at).update(is_active=False) except (APNSException, TypeError, SSLError, crypto.Error): logger.warning( 'Error occurred during processing of APNS Feedback in Instance[pk=%s]', self.instance.pk, exc_info=1) finally: os.unlink(certfile.name) os.unlink(keyfile.name)
def validate(self, data): config = Cached(GCMConfig, kwargs={'id': 1}).get() if 'content' in data: environment = data['content']['environment'] else: environment = self.instance.content['environment'] api_key = getattr(config, '{}_api_key'.format(environment)) if not api_key: raise ValidationError('GCM api key for "{}" environment is required.'.format(environment)) return data
def get_admin_from_token(cls, token, instance): if not instance: return instance_pk = verify_token(token) if instance_pk != instance.pk: return try: return Cached(Admin, kwargs={'id': instance.owner_id}).get() except ApiKey.DoesNotExist: raise exceptions.AuthenticationFailed('Invalid token.')
def get_incentive(self, instance, incentive_pk): set_current_instance(instance) try: incentive = Cached(self.incentive_class, kwargs={'pk': incentive_pk}).get() except self.incentive_class.DoesNotExist: self.get_logger().warning( "%s[pk=%s] for %s cannot be run, because script was not found.", self.incentive_class.__name__, incentive_pk, instance) return None if hasattr(incentive, 'codebox_id'): try: incentive.codebox = Cached(CodeBox, kwargs={'pk': incentive.codebox_id}).get() except CodeBox.DoesNotExist: self.get_logger().warning( "%s[pk=%s] for %s cannot be run, because script was not found.", self.incentive_class.__name__, incentive.pk, instance) return None socket = None if incentive.socket_id: try: socket = Cached(Socket, kwargs={'pk': incentive.socket_id}).get() except Socket.DoesNotExist: pass incentive.socket = socket return incentive
def process_object(self, socket, **kwargs): if socket.install_url and not socket.zip_file: self.download_socket_zip(socket) self.socket_install = { 'endpoints_count': 0, 'dependencies_count': 0, 'data': [] } is_trusted = Cached(Admin, kwargs={'id': get_current_instance().owner_id}).get().is_trusted dependencies, is_partial = self.importer(socket, is_trusted=is_trusted).process() self.add_socket_for_installation(socket, dependencies, is_partial)
def run(self, **kwargs): logger = self.get_logger() for domain in os.listdir(CERTS_PATH): pem_cert = os.path.join(CERTS_PEM_PATH, '{}.pem'.format(domain)) # Skip non-domain files/directories if domain.startswith('.') or '.' not in domain: continue # Try to get instance try: instance = Cached(Instance, kwargs={ 'domains__contains': [domain], 'location': settings.LOCATION }).get() except Instance.DoesNotExist: instance = None logger.warning('Instance with domain %s no longer exists', domain) # If instance does not exist or PEM file is missing - delete the redundant cert files if instance is None or not os.path.exists(pem_cert): self.remove_domain(logger, domain=domain) continue try: subprocess.check_output([RENEW_SSL_CERT_SCRIPT, domain], stderr=subprocess.STDOUT) os.utime(pem_cert, None) except subprocess.CalledProcessError as exc: # If refresh was skipped, move along if exc.returncode == REFRESH_SKIP_RETCODE: os.utime(pem_cert, None) continue mtime = os.path.getmtime(pem_cert) diff = datetime.now() - datetime.fromtimestamp(mtime) logger.warning( 'Refresh failed for domain "%s" (last success: %s days ago) with output: %s', domain, diff.days, exc.output, exc_info=1) if diff.days > MAX_DELETION_DAYS: # Disabling SSL for that domain self.remove_domain(logger, domain=domain, instance=instance)
def initial(self, request, *args, **kwargs): parents_lookup_values = [] # Get all lookup values for lookup in self.parents_query_lookups: value = kwargs.pop('%s%s' % (extensions_api_settings.DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX, lookup), None) parents_lookup_values.append(value) # Loop through viewset breadcrumb and get all related objects in hierarchy for i, parent_viewset in enumerate(self.viewset_breadcrumb): value = parents_lookup_values[i] model = getattr(parent_viewset, 'model', parent_viewset.queryset.model) lookup_field = parent_viewset.lookup_field if lookup_field == 'pk': field = model._meta.pk else: field = model._meta.get_field(lookup_field) # Validate field value first if not isinstance(value, model): try: value = validate_field(field, value) lookup_kwargs = {lookup_field: value} if hasattr(parent_viewset, 'get_lookup'): obj = parent_viewset.get_lookup(value) elif issubclass(model, CacheableAbstractModel): obj = Cached(model, kwargs=lookup_kwargs).get() else: obj = model.objects.get(**lookup_kwargs) except (ValidationError, ObjectDoesNotExist, ValueError): # If value is not valid for a field or model not found - raise an error raise ModelNotFound(model) else: obj = value class_name = obj.__class__.__name__.lower() # Optional validation of model validate_func = getattr(self, 'validate_{}'.format(class_name), None) if validate_func is not None and not validate_func(obj): raise ModelNotFound(model) # Set object as both a field and in kwargs setattr(self, class_name, obj) kwargs[self.parents_query_lookups[i]] = obj # Save updated kwargs self.kwargs = kwargs return super().initial(request, *args, **kwargs)