def run_script_view(self, request, endpoint, path, **kwargs):
# Use method specific metadata if defined, default to full metadata info
metadata = endpoint.metadata.get(self.request.method,
endpoint.metadata)
socket = endpoint.socket = Cached(Socket,
kwargs={
'pk': endpoint.socket_id
}).get()
if socket.is_new_format:
return self.run_codebox_script(request, socket, endpoint, metadata,
path, **kwargs)
# If it was skipped, set full data to Empty so it does get parsed and consumed.
if request._empty_data:
request._full_data = Empty
# Old sockets have lower payload limit.
if self.get_request_content_length(
request) > settings.SOCKETS_MAX_PARSED_PAYLOAD:
raise RequestLimitExceeded(settings.SOCKETS_MAX_PARSED_PAYLOAD)
try:
script = Cached(CodeBox,
kwargs={
'socket': endpoint.socket_id,
'path': path
}).get()
except CodeBox.DoesNotExist:
raise Http404()
return self.run_view(request,
obj=endpoint,
script=script,
metadata=metadata,
endpoint=endpoint)
def initialize_request(self, request, *args, **kwargs):
instance = getattr(request, 'instance', kwargs.get('instance', None))
if not isinstance(instance, Instance):
value = instance.lower()
instance = None
if INSTANCE_NAME_REGEX.match(value):
try:
instance = Cached(Instance, kwargs=dict(name=value)).get()
except Instance.DoesNotExist:
pass
if instance is not None:
try:
self.validate_instance(instance)
if getattr(request, 'instance',
None) is None and request.META.get(
'HTTP_HOST_TYPE') != 'hosting':
admin = Cached(Admin, kwargs={
'id': instance.owner_id
}).get()
admin.update_last_access()
self.kwargs['instance'] = instance
set_current_instance(instance)
except SyncanoException as ex:
request.error = ex
instance = None
request.instance = instance
return super().initialize_request(request, *args, **kwargs)
def create(self, request, *args, **kwargs):
"""Create new discount by redeeming a coupon"""
try:
try:
coupon = Coupon.objects.get(
name=get_from_request_data(request, 'coupon'))
except Coupon.DoesNotExist:
return Response(COUPON_DOESNT_EXIST,
status=status.HTTP_400_BAD_REQUEST)
try:
# this way should support both name and primary key (only supported option in django rest framework)
instance_identifier = get_from_request_data(
request, 'instance')
if isinstance(instance_identifier, int):
instance = Cached(
Instance, kwargs=dict(pk=instance_identifier)).get()
else:
instance = Cached(
Instance, kwargs=dict(name=instance_identifier)).get()
except Instance.DoesNotExist:
return Response(INSTANCE_DOESNT_EXIST,
status=status.HTTP_400_BAD_REQUEST)
customer = request.user
discount = coupon.redeem(instance=instance, customer=customer)
serialized_discount = DiscountSerializer(
discount, context={'request': request})
return Response(serialized_discount.data,
status=status.HTTP_201_CREATED)
except ValidationError as e:
return Response(e.message_dict, status=status.HTTP_400_BAD_REQUEST)
def __call__(self, request):
host = request.META.get('HTTP_HOST')
if not host or request.META.get('HTTP_HOST_TYPE') != 'hosting':
return self.get_response(request)
host = host.split(':', 1)[0]
is_custom_domain = not host.endswith(settings.HOSTING_DOMAIN)
if is_custom_domain:
try:
instance = Cached(Instance, kwargs={'domains__contains': [host], 'location': settings.LOCATION}).get()
except Instance.DoesNotExist:
raise Http404()
else:
instance = host.split('.')[0]
# Check if we're dealing with: <prefix>--<instance_name>
domain_data = instance.rsplit('--', 1)
if len(domain_data) == 2:
host, instance = domain_data
else:
host = '_default'
kwargs = {
'domain': host,
'instance': instance
}
return HostingView.as_view()(request, **kwargs)
def run(self, instance_pks):
for instance_pk in instance_pks:
instance = _get_instance(instance_pk)
if instance is None:
continue
if not OwnerInGoodStanding.is_admin_in_good_standing(
instance.owner_id):
continue
set_current_instance(instance)
schedules = CodeBoxSchedule.objects.get_for_process()
for schedule in schedules:
key = PERIODIC_SCHEDULE_TEMPLATE.format(
instance_pk=instance.pk,
schedule_pk=schedule.pk,
scheduled_at=schedule.scheduled_next.isoformat())
can_run = redis.set(name=key,
value=1,
nx=True,
ex=SCHEDULE_TRACE_TIMEOUT)
if not can_run:
time_in_queue = timezone.now() - schedule.scheduled_next
if time_in_queue > timedelta(minutes=2):
self.get_logger().warning(
"%s in %s cannot be run yet, because it was already scheduled for %s, time in queue: %s.",
schedule, instance,
schedule.scheduled_next.isoformat(), time_in_queue)
else:
ScheduleTask.delay(schedule.id, instance_pk)
if schedules:
admin = Cached(Admin, kwargs={'id': instance.owner_id}).get()
admin.update_last_access()
def test_automatic_invalidation_on_update(self):
obj = G(CacheableModel, value='initial')
obj_cached = Cached(CacheableModel, kwargs={'pk': obj.pk}).get()
self.assertEqual(obj_cached.value, obj.value)
obj.value = 'new'
obj.save()
obj_cached = Cached(CacheableModel, kwargs={'pk': obj.pk}).get()
self.assertEqual(obj_cached.value, obj.value)
def test_automatic_invalidation_on_delete(self):
self.assertRaises(CacheableModel.DoesNotExist, Cached(CacheableModel, kwargs={'pk': 1}).get)
obj = G(CacheableModel, value='initial')
obj_pk = obj.pk
obj_cached = Cached(CacheableModel, kwargs={'pk': obj_pk}).get()
self.assertEqual(obj_cached.value, obj.value)
obj.delete()
self.assertRaises(CacheableModel.DoesNotExist, Cached(CacheableModel, kwargs={'pk': obj_pk}).get)
def invalidate(self, request, *args, **kwargs):
endpoint = self.get_object()
socket = Cached(Socket, kwargs={'pk': endpoint.socket_id}).get()
cache_key = ENDPOINT_CACHE_KEY_TEMPLATE.format(
schema=request.instance.pk,
name=endpoint.name,
hash=socket.get_hash(),
)
redis.delete(cache_key)
return HttpResponse(status=status.HTTP_204_NO_CONTENT)
def test_sync_invalidation(self, task_mock):
obj = G(CacheableSyncModel, value='initial')
obj_cached = Cached(CacheableSyncModel, kwargs={'pk': obj.pk}).get()
self.assertEqual(obj_cached.value, obj.value)
self.assertFalse(task_mock.delay.called)
obj.value = 'new'
obj.save()
self.assertTrue(task_mock.delay.called)
version_key = Cached(CacheableSyncModel).get_version_key(obj)
task_mock.delay.assert_called_with(version_key)
def validate_value(self, view, field, value, expected_value_type=None):
target = field.target
query_fields = None
query_fields_extra = None
id_field = 'id'
if target == 'user':
# For target=user, use query_fields from v2 UserViewSet
from apps.users.v2.views import UserViewSet
target = 'user_profile'
query_fields = UserViewSet.query_fields
query_fields_extra = UserViewSet.query_fields_extra
id_field = 'owner_id'
if target == 'self':
klass = view.klass
else:
try:
klass = Cached(Klass, kwargs=dict(name=target)).get()
except Klass.DoesNotExist:
raise InvalidQuery('Referenced class "{klass}" does not exist.'.format(klass=target))
with loaded_klass(klass):
queryset = DataObject.objects.values(id_field).filter(_klass=klass)
for filter_backend in view.filter_backends:
queryset = filter_backend().process_query(view, queryset, value,
query_fields=query_fields,
query_fields_extra=query_fields_extra)
queryset = queryset[:settings.DATA_OBJECT_NESTED_QUERY_LIMIT]
return RawSQL(*queryset.query.get_compiler(using=queryset.db).as_sql())
def _validate_limit(self, limit, adjustable_limits):
period = Invoice.current_period()
value = getattr(self, limit)
if self.has_changed(limit):
if not adjustable_limits:
raise serializers.ValidationError({
limit:
'Limits are not adjustable on your current pricing plan.'
})
# Check if new limit doesn't make us hit the limit or should it be cleared
if Invoice.objects.filter(admin=self.admin_id,
period=period,
overage_amount__gt=value).exists():
reached_time = period
else:
# Otherwise clear the limit if it was reached before
reached_time = Profile.LIMIT_NOT_REACHED
invalidate = False
if getattr(self, '%s_reached' % limit) != reached_time:
setattr(self, '%s_reached' % limit, reached_time)
invalidate = True
if invalidate:
# Invalidate if needed
Cached(Profile).invalidate(self)
if value < 0:
raise serializers.ValidationError(
{limit: 'Needs to be equal or greater than "0".'})
def run(self, message_pk, **kwargs):
config = Cached(GCMConfig, kwargs={'id': 1}).get()
message = GCMMessage.objects.get(pk=message_pk)
environment = message.content.pop('environment')
api_key = getattr(config, '{}_api_key'.format(environment))
status = GCMMessage.STATUSES.ERROR
try:
if not api_key:
raise GCMException(
'GCM api key for "{}" environment is required.'.format(
environment))
status, result = self.make_request(api_key, message)
except GCMException as ex:
result = str(ex)
except Exception:
result = 'Internal server error.'
self.get_logger().error(
'Unhandled error during processing of GCMMessage[pk=%s] in Instance[pk=%s]',
message_pk,
self.instance.pk,
exc_info=1)
GCMMessage.objects.filter(pk=message_pk).update(status=status,
result=result)
def create_spec(cls,
instance,
obj,
additional_args,
result_key,
meta,
trace_pk,
expire_at=None,
template_name=None,
script_pk=None):
if script_pk is not None:
script = Cached(CodeBox, kwargs={'pk': script_pk}).get()
else:
script = obj.codebox
meta = json.loads(meta)
meta.update({
'executed_by': cls.trace_type,
'executor': obj.name,
'instance': instance.name
})
trace_spec = cls.create_trace_spec(instance,
obj=obj,
trace_pk=trace_pk)
return cls.create_script_spec(instance, script, additional_args, meta,
result_key, trace_spec, obj.socket,
expire_at, template_name)
def _validate_target(self, field_dict):
from apps.data.models import Klass
target = field_dict.get(self.reference_class_key)
if target is None:
raise serializers.ValidationError(
'Target must specify class name or "self".')
if not target or not isinstance(target, str):
raise serializers.ValidationError(
'Invalid value of target. Expected non-empty string.')
target = target.lower()
if target == 'users':
raise serializers.ValidationError(
'Target cannot be set to "{}". '
'Use "user" instead.'.format(target))
if target not in (
'self', 'user') and target not in self.ignored_target_classes:
try:
Cached(Klass, kwargs=dict(name=target)).get()
except Klass.DoesNotExist:
raise serializers.ValidationError(
'Class specified by target is missing.')
def run(self, model_class_name, object_pk, instance_pk=None, **kwargs):
logger = self.get_logger()
try:
model_class = apps.get_model(model_class_name)
except (LookupError, ValueError):
# do not retry
raise
if instance_pk is not None:
try:
instance = Cached(Instance, kwargs=dict(pk=instance_pk)).get()
set_current_instance(instance)
except Instance.DoesNotExist:
logger.warning(
'Error occurred during deletion of %s[pk=%s] in Instance[pk=%s]. '
'Instance no longer exists.', model_class_name, object_pk,
instance_pk)
return
logger.info('Deleting %s[pk=%s] in %s.', model_class_name,
object_pk, instance)
else:
logger.info('Deleting %s[pk=%s] in public schema.',
model_class_name, object_pk)
try:
db = router.db_for_write(model_class)
with transaction.atomic(db):
model_class.all_objects.filter(pk=object_pk).delete()
except Exception as exc:
raise self.retry(exc=exc)
def get_auth_user(cls, request):
user_key = cls.get_user_key(request)
if user_key:
try:
return Cached(User, kwargs=dict(key=user_key)).get()
except User.DoesNotExist:
pass
def get_billing_status(admin_id):
return Cached(
Profile,
args=('billing_status', timezone.now().month),
kwargs={
'pk': admin_id
},
compute_func=lambda: Profile._get_billing_status(admin_id)).get()
def get_active_subscription(admin_id):
return Cached(
Profile,
args=('active_subscription', timezone.now().month),
kwargs={
'pk': admin_id
},
compute_func=lambda: Subscription.objects.select_related(
'plan').active_for_admin(admin_id=admin_id).get()).get()
def match_cached(cls, instance_pk, event, signal):
def _match_cached(signal):
return cls.objects.match(event, signal)
return Cached(_match_cached,
args=(signal, ),
key='Trigger.Match',
version_key='i=%d;e=%s' %
(instance_pk, json.dumps(event, sort_keys=True)))
def get_auth(cls, api_key, instance):
lookup_kwargs = {'key': api_key}
if instance:
lookup_kwargs['instance'] = instance
try:
return Cached(ApiKey, kwargs=lookup_kwargs).get()
except ApiKey.DoesNotExist:
raise exceptions.AuthenticationFailed('No such API Key.')
def run_codebox_script(self, request, socket, endpoint, metadata, path, **kwargs):
skip_payload = request._empty_data
# Skip payload if we're dealing with small text/plain requests
# (for backwards compatibility when depending on META.user, META.admin)
uwsgi.add_var('PAYLOAD_PARSED', '0' if skip_payload else '1')
script = Munch(config={
'allow_full_access': True,
'timeout': kwargs.get('timeout', settings.SOCKETS_DEFAULT_TIMEOUT),
'async': kwargs.get('async', settings.SOCKETS_DEFAULT_ASYNC),
'mcpu': kwargs.get('mcpu', settings.SOCKETS_DEFAULT_MCPU),
},
runtime_name=kwargs.get('runtime', LATEST_NODEJS_RUNTIME),
source='')
if path in socket.file_list:
# Prepare spec.
script_files = socket.get_files()
entrypoint = socket.get_local_path(path)
spec = {
'files': script_files,
'source_hash': socket.get_hash(),
'entrypoint': entrypoint,
'output_limit': settings.SOCKETS_MAX_RESULT_SIZE,
'name': endpoint.name,
'cache': kwargs.get('cache', 0),
}
# Add environment
if socket.environment_id:
environment = Cached(SocketEnvironment, kwargs={'pk': socket.environment_id}).get()
if not environment.is_ready:
if environment.status == SocketEnvironment.STATUSES.ERROR:
raise SocketEnvironmentFailure()
raise SocketEnvironmentNotReady()
spec['environment'] = environment.get_hash()
spec['environment_url'] = environment.get_url()
return self.run_view(request, obj=endpoint, script=script, metadata=metadata, endpoint=endpoint,
spec=spec,
skip_payload=skip_payload, flat_args=True,
uwsgi_handler=self.get_codebox_handler)
def get_object(self):
lookup_url_kwarg = self.lookup_url_kwarg or self.lookup_field
# Raise 404 also when kwargs don't match the required types
try:
obj = Cached(self.model, kwargs={self.lookup_field: self.kwargs[lookup_url_kwarg]}).get()
except (self.model.DoesNotExist, TypeError, ValueError):
raise Http404
self.check_object_permissions(self.request, obj)
return obj
def get_file_cached(cls, hosting_id, path):
def _get_file():
try:
return cls.objects.filter(path=path, hosting=hosting_id).get()
except cls.DoesNotExist:
return 'DoesNotExist'
return Cached(_get_file,
key='Hosting.GetFile',
version_key='i=%d;h=%d;p=%s' % (get_current_instance().id, hosting_id, path))
def run(self, environment, **kwargs):
logger = self.get_logger()
config = Cached(APNSConfig, kwargs={'id': 1}).get()
certificate = getattr(config, '{}_certificate'.format(environment))
bundle_identifier = getattr(config,
'{}_bundle_identifier'.format(environment))
certfile = NamedTemporaryFile(mode='w', delete=False)
keyfile = NamedTemporaryFile(mode='w', delete=False)
try:
if not certificate:
raise APNSException(
'APNS certificate for "{}" environment is required.'.
format(environment))
if not bundle_identifier:
raise APNSException(
'APNS bundle identifier for "{}" environment is required.'.
format(environment))
p12 = crypto.load_pkcs12(str(certificate), '')
# Looks like Python 2.5 :(
certfile.write(
crypto.dump_certificate(crypto.FILETYPE_PEM,
p12.get_certificate()))
certfile.close()
keyfile.write(
crypto.dump_privatekey(crypto.FILETYPE_PEM,
p12.get_privatekey()))
keyfile.close()
socket = APNSFeedbackSocket(environment,
certfile=certfile.name,
keyfile=keyfile.name)
for timestamp, token in socket.read():
created_at = datetime.fromtimestamp(timestamp, tz=pytz.UTC)
registration_id = codecs.encode(token, 'hex_codec').decode()
logger.debug('Updating %s %s', timestamp, registration_id)
APNSDevice.objects.filter(
registration_id=registration_id,
created_at__lte=created_at).update(is_active=False)
except (APNSException, TypeError, SSLError, crypto.Error):
logger.warning(
'Error occurred during processing of APNS Feedback in Instance[pk=%s]',
self.instance.pk,
exc_info=1)
finally:
os.unlink(certfile.name)
os.unlink(keyfile.name)
def validate(self, data):
config = Cached(GCMConfig, kwargs={'id': 1}).get()
if 'content' in data:
environment = data['content']['environment']
else:
environment = self.instance.content['environment']
api_key = getattr(config, '{}_api_key'.format(environment))
if not api_key:
raise ValidationError('GCM api key for "{}" environment is required.'.format(environment))
return data
def get_admin_from_token(cls, token, instance):
if not instance:
return
instance_pk = verify_token(token)
if instance_pk != instance.pk:
return
try:
return Cached(Admin, kwargs={'id': instance.owner_id}).get()
except ApiKey.DoesNotExist:
raise exceptions.AuthenticationFailed('Invalid token.')
def get_incentive(self, instance, incentive_pk):
set_current_instance(instance)
try:
incentive = Cached(self.incentive_class, kwargs={'pk': incentive_pk}).get()
except self.incentive_class.DoesNotExist:
self.get_logger().warning(
"%s[pk=%s] for %s cannot be run, because script was not found.",
self.incentive_class.__name__, incentive_pk, instance)
return None
if hasattr(incentive, 'codebox_id'):
try:
incentive.codebox = Cached(CodeBox, kwargs={'pk': incentive.codebox_id}).get()
except CodeBox.DoesNotExist:
self.get_logger().warning(
"%s[pk=%s] for %s cannot be run, because script was not found.",
self.incentive_class.__name__, incentive.pk, instance)
return None
socket = None
if incentive.socket_id:
try:
socket = Cached(Socket, kwargs={'pk': incentive.socket_id}).get()
except Socket.DoesNotExist:
pass
incentive.socket = socket
return incentive
def process_object(self, socket, **kwargs):
if socket.install_url and not socket.zip_file:
self.download_socket_zip(socket)
self.socket_install = {
'endpoints_count': 0,
'dependencies_count': 0,
'data': []
}
is_trusted = Cached(Admin, kwargs={'id': get_current_instance().owner_id}).get().is_trusted
dependencies, is_partial = self.importer(socket, is_trusted=is_trusted).process()
self.add_socket_for_installation(socket, dependencies, is_partial)
def run(self, **kwargs):
logger = self.get_logger()
for domain in os.listdir(CERTS_PATH):
pem_cert = os.path.join(CERTS_PEM_PATH, '{}.pem'.format(domain))
# Skip non-domain files/directories
if domain.startswith('.') or '.' not in domain:
continue
# Try to get instance
try:
instance = Cached(Instance,
kwargs={
'domains__contains': [domain],
'location': settings.LOCATION
}).get()
except Instance.DoesNotExist:
instance = None
logger.warning('Instance with domain %s no longer exists',
domain)
# If instance does not exist or PEM file is missing - delete the redundant cert files
if instance is None or not os.path.exists(pem_cert):
self.remove_domain(logger, domain=domain)
continue
try:
subprocess.check_output([RENEW_SSL_CERT_SCRIPT, domain],
stderr=subprocess.STDOUT)
os.utime(pem_cert, None)
except subprocess.CalledProcessError as exc:
# If refresh was skipped, move along
if exc.returncode == REFRESH_SKIP_RETCODE:
os.utime(pem_cert, None)
continue
mtime = os.path.getmtime(pem_cert)
diff = datetime.now() - datetime.fromtimestamp(mtime)
logger.warning(
'Refresh failed for domain "%s" (last success: %s days ago) with output: %s',
domain,
diff.days,
exc.output,
exc_info=1)
if diff.days > MAX_DELETION_DAYS:
# Disabling SSL for that domain
self.remove_domain(logger,
domain=domain,
instance=instance)
def initial(self, request, *args, **kwargs):
parents_lookup_values = []
# Get all lookup values
for lookup in self.parents_query_lookups:
value = kwargs.pop('%s%s' % (extensions_api_settings.DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX, lookup), None)
parents_lookup_values.append(value)
# Loop through viewset breadcrumb and get all related objects in hierarchy
for i, parent_viewset in enumerate(self.viewset_breadcrumb):
value = parents_lookup_values[i]
model = getattr(parent_viewset, 'model', parent_viewset.queryset.model)
lookup_field = parent_viewset.lookup_field
if lookup_field == 'pk':
field = model._meta.pk
else:
field = model._meta.get_field(lookup_field)
# Validate field value first
if not isinstance(value, model):
try:
value = validate_field(field, value)
lookup_kwargs = {lookup_field: value}
if hasattr(parent_viewset, 'get_lookup'):
obj = parent_viewset.get_lookup(value)
elif issubclass(model, CacheableAbstractModel):
obj = Cached(model, kwargs=lookup_kwargs).get()
else:
obj = model.objects.get(**lookup_kwargs)
except (ValidationError, ObjectDoesNotExist, ValueError):
# If value is not valid for a field or model not found - raise an error
raise ModelNotFound(model)
else:
obj = value
class_name = obj.__class__.__name__.lower()
# Optional validation of model
validate_func = getattr(self, 'validate_{}'.format(class_name), None)
if validate_func is not None and not validate_func(obj):
raise ModelNotFound(model)
# Set object as both a field and in kwargs
setattr(self, class_name, obj)
kwargs[self.parents_query_lookups[i]] = obj
# Save updated kwargs
self.kwargs = kwargs
return super().initial(request, *args, **kwargs)