def test_require_all_override_by_required():
schema = {'foo': {'type': 'string', 'required': False}}
validator = Validator(require_all=True)
assert_success({}, schema, validator)
assert_success({'foo': 'bar'}, schema, validator)
validator.require_all = False
assert_success({}, schema, validator)
assert_success({'foo': 'bar'}, schema, validator)
schema = {'foo': {'type': 'string', 'required': True}}
validator.require_all = True
assert_fail(
{},
schema,
validator,
error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True),
)
assert_success({'foo': 'bar'}, schema, validator)
validator.require_all = False
assert_fail(
{},
schema,
validator,
error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True),
)
assert_success({'foo': 'bar'}, schema, validator)
def test_require_all_and_exclude():
schema = {
'foo': {
'type': 'string',
'excludes': 'bar'
},
'bar': {
'type': 'string',
'excludes': 'foo'
},
}
validator = Validator(require_all=True)
assert_fail(
{},
schema,
validator,
errors=[
('foo', '__require_all__', errors.REQUIRED_FIELD, True),
('bar', '__require_all__', errors.REQUIRED_FIELD, True),
],
)
assert_success({'foo': 'value'}, schema, validator)
assert_success({'bar': 'value'}, schema, validator)
assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator)
validator.require_all = False
assert_success({}, schema, validator)
assert_success({'foo': 'value'}, schema, validator)
assert_success({'bar': 'value'}, schema, validator)
assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator)
def test_require_all_simple():
schema = {'foo': {'type': 'string'}}
validator = Validator(require_all=True)
assert_fail(
{},
schema,
validator,
error=('foo', '__require_all__', errors.REQUIRED_FIELD, True),
)
assert_success({'foo': 'bar'}, schema, validator)
validator.require_all = False
assert_success({}, schema, validator)
assert_success({'foo': 'bar'}, schema, validator)
"type": "list",
"schema": {
"type": "string"
}
},
"capital": {
"type": "string",
},
"stateflower": {
"type": "string"
},
},
}
v = Validator()
v.require_all = True
v.schema = {
"states": {
"type": "dict",
"schema": {
"utah": state_schema,
"texas": state_schema,
"illinois": state_schema,
"missouri": state_schema,
},
},
}
with open("cities.yaml") as f:
data = yaml.load(f, Loader=yaml.FullLoader)
import os
from cerberus import Validator
from flask import Flask
from flask_cors import CORS
from flask_jwt_extended import JWTManager
from visualize.models import db, bcrypt, blacklist
jwt = JWTManager()
validator = Validator()
validator.require_all = True
@jwt.token_in_blacklist_loader
def check_if_token_in_blacklist(decrypted_token):
jti = decrypted_token['jti']
return blacklist.is_jti_in_blacklist(jti)
def create_app(test_config=None):
# create and configure the app
app = Flask(__name__, instance_relative_config=True)
# load the the standard configuartion depending on environment
if app.config["ENV"] == "production":
app.config.from_object("config.ProductionConfig")
elif app.config["ENV"] == "development":
app.config.from_object("config.DevelopmentConfig")
elif app.config["ENV"] == "testing":
app.config.from_object("config.TestingConfig")
def put(self, request, slug, pk):
if slug == "Song":
print(request.data)
try:
song = Song.objects.get(pk=pk)
except ObjectDoesNotExist:
return Response(
{"errors": "Song with that id does not exists"},
status.HTTP_500_INTERNAL_SERVER_ERROR)
v = Validator(self.schema1)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
name = request.data["name"]
duration = request.data['duration']
song = Song.objects.get(pk=pk)
song.name = name
song.duration_in_number_of_seconds = duration
song.save()
return Response({"status": "success", "data": model_to_dict(song)})
elif slug == "Audiobook":
try:
audiobook = Audiobook.objects.get(pk=pk)
except ObjectDoesNotExist:
return Response(
{"errors": "Audiobook with that id does not exists"},
status.HTTP_500_INTERNAL_SERVER_ERROR)
v = Validator(self.schema3)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
title = request.data["title"]
author = request.data["author"]
narrator = request.data["narrator"]
duration = request.data['duration']
audiobook = Audiobook.objects.get(pk=pk)
audiobook.title = title
audiobook.author = author
audiobook.narrator = narrator
audiobook.duration_in_number_of_seconds = duration
audiobook.save()
return Response({
"status": "success",
"data": model_to_dict(audiobook)
})
elif slug == "Podcast":
try:
podcast = Podcast.objects.get(pk=pk)
except ObjectDoesNotExist:
return Response(
{"errors": "Podcast with that id does not exists"},
status.HTTP_500_INTERNAL_SERVER_ERROR)
v = Validator(self.schema2)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
name = request.data["name"]
duration = request.data['duration']
host = request.data['host']
if "participant" in request.data:
participant = request.data['participant']
podcast = Podcast.objects.get(pk=pk)
podcast.name = name
podcast.duration_in_number_of_seconds = duration
podcast.host = host
if "participant" in request.data:
podcast.participant = participant
podcast.save()
return Response({
"status": "success",
"data": model_to_dict(podcast)
})
else:
return Response(
{
"errors":
"Invalid audioFileType. audioFileType can either be Song,Podcast or Audiobook "
}, status.HTTP_400_BAD_REQUEST)
def post(self, request):
file_type = request.data["audioFileType"]
if file_type == "Song":
v = Validator(self.schema1)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
name = request.data["name"]
duration = request.data['duration']
song = Song.objects.create(name=name,
duration_in_number_of_seconds=duration)
return Response({"status": "success", "data": model_to_dict(song)})
elif file_type == "Podcast":
v = Validator(self.schema2)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
name = request.data["name"]
duration = request.data['duration']
host = request.data['host']
if "participant" in request.data:
participant = request.data['participant']
podcast = Podcast.objects.create(
name=name,
duration_in_number_of_seconds=duration,
host=host,
participant=participant)
else:
podcast = Podcast.objects.create(
name=name,
duration_in_number_of_seconds=duration,
host=host)
return Response({
"status": "success",
"data": model_to_dict(podcast)
})
elif file_type == "Audiobook":
v = Validator(self.schema3)
v.require_all = True
if not v.validate(request.data):
return Response({"errors": v.errors},
status.HTTP_400_BAD_REQUEST)
title = request.data["title"]
author = request.data["author"]
narrator = request.data["narrator"]
duration = request.data['duration']
audiobook = Audiobook.objects.create(
title=title,
duration_in_number_of_seconds=duration,
author=author,
narrator=narrator)
return Response({
"status": "success",
"data": model_to_dict(audiobook)
})
else:
return Response(
{
"errors":
"Invalid audioFileType. audioFileType can either be Song,Podcast or Audiobook "
}, status.HTTP_400_BAD_REQUEST)
def lambda_handler(event, context):
'''
This function validates input raw dataset against schema specified in the env variable
'''
print(event)
result = {}
s3_resource = boto3.resource('s3')
#to_date = lambda s: datetime.strptime(s, os.environ['dateformat'])
bucket_name = event['bucket_name']
key_name = event['key_name']
source_file_name = event['file_name']
#source_file_name_to_copy = bucket_name + "/" + key_name
#error_file_name="error/" + source_file_name
schema = {}
schema = json.loads(os.environ['schema'])
for keys in schema:
if 'format' in schema[keys]:
date_format_provided = schema[keys]['format']
to_date = lambda s: datetime.strptime(s, date_format_provided)
schema[keys].pop("format")
schema[keys]['coerce'] = to_date
v = Validator(schema)
v.allow_unknown = False
v.require_all = True
source_file_path = "s3://" + bucket_name + "/" + key_name
try:
df = pd.read_csv(source_file_path)
print("Successfuly read : " + source_file_path)
except:
result['Validation'] = "FAILURE"
result['Reason'] = "Errro while reading csv"
result['Location'] = os.environ['source_folder_name']
print("Error while reading csv")
return (result)
result['Validation'] = "SUCCESS"
result['Location'] = os.environ['source_folder_name']
df_dict = df.to_dict(orient='records')
transformed_file_name = "s3://" + bucket_name + "/" + str(
os.environ['stage_folder_name']) + "/" + source_file_name
if len(df_dict) == 0:
result['Validation'] = "FAILURE"
result['Reason'] = "NO RECORD FOUND"
result['Location'] = os.environ['source_folder_name']
print("Moving file to error folder")
return (result)
for idx, record in enumerate(df_dict):
if not v.validate(record):
result['Validation'] = "FAILURE"
result['Reason'] = str(v.errors) + " in record number " + str(idx)
result['Location'] = os.environ['source_folder_name']
print("Moving file to error folder")
return (result)
break
df['Month'] = df['Date'].astype(str).str[0:2]
df['Day'] = df['Date'].astype(str).str[3:5]
df['Year'] = df['Date'].astype(str).str[6:10]
df.to_csv(transformed_file_name, index=False)
s3_resource.Object(bucket_name, key_name).delete()
print("Successfuly moved file to : " + transformed_file_name)
return (result)
