class DatasetRepo:
def __init__(self):
self.model = Dataset
self.resource = "Dataset"
self.response = Response(self.resource)
def create(self, data: dict):
try:
return self.model(**data).save()
except NotUniqueError as e:
raise e
except ValidationError as e:
raise e
except Exception as e:
raise e
def by_id(self, object_id):
try:
return self.model.objects.get(id=object_id)
except Exception as e:
return self.response.exception(description=e.__str__())
def by_criteria(self,
criteria: dict,
order: str = "created",
items: int = None,
status=None):
try:
if status:
dataset = self.model.objects(status,
**criteria).order_by(order)
else:
dataset = self.model.objects(**criteria).order_by(order)
if items:
return dataset[:items]
return dataset
except Exception as e:
return self.response.exception(description=e.__str__())
def paginate(self, criteria: dict, page, per_page, order: str = "created"):
try:
return (self.model.objects(**criteria).order_by(order).paginate(
page, per_page))
except Exception as e:
raise e
def post(self, *args, **kwargs):
resource = "Dataset upload"
response = Response(resource)
repo = repositories.DatasetRepo()
parse = reqparse.RequestParser()
parse.add_argument("dataset_file", type=FileStorage, location="files")
parse.add_argument("kind", type=str, location="form")
args = parse.parse_args()
file = args["dataset_file"]
kind = args["kind"]
if file:
extension = os.path.splitext(file.filename)[1]
filename = str(uuid.uuid4()) + extension
# TODO: Improve it to save the file on s3 bucket
# TODO: The file is too large to upload, so the best choice is:
# TODO: - save the zipped file
# TODO: - put the file directly on s3 from FRONT END
path = os.path.join(current_app.config["UPLOAD_FOLDER"], filename)
file.save(path)
try:
payload = {
"filename": filename,
"path": current_app.config["UPLOAD_FOLDER"],
}
dataset = repo.create(payload)
# TODO: If the upload was did directly to s3 we can send the create event
# to rabbitmq or sqs to process this file uploaded instead use celery
tasks.process.delay(f"{dataset.id}", kind)
except (NotUniqueError, ValidationError, Exception) as e:
response.exception(description=e.__str__())
return response.ok(
Messages.RESOURCE_CREATED.value.format(resource),
data={"id": f"{dataset.id}"},
)
return response.data_invalid(
{"file": "Não encontramos um arquivo a ser carregado"}
)
def get(self, *args, **kwargs):
resource = "Logs from dataset"
response = Response(resource)
repo = repositories.LogRepo()
schema = schemas.ListLogSchema(many=True)
parser = reqparse.RequestParser()
parser.add_argument(
"per_page", type=int, required=False, default=10, location="args"
)
args = parser.parse_args()
schema = schemas.ListLogSchema(many=True)
page = kwargs["page_id"]
page_size = args["per_page"]
criteria = {"dataset_id": kwargs["dataset_id"]}
try:
logs = repo.paginate(criteria, page, page_size)
except Exception as e:
return response.exception(description=e.__str__())
# criamos dados extras a serem respondidos
extra = {
"page": logs.page,
"pages": logs.pages,
"total": logs.total,
"params": {"page_size": page_size},
}
# fazemos um dump dos objetos pesquisados
result = schema.dump(logs.items)
return response.ok(
Messages.RESOURCE_FETCHED_PAGINATED.value.format(resource),
data=result.data,
**extra,
)