def get(self, request, *args, **kwargs):
table_name = kwargs.get('table', None)
connection_id = kwargs.get('pk', None)
page = request.GET.get('page', 1)
page_size = request.GET.get('page_size', 20)
column_sort = request.GET.get('column_sort', None)
sort = request.GET.get('sort', None)
try:
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider:
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
try:
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=provider_connection)
documents, count = mongo_db_manager.get_all_documents(db=db, collection=table_name,
column_sort=column_sort,
sort=sort, page=page, page_size=page_size)
data = list(documents)
result = json.loads(dumps(data))
return responses.paging_data(data=result, total_count=count, method=constant.POST,
entity_name='db_provider_connection')
except Exception as err:
return responses.bad_request(data=str(err), message_code='BD_ERROR')
else:
# TODO: implement another phase
pass
else:
return responses.bad_request(data='Provider not found', message_code='PROVIDER_NOT_FOUND')
except DBProviderConnection.DoesNotExist as err:
return responses.not_found(data=None, message_code='PROVIDER_CONNECTION_NOT_FOUND', message_system=err)
def get(self, request, *args, **kwargs):
custom_column_type_id = kwargs.get("custom_column_type_id")
try:
custom_column_validators = CustomColumnTypeValidator.objects.filter(
custom_column_type__id=custom_column_type_id)
serializer = self.get_serializer(custom_column_validators,
many=True)
return responses.ok(data=serializer.data,
method=constant.GET,
entity_name='custom-column-validator')
except Exception as err:
return responses.not_found(
data=None,
message_system=err,
message_code='GET_CUSTOM_COLUMN_VALIDATOR_NOT_FOUND')
def get(self, request, *args, **kwargs):
connection_id = kwargs.get("connection_id")
table_name = kwargs.get("table_name")
try:
sharing_key_decode = connection_id + ";" + table_name
sharing_key_encode = base64.b64encode(
bytes(sharing_key_decode, "utf-8"))
return responses.ok(data=sharing_key_encode.decode('utf-8'),
method=constant.GET,
entity_name='sharing-files')
except Exception as err:
return responses.not_found(
data=None,
message_code='SHARING_FILES_CAN_NOT_CREATE_LINK',
message_system=err)
def get(self, request, *args, **kwargs):
table_name = kwargs.get('table_name', None)
list_filter = kwargs.get('list_filter', None)
list_column = kwargs.get('list_column', None)
connection_id = kwargs.get('connection', None)
try:
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider:
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
try:
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=provider_connection)
# columns = mongo_db_manager.get_all_keys(db=db, collection=table_name)
# documents, count = mongo_db_manager.get_all_documents(db=db, collection=table_name,
# column_sort=None,
# sort=None, page=1, page_size=20)
documents = mongo_db_manager.export_db_by_column(db=db, table=table_name,
list_filter=list_filter,
list_column=list_column)
data = list(documents)
result = json.loads(dumps(data))
# final_data = []
# for d in result:
# i = []
# for k,v in d.items():
# i.append(v)
# final_data.append(i)
pdf = GeneratePdf(result, table_name, list_column)
response = pdf.generate_pdf(context={})
return response
except Exception as err:
return responses.bad_request(data=str(err), message_code='BD_ERROR')
else:
# TODO: implement another phase
pass
else:
return responses.bad_request(data='Provider not found', message_code='PROVIDER_NOT_FOUND')
except DBProviderConnection.DoesNotExist as err:
return responses.not_found(data=None, message_code='EXPORT_ERROR', message_system=err)
def get(self, request, *args, **kwargs):
connection_id = kwargs.get('pk', None)
try:
if connection_id:
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider:
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=provider_connection)
data = mongo_db_manager.get_all_collections(db=db, cache_db=cache_db)
return responses.ok(data=data, method=constant.POST, entity_name='db_provider_connection')
else:
# TODO: implement another phase
pass
else:
return responses.bad_request(data='Provider not found', message_code='PROVIDER_NOT_FOUND')
else:
return responses.bad_request(data=None, message_code='PROVIDER_CONNECTION_ID_EMPTY')
except Exception as err:
return responses.not_found(data=None, message_code='PROVIDER_CONNECTION_NOT_FOUND', message_system=err)
def get(self, request, *args, **kwargs):
try:
user = request.user
connection_id = kwargs.get("connection")
table_name = kwargs.get("table_name")
list_filter = kwargs.get('list_filter', None)
list_column = kwargs.get('list_column', None)
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
db, cache_db = mongo_db_manager.connection_mongo_by_provider(provider_connection=provider_connection)
documents = mongo_db_manager.export_db_by_column(db=db, table=table_name,
list_filter=list_filter,
list_column=list_column)
if documents.count() <= 1000:
result = json.loads(dumps(list(documents)))
headers = list(result[0].keys())
output = io.BytesIO()
workbook = xlsxwriter.Workbook(output)
worksheet = workbook.add_worksheet()
cell_format_header = workbook.add_format()
cell_format_header.set_bold()
for index in range(len(headers)):
worksheet.write(0, index, headers[index], cell_format_header)
for row_num, columns in enumerate(result):
for index in range(len(headers)):
value = columns.get(headers[index]) if index != 0 else columns.get(headers[index]).get(
'$oid')
worksheet.write(row_num + 1, index, value)
workbook.close()
output.seek(0)
today = datetime.now().strftime("%d%m%Y_%H%M%S")
filename = f"ExportData-{table_name}-{today}.xlsx"
response = HttpResponse(
output,
content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
)
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
else:
ExportData.objects.create(
provider_connection_id=provider_connection.id,
username=user.username,
table=table_name,
status=ExportData.INIT,
file_type=ExportData.EXCEL,
list_filter=list_filter,
list_column=list_column
)
return responses.ok(data="Waiting notify for export", method=constant.GET,
entity_name='export-data')
return responses.bad_request(data=None, message_code="SQL_PROVIDER_NOT_FOUND")
except Exception as err:
return responses.not_found(data=None, message_code='SQL_FUNCTION_NOT_FOUND', message_system=err)
def get(self, request, *args, **kwargs):
try:
user = request.user
connection_id = kwargs.get("connection")
table_name = kwargs.get("table_name")
list_filter = kwargs.get('list_filter', None)
list_column = kwargs.get('list_column', None)
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
db, cache_db = mongo_db_manager.connection_mongo_by_provider(provider_connection=provider_connection)
# c = db.__getattr__(table_name).find().limit(20)
documents = mongo_db_manager.export_db_by_column(db=db, table=table_name,
list_filter=list_filter,
list_column=list_column)
if documents.count() <= 1000:
result = json.loads(dumps(list(documents)))
headers = list(result[0].keys())
content = ''
for header in headers:
content += header
if headers.index(header) != len(headers) - 1:
content += ', '
else:
content += '\n'
for value in result:
for header in headers:
if header == "_id":
content += value.get(header).get('$oid')
else:
try:
content += value.get(header)
except:
content += ''
if headers.index(header) != len(headers) - 1:
content += ', '
else:
content += '\n'
today = datetime.now().strftime("%d%m%Y_%H%M%S")
filename = f"ExportData-{table_name}-{today}.txt"
response = HttpResponse(content, content_type='text/plain')
response['Content-Disposition'] = 'attachment; filename=%s' % filename
return response
else:
ExportData.objects.create(
provider_connection_id=provider_connection.id,
username=user.username,
table=table_name,
status=ExportData.INIT,
file_type=ExportData.EXCEL,
list_filter=list_filter,
list_column=list_column
)
return responses.ok(data="Waiting notify for export", method=constant.GET,
entity_name='export-data')
return responses.bad_request(data=None, message_code="SQL_PROVIDER_NOT_FOUND")
except Exception as err:
return responses.not_found(data=None, message_code='SQL_FUNCTION_NOT_FOUND', message_system=err)
def get(self, request, *args, **kwargs):
data = request.query_params
page = int(data.get("page", '1'))
page_size = int(data.get("page_size", '20'))
sharing_key_encode = kwargs.get("sharing_key")
try:
sharing_key_decode_utf8 = base64.b64decode(sharing_key_encode)
sharing_key_decode = sharing_key_decode_utf8.decode("utf-8")
sharing_key_array = sharing_key_decode.split(";")
if len(sharing_key_array) == 2:
connection_id = sharing_key_array[0]
table_name = sharing_key_array[1]
# Get data from connection and table name
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider:
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
try:
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=provider_connection)
columns = mongo_db_manager.get_all_keys(
db=db, collection=table_name)
documents, count = mongo_db_manager.get_all_documents(
db=db,
collection=table_name,
column_sort=None,
sort=None,
page=page,
page_size=page_size)
data = list(documents)
result_document = json.loads(json_util.dumps(data))
result = {
'columns': columns,
'documents': result_document
}
return responses.paging_data(
data=result,
total_count=count,
method=constant.GET,
entity_name='sharing_files')
except Exception as err:
return responses.bad_request(
data=err, message_code='BD_ERROR')
else:
# TODO: implement another phase
return responses.ok(data=None,
method=constant.GET,
entity_name='sharing_files')
else:
return responses.bad_request(
data='Provider not found',
message_code='PROVIDER_NOT_FOUND')
else:
return responses.not_found(
data=None, message_code='SHARING_FILES_GET_DATA_NOT_FOUND')
except Exception as err:
return responses.not_found(
data=None,
message_code='SHARING_FILES_GET_DATA_NOT_FOUND',
message_system=err)
def get(self, request, *args, **kwargs):
table_name = kwargs.get('table', None)
connection_id = kwargs.get('pk', None)
try:
provider_connection = self.queryset.get(id=connection_id)
provider = provider_connection.provider
if provider:
if provider.name == MONGO:
mongo_db_manager = MongoDBManager()
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=provider_connection)
documents, count = mongo_db_manager.get_all_documents(db=db, collection=table_name,
column_sort=None,
sort=None, page=1, page_size=20)
# TODO: PhuongTN -> get real column from database
columns = []
real_columns = mongo_db_manager.get_all_keys(db=db, collection=table_name)
# TODO: PhuongTN -> get custom column mapping
custom_columns = CustomColumnMapping.objects.filter(connection_id=connection_id,
table_name=table_name)
if custom_columns.exists():
if count == 0:
for cc in custom_columns:
obj = {
'id': cc.id,
'real_column': cc.real_column,
'custom_column_name': cc.custom_column_name,
'custom_column_id': cc.custom_column.id
}
columns.append(obj)
else:
custom_columns = custom_columns
for rc in real_columns:
is_append = False
for cc in custom_columns:
if rc == cc.real_column:
obj = {
'id': cc.id,
'real_column': cc.real_column,
'custom_column_name': cc.custom_column_name,
'custom_column_id': cc.custom_column.id
}
columns.append(obj)
is_append = True
break
if not is_append:
obj = {
'id': None,
'real_column': rc,
'custom_column_name': None,
'custom_column_id': None
}
columns.append(obj)
else:
for rc in real_columns:
obj = {
'id': None,
'real_column': rc,
'custom_column_name': None,
'custom_column_id': None
}
columns.append(obj)
return responses.ok(data=columns, method=constant.POST, entity_name='db_provider_connection')
else:
# TODO: implement another phase
pass
else:
return responses.bad_request(data='Provider not found', message_code='PROVIDER_NOT_FOUND')
except Exception as err:
return responses.not_found(data=None, message_code='PROVIDER_CONNECTION_NOT_FOUND', message_system=str(err))
def post(self, request, *args, **kwargs):
try:
user = request.user
file_obj = request.FILES['file']
# url_csv = upload_to_s3("import", file_obj.name, file_obj, user_id=user.id)
file = file_obj.read().decode('utf-8')
csv_data = csv.DictReader(StringIO(file))
connection_id = kwargs.get("connection")
table_name = kwargs.get("table_name")
connection = DBProviderConnection.objects.filter(
id=connection_id).first()
if connection.provider.name == MONGO:
mongo_db_manager = MongoDBManager()
db, cache_db = mongo_db_manager.connection_mongo_by_provider(
provider_connection=connection)
collections = mongo_db_manager.get_all_collections(
db=db, cache_db=cache_db)
if table_name not in collections:
mongo_db_manager.create_new_collection(db, table_name)
else:
headers = list(csv_data.fieldnames)
try:
columns = mongo_db_manager.get_all_keys(
db=db, collection=table_name)
if columns:
for header in headers:
if header not in columns:
return responses.bad_request(
data=
f"Column '{header}' is not exists in table {table_name}",
message_code="Column is not exists")
except Exception as err:
return responses.bad_request(data=str(err),
message_code=str(err))
# list_insert = []
# for row in csv_data:
# data = dict(row)
# # data["_id"] = str(ObjectId())
# list_insert.append(data)
#
# print(list_insert)
file_full_name = file_obj.name.split(".")
time_stamp = datetime.datetime.now().timestamp()
file_name = f"{file_full_name[0]}_{str(int(time_stamp))}.{file_full_name[1]}"
file_name = file_name.replace(" ", "_")
fs = FileSystemStorage(
location=f"{settings.MEDIA_ROOT}/import",
base_url=f"{settings.MEDIA_ROOT}/import")
filename = fs.save(file_name, file_obj)
uploaded_file_url = fs.url(filename)
static_dir = f"{settings.MEDIA_ROOT}/import/{filename}"
import_record = ImportData.objects.create(
provider_connection_id=connection.id,
username=user.username,
table=table_name,
file_url=static_dir)
# process_import_database.delay(import_id=import_record.id)
return responses.ok(data="waiting import data",
method='post',
entity_name='import_database')
return responses.bad_request(data=None,
message_code="SQL_PROVIDER_NOT_FOUND")
except Exception as err:
return responses.not_found(data=None,
message_code='SQL_FUNCTION_NOT_FOUND',
message_system=err)