def get(self, request, *args, **kwargs): table_name = kwargs.get('table', None) connection_id = kwargs.get('pk', None) try: provider_connection = self.queryset.get(id=connection_id) provider = provider_connection.provider if provider: if provider.name == MONGO: mongo_db_manager = MongoDBManager() db, cache_db = mongo_db_manager.connection_mongo_by_provider( provider_connection=provider_connection) documents, count = mongo_db_manager.get_all_documents(db=db, collection=table_name, column_sort=None, sort=None, page=1, page_size=20) # TODO: PhuongTN -> get real column from database columns = [] real_columns = mongo_db_manager.get_all_keys(db=db, collection=table_name) # TODO: PhuongTN -> get custom column mapping custom_columns = CustomColumnMapping.objects.filter(connection_id=connection_id, table_name=table_name) if custom_columns.exists(): if count == 0: for cc in custom_columns: obj = { 'id': cc.id, 'real_column': cc.real_column, 'custom_column_name': cc.custom_column_name, 'custom_column_id': cc.custom_column.id } columns.append(obj) else: custom_columns = custom_columns for rc in real_columns: is_append = False for cc in custom_columns: if rc == cc.real_column: obj = { 'id': cc.id, 'real_column': cc.real_column, 'custom_column_name': cc.custom_column_name, 'custom_column_id': cc.custom_column.id } columns.append(obj) is_append = True break if not is_append: obj = { 'id': None, 'real_column': rc, 'custom_column_name': None, 'custom_column_id': None } columns.append(obj) else: for rc in real_columns: obj = { 'id': None, 'real_column': rc, 'custom_column_name': None, 'custom_column_id': None } columns.append(obj) return responses.ok(data=columns, method=constant.POST, entity_name='db_provider_connection') else: # TODO: implement another phase pass else: return responses.bad_request(data='Provider not found', message_code='PROVIDER_NOT_FOUND') except Exception as err: return responses.not_found(data=None, message_code='PROVIDER_CONNECTION_NOT_FOUND', message_system=str(err))
def get(self, request, *args, **kwargs): data = request.query_params page = int(data.get("page", '1')) page_size = int(data.get("page_size", '20')) sharing_key_encode = kwargs.get("sharing_key") try: sharing_key_decode_utf8 = base64.b64decode(sharing_key_encode) sharing_key_decode = sharing_key_decode_utf8.decode("utf-8") sharing_key_array = sharing_key_decode.split(";") if len(sharing_key_array) == 2: connection_id = sharing_key_array[0] table_name = sharing_key_array[1] # Get data from connection and table name provider_connection = self.queryset.get(id=connection_id) provider = provider_connection.provider if provider: if provider.name == MONGO: mongo_db_manager = MongoDBManager() try: db, cache_db = mongo_db_manager.connection_mongo_by_provider( provider_connection=provider_connection) columns = mongo_db_manager.get_all_keys( db=db, collection=table_name) documents, count = mongo_db_manager.get_all_documents( db=db, collection=table_name, column_sort=None, sort=None, page=page, page_size=page_size) data = list(documents) result_document = json.loads(json_util.dumps(data)) result = { 'columns': columns, 'documents': result_document } return responses.paging_data( data=result, total_count=count, method=constant.GET, entity_name='sharing_files') except Exception as err: return responses.bad_request( data=err, message_code='BD_ERROR') else: # TODO: implement another phase return responses.ok(data=None, method=constant.GET, entity_name='sharing_files') else: return responses.bad_request( data='Provider not found', message_code='PROVIDER_NOT_FOUND') else: return responses.not_found( data=None, message_code='SHARING_FILES_GET_DATA_NOT_FOUND') except Exception as err: return responses.not_found( data=None, message_code='SHARING_FILES_GET_DATA_NOT_FOUND', message_system=err)
def post(self, request, *args, **kwargs): try: user = request.user file_obj = request.FILES['file'] # url_csv = upload_to_s3("import", file_obj.name, file_obj, user_id=user.id) file = file_obj.read().decode('utf-8') csv_data = csv.DictReader(StringIO(file)) connection_id = kwargs.get("connection") table_name = kwargs.get("table_name") connection = DBProviderConnection.objects.filter( id=connection_id).first() if connection.provider.name == MONGO: mongo_db_manager = MongoDBManager() db, cache_db = mongo_db_manager.connection_mongo_by_provider( provider_connection=connection) collections = mongo_db_manager.get_all_collections( db=db, cache_db=cache_db) if table_name not in collections: mongo_db_manager.create_new_collection(db, table_name) else: headers = list(csv_data.fieldnames) try: columns = mongo_db_manager.get_all_keys( db=db, collection=table_name) if columns: for header in headers: if header not in columns: return responses.bad_request( data= f"Column '{header}' is not exists in table {table_name}", message_code="Column is not exists") except Exception as err: return responses.bad_request(data=str(err), message_code=str(err)) # list_insert = [] # for row in csv_data: # data = dict(row) # # data["_id"] = str(ObjectId()) # list_insert.append(data) # # print(list_insert) file_full_name = file_obj.name.split(".") time_stamp = datetime.datetime.now().timestamp() file_name = f"{file_full_name[0]}_{str(int(time_stamp))}.{file_full_name[1]}" file_name = file_name.replace(" ", "_") fs = FileSystemStorage( location=f"{settings.MEDIA_ROOT}/import", base_url=f"{settings.MEDIA_ROOT}/import") filename = fs.save(file_name, file_obj) uploaded_file_url = fs.url(filename) static_dir = f"{settings.MEDIA_ROOT}/import/{filename}" import_record = ImportData.objects.create( provider_connection_id=connection.id, username=user.username, table=table_name, file_url=static_dir) # process_import_database.delay(import_id=import_record.id) return responses.ok(data="waiting import data", method='post', entity_name='import_database') return responses.bad_request(data=None, message_code="SQL_PROVIDER_NOT_FOUND") except Exception as err: return responses.not_found(data=None, message_code='SQL_FUNCTION_NOT_FOUND', message_system=err)