def collections_remove(request): if request.method != 'POST': raise PopupException(_('POST request required.')) response = {'status': -1} collections = json.loads(request.POST.get('collections', '[]')) if not collections: response['message'] = _('No collections to remove.') if response.get('message', None) is None: searcher = CollectionManagerController(request.user) solr_collections = searcher.get_collections() for collection in collections: if collection.get('name') in solr_collections: # Remove collection and instancedir searcher.delete_collection(collection.get('name'), collection.get('isCoreOnly')) response['status'] = 0 response['message'] = _('Collections removed!') return JsonResponse(response)
def test_end_to_end(self): if not is_live_cluster(): raise SkipTest() cluster = shared_cluster() fs = cluster.fs collection_name = "test_collection" indexer = Indexer("test", fs=fs, jt=cluster.jt) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format( {'file': { "stream": stream, "name": "test.csv" }}) field_types = indexer.guess_field_types({ "file": { "stream": stream, "name": "test.csv" }, "format": file_type_format }) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_unique_field(format_) is_unique_generated = indexer.is_unique_generated(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = indexer.get_kept_field_list(format_['columns']) if is_unique_generated: schema_fields += [{"name": unique_field, "type": "string"}] # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(collection_name, morphline, input_loc)
def test_create_collection(self): db = CollectionManagerController(self.user) name = get_db_prefix(name='solr') + 'test_create_collection' fields = [{'name': 'my_test', 'type': 'text'}] try: db.create_collection(name, fields, unique_key_field='id', df='text') finally: db.delete_collection(name, core=False)
def test_create_collection(self): db = CollectionManagerController(self.user) name = get_db_prefix(name='solr') + 'test_create_collection' fields = [{'name': 'my_test', 'type': 'text'}] try: db.create_collection(name, fields, unique_key_field='id', df='text') finally: db.delete_collection(name, core=False)
def test_end_to_end(self): if not is_live_cluster() or True: # Skipping as requires morplines libs to be setup raise SkipTest() cluster = shared_cluster() fs = cluster.fs make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False) user = User.objects.get(username="******") collection_name = "test_collection" indexer = Indexer("test", fs=fs, jt=cluster.jt) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format}) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_unique_field(format_) is_unique_generated = indexer.is_unique_generated(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = indexer.get_kept_field_list(format_['columns']) if is_unique_generated: schema_fields += [{"name": unique_field, "type": "string"}] # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)
def test_end_to_end(self): if not is_live_cluster() or True: # Skipping as requires morplines libs to be setup raise SkipTest() cluster = shared_cluster() fs = cluster.fs make_logged_in_client(username="******", groupname="default", recreate=True, is_superuser=False) user = User.objects.get(username="******") collection_name = "test_collection" indexer = MorphlineIndexer("test", fs=fs, jt=cluster.jt, solr_client=self.solr_client) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=TestIndexer.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format}) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_unique_field(format_) is_unique_generated = indexer.is_unique_generated(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = indexer.get_kept_field_list(format_['columns']) if is_unique_generated: schema_fields += [{"name": unique_field, "type": "string"}] # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(MockedRequest(user=user, fs=cluster.fs, jt=cluster.jt), collection_name, morphline, input_loc)
def collections_remove(request): if request.method != 'POST': raise PopupException(_('POST request required.')) response = {'status': -1} collections = json.loads(request.POST.get('collections', '[]')) if not collections: response['message'] = _('No collections to remove.') if response.get('message', None) is None: searcher = CollectionManagerController(request.user) solr_collections = searcher.get_collections() for collection in collections: if collection.get('name') in solr_collections: # Remove collection and instancedir searcher.delete_collection(collection.get('name'), collection.get('isCoreOnly')) response['status'] = 0 response['message'] = _('Collections removed!') return JsonResponse(response)
def test_end_to_end(self): fs = cluster.get_hdfs() collection_name = "test_collection" indexer = Indexer("test", fs) input_loc = "/tmp/test.csv" # upload the test file to hdfs fs.create(input_loc, data=IndexerTest.simpleCSVString, overwrite=True) # open a filestream for the file on hdfs stream = fs.open(input_loc) # guess the format of the file file_type_format = indexer.guess_format({'file': {"stream": stream, "name": "test.csv"}}) field_types = indexer.guess_field_types({"file":{"stream": stream, "name": "test.csv"}, "format": file_type_format}) format_ = field_types.copy() format_['format'] = file_type_format # find a field name available to use for the record's uuid unique_field = indexer.get_uuid_name(format_) # generate morphline morphline = indexer.generate_morphline_config(collection_name, format_, unique_field) schema_fields = [{"name": unique_field, "type": "string"}] + indexer.get_kept_field_list(format_['columns']) # create the collection from the specified fields collection_manager = CollectionManagerController("test") if collection_manager.collection_exists(collection_name): collection_manager.delete_collection(collection_name, None) collection_manager.create_collection(collection_name, schema_fields, unique_key_field=unique_field) # index the file indexer.run_morphline(collection_name, morphline, input_loc)