for i, doc in enumerate(docs):
if 'short_description' not in docs[i].keys():
short_description = summarise(doc['description'])
docs[i]['short_description'] = short_description
# LOGGER.debug(short_description)
vi_client = ViClient(os.environ['VH_USERNAME'], os.environ['VH_API_KEY'])
ids = vi_client.get_field_across_documents('_id', docs)
if args.reset_collection:
if args.collection_name in vi_client.list_collections():
vi_client.delete_collection(args.collection_name)
time.sleep(5)
text_encoder = ViText2Vec(os.environ['VH_USERNAME'], os.environ['VH_API_KEY'])
response = vi_client.insert_documents(args.collection_name, docs, models={'description': text_encoder})
LOGGER.debug(response)
print(response)
if response['failed'] != 0:
raise ValueError("Failed IDs")
if args.evaluate_results:
LOGGER.debug("Checking Documents:")
LOGGER.debug(vi_client.head(args.collection_name))
LOGGER.debug(vi_client.head(args.collection_name)['vector_length'])
LOGGER.debug(vi_client.collection_schema(args.collection_name))
import pandas as pd
pd.set_option('display.max_colwidth', None)
LOGGER.debug(vi_client.show_json(vi_client.random_documents(args.collection_name), selected_fields=['markdown_without_example']))
# 'markdown_without_example': 'text',
# 'model_id': 'text',
# 'model_name': 'text',
# 'model_versions': 'text',
# 'other_notes:': 'text',
# 'paper': 'text',
# 'release_date': 'date',
# 'repo': 'text',
# 'tasks': 'text',
# 'training_corpora:': 'text',
# 'vector_length': 'text'})
response = vi_client.insert_documents(args.collection_name,
docs,
models={'description': text_encoder})
if response['failed'] != 0:
print(response)
raise ValueError("Failed IDs")
if args.evaluate_results:
print("Checking Documents:")
print(vi_client.head(args.collection_name))
print(vi_client.head(args.collection_name)['vector_length'])
print(vi_client.collection_schema(args.collection_name))
import pandas as pd
pd.set_option('display.max_colwidth', None)
print(
vi_client.show_json(vi_client.random_documents(
args.collection_name),
selected_fields=['markdown_without_example']))