def search_with_dataset(dataset_path, query=None, **kwargs):
"""Search the datamart using a dataset"""
if not isfile(dataset_path):
user_msg = ('The dataset file could not be found.')
return err_resp(user_msg)
search_url = get_nyu_url() + '/search'
# --------------------------------
# Behavioral logging
# --------------------------------
if 'user_workspace' in kwargs:
log_data = dict(feature_id=f'POST|by-dataset|{search_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_SEARCH,
path=search_url)
LogEntryMaker.create_datamart_entry(kwargs['user_workspace'],
log_data)
# --------------------------------
# --------------------------------
# Query the datamart
# --------------------------------
try:
with open(dataset_path, 'rb') as dataset_p:
search_files = dict(data=dataset_p)
if query:
search_files['query'] = query
try:
response = requests.post(\
search_url,
files=search_files,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' %
err_obj)
except IOError as err_obj:
user_msg = (f'Failed to search with the dataset file.'
f' Technical: {err_obj}')
return err_resp(user_msg)
if response.status_code != 200:
print(str(response))
print(response.text)
return err_resp(('NYU Datamart internal server error.'
' status_code: %s') % response.status_code)
json_results = response.json()['results']
if not json_results:
return err_resp('No datasets found. (%s)' % \
(get_timestamp_string_readable(time_only=True),))
print('num results: ', len(json_results))
return ok_resp(json_results)
def datamart_augment(user_workspace,
data_path,
search_result,
exact_match=False,
**kwargs):
if not isinstance(user_workspace, UserWorkspace):
return err_resp('user_workspace must be a UserWorkspace')
if not isfile(data_path):
user_msg = f'Original data file not found: {data_path}'
return err_resp(user_msg)
# ----------------------------
# mock call
# ----------------------------
# 291780000
"""
right_data = '291770000'
left_columns= '[[2]]'
right_columns = '[[6]]'
exact_match = True
data_path = '/Users/ramanprasad/Documents/github-rp/TwoRavens/ravens_volume/test_data/TR1_Greed_Versus_Grievance/TRAIN/dataset_TRAIN/tables/learningData.csv'
"""
# ----------------------------
LOGGER.info('(1) build path')
datamart_id = search_result[dm_static.KEY_ISI_DATAMART_ID]
dest_filepath_info = DatamartJobUtilISI.get_output_filepath(
user_workspace,
f'{datamart_id}-{get_timestamp_string()}',
dir_type=dm_static.KEY_AUGMENT)
if not dest_filepath_info.success:
return err_resp(dest_filepath_info.err_msg)
augment_filepath = dest_filepath_info.result_obj
augment_url = get_isi_url() + '/augment'
# ----------------------------
# Behavioral logging
# ----------------------------
log_data = dict(feature_id=f'POST|{augment_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_AUGMENT,
path=augment_url)
LogEntryMaker.create_datamart_entry(user_workspace, log_data)
# ----------------------------
try:
response = requests.post(
augment_url,
data={
'task': json.dumps(search_result),
'format': 'd3m'
},
files={'data': open(data_path, 'r')},
verify=False,
timeout=settings.DATAMART_VERY_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' % err_obj)
if not response.status_code == 200:
user_msg = (f'ISI Augment response failed with status code: '
f'{response.status_code}.')
return err_resp(user_msg)
save_info = DatamartJobUtilISI.save_datamart_file(\
augment_filepath,
response)
if not save_info.success:
return err_resp(save_info.err_msg)
save_info = save_info.result_obj
# -----------------------------------------
# Retrieve preview rows and return response
# -----------------------------------------
# preview rows
#
preview_info = read_file_rows(save_info[dm_static.KEY_DATA_PATH],
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
# Format/return reponse
#
info_dict = DatamartJobUtilISI.format_materialize_response( \
datamart_id,
dm_static.DATAMART_ISI_NAME,
save_info[dm_static.KEY_DATA_PATH],
preview_info,
**save_info)
return ok_resp(info_dict)
def datamart_materialize(user_workspace, search_result):
"""Materialize the dataset"""
LOGGER.info('-- atttempt to materialize ISI dataset --')
if not isinstance(user_workspace, UserWorkspace):
return err_resp('user_workspace must be a UserWorkspace')
if not isinstance(search_result, dict):
return err_resp('search_result must be a python dictionary')
if dm_static.KEY_ISI_DATAMART_ID not in search_result:
user_msg = (f'"search_result" did not contain'
f' "{dm_static.KEY_ISI_DATAMART_ID}" key')
return err_resp(user_msg)
# -----------------------------------------
# Format output file path
# -----------------------------------------
LOGGER.info('(1) build path')
datamart_id = search_result[dm_static.KEY_ISI_DATAMART_ID]
dest_filepath_info = DatamartJobUtilISI.get_output_filepath(\
user_workspace,
datamart_id,
dir_type='materialize')
if not dest_filepath_info.success:
return err_resp(dest_filepath_info.err_msg)
dest_filepath = dest_filepath_info.result_obj
LOGGER.info('(2) Download file')
# -----------------------------------------
# Has the file already been downloaded?
# -----------------------------------------
print('dest_filepath', dest_filepath)
if isfile(dest_filepath):
LOGGER.info('(2a) file already downloaded')
# Get preview rows
#
preview_info = read_file_rows(dest_filepath,
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
info_dict = DatamartJobUtilISI.format_materialize_response(\
datamart_id, dm_static.DATAMART_ISI_NAME,
dest_filepath, preview_info)
return ok_resp(info_dict)
# -----------------------------------------
# Download the file
# -----------------------------------------
# can this be streamed to a file?
LOGGER.info('(2b) attempting download')
# ----------------------------
# Behavioral logging
# ----------------------------
isi_materialize_url = get_isi_url() + f'/download/{datamart_id}'
log_data = dict(feature_id=f'GET|{isi_materialize_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_DOWNLOAD,
path=isi_materialize_url)
LogEntryMaker.create_datamart_entry(user_workspace, log_data)
try:
print('isi_materialize_url', isi_materialize_url)
response = requests.get(\
isi_materialize_url,
params={'id': datamart_id, 'format': 'd3m'},
verify=False,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' % err_obj)
if response.status_code != 200:
user_msg = (f'Materialize failed. Status code:'
f' {response.status_code}. response: {response.text}')
return err_resp(user_msg)
LOGGER.info('(3) Download complete. Save file')
# -----------------------------------------
# Save the downloaded file
# -----------------------------------------
save_info = DatamartJobUtilISI.save_datamart_file(\
dest_filepath,
response)
if not save_info.success:
return err_resp(save_info.err_msg)
save_info = save_info.result_obj
# -----------------------------------------
# Retrieve preview rows and return response
# -----------------------------------------
LOGGER.info('(4) File saved')
# preview rows
#
preview_info = read_file_rows(save_info[dm_static.KEY_DATA_PATH],
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
# Format/return reponse
#
info_dict = DatamartJobUtilISI.format_materialize_response(
datamart_id, dm_static.DATAMART_ISI_NAME,
save_info[dm_static.KEY_DATA_PATH], preview_info, **save_info)
return ok_resp(info_dict)
def datamart_search(query_dict=None, dataset_path=None, **kwargs):
"""Search the ISI datamart"""
if query_dict is None and dataset_path is None:
return err_resp('Either a query or dataset path must be supplied.')
if query_dict is not None and not isinstance(query_dict, dict):
user_msg = ('There is something wrong with the search parameters.'
' Please try again. (expected a dictionary)')
return err_resp(user_msg)
search_url = get_isi_url() + '/search'
# --------------------------------
# Behavioral logging
# --------------------------------
if 'user' in kwargs:
log_data = dict(feature_id=f'POST|{search_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_SEARCH,
path=search_url)
LogEntryMaker.create_datamart_entry(kwargs['user'], log_data)
# --------------------------------
# --------------------------------
# Query the datamart
# --------------------------------
query_json = None
if query_dict:
formatted_json_info = json_dumps(query_dict)
if not formatted_json_info.success:
return err_resp('Failed to convert query to JSON. %s' % \
formatted_json_info.err_msg)
query_json = formatted_json_info.result_obj
print(f'formatted query: {query_json}')
if dataset_path:
limit = kwargs.get('limit', 20)
if not isinstance(limit, int):
user_msg = ('The results limit must be an'
' integer (datamart_search)')
return err_resp(user_msg)
if not USE_CACHED_SEARCH:
try:
with open(dataset_path, 'rb') as dataset_p:
try:
response = requests.post(
search_url,
params={'max_return_docs': limit},
json={'query_json': query_json},
files={'data': dataset_p},
verify=False,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp(
'Request timed out. responded with: %s' %
err_obj)
except IOError as err_obj:
user_msg = (f'Failed to search with the dataset file.'
f' Technical: {err_obj}')
return err_resp(user_msg)
else:
raise NotImplementedError(
'Augmentations on results without a dataset path are not implemented by ISI.'
)
if not USE_CACHED_SEARCH:
if response.status_code != 200:
return err_resp(response['reason'])
response_json = response.json()
if response_json['code'] != "0000":
return err_resp(response_json['message'])
else:
import json
print('loading file')
response_json = json.load(
open('/datamart_endpoints/cached_isi_search_response.json',
'r'))
json_results = response_json['search_results']['results']
#num_datasets = len(response['data'])
#print('num_datasets', num_datasets)
#print('iterating through....')
sorted_data = sorted(
json_results, #response['data'],
key=lambda k: k['score'],
reverse=True)
#print([ds['score'] for ds in sorted_data])
return ok_resp(sorted_data[:limit])
def search_with_dataset(dataset_path, query=None, **kwargs):
"""Search the datamart using a dataset"""
if not isfile(dataset_path):
user_msg = ('The dataset file could not be found.')
return err_resp(user_msg)
search_url = get_isi_url() + '/search'
# --------------------------------
# Behavioral logging
# --------------------------------
if 'user_workspace' in kwargs:
log_data = dict(feature_id=f'POST|by-dataset|{search_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_SEARCH,
path=search_url)
LogEntryMaker.create_datamart_entry(kwargs['user_workspace'],
log_data)
# --------------------------------
# --------------------------------
# Query the datamart
# --------------------------------
query_json = None
if query:
formatted_json_info = json_dumps(query)
if not formatted_json_info.success:
return err_resp('Failed to convert query to JSON. %s' % \
formatted_json_info.err_msg)
query_json = formatted_json_info.result_obj
print(f'formatted query: {query_json}')
limit = kwargs.get('limit', 20)
if not isinstance(limit, int):
user_msg = ('The results limit must be an'
' integer (datamart_search)')
return err_resp(user_msg)
if not USE_CACHED_SEARCH:
try:
with open(dataset_path, 'rb') as dataset_p:
try:
response = requests.post(
search_url,
params={'max_return_docs': limit},
json={'query_json': query_json},
files={'data': dataset_p},
verify=False,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp(
'Request timed out. responded with: %s' % err_obj)
except IOError as err_obj:
user_msg = (f'Failed to search with the dataset file.'
f' Technical: {err_obj}')
return err_resp(user_msg)
if response.status_code != 200:
print(str(response))
print(response.text)
return err_resp(('ISI Datamart internal server error.'
' status_code: %s') % response.status_code)
response_json = response.json()
else:
import json
print('loading file')
response_json = json.load(
open('/datamart_endpoints/cached_isi_search_response.json',
'r'))
#print('response_json', response_json)
if not 'results' in response_json:
return err_resp('No datasets found. (%s)' % \
(get_timestamp_string_readable(time_only=True),))
json_results = response_json['results']
print('num results: ', len(json_results))
return ok_resp(json_results)
def datamart_augment(user_workspace, dataset_path, task_data, **kwargs):
"""Augment the file via the NYU API"""
if not isinstance(user_workspace, UserWorkspace):
return err_resp('user_workspace must be a UserWorkspace')
# Make sure the soure file exists
#
if not isfile(dataset_path):
user_msg = f'Original data file not found: {dataset_path}'
return err_resp(user_msg)
# Make sure the NYU datamart id is in the task_data
#
if not dm_static.KEY_NYU_DATAMART_ID in task_data:
user_msg = (f'"task_data" did not contain'
f' "{dm_static.KEY_NYU_DATAMART_ID}" key')
return err_resp(user_msg)
# used for folder naming
#
datamart_id = task_data[dm_static.KEY_NYU_DATAMART_ID]
# ---------------------------------
# The augment url...
# ---------------------------------
augment_url = f"{ get_nyu_url() }/augment"
# ----------------------------
# Behavioral logging
# ----------------------------
log_data = dict(feature_id=f'POST|{augment_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_AUGMENT,
path=augment_url)
LogEntryMaker.create_datamart_entry(user_workspace, log_data)
# ----------------------------
# ---------------------------------
# Ready the query parameters
# ---------------------------------
data_params = dict(data=open(dataset_path, 'rb'),
task=json.dumps(task_data))
# ---------------------------------
# Make the augment request
# ---------------------------------
try:
response = requests.post(augment_url,
files=data_params,
stream=True,
allow_redirects=True,
verify=False,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' % err_obj)
# Any errors?
#
if response.status_code != 200:
user_msg = (f'NYU Datamart internal server error. Status code:'
f' "{response.status_code}".'
f' <hr />Technical: {response.content}')
# print(response.content)
return err_resp(user_msg)
# Write the augmented file
#
dest_folderpath_info = DatamartJobUtilNYU.get_output_folderpath(\
user_workspace,
datamart_id,
dir_type=dm_static.KEY_AUGMENT)
if not dest_folderpath_info.success:
return err_resp(dest_folderpath_info.err_msg)
augment_folderpath = dest_folderpath_info.result_obj
# Set the output file
#
dest_filepath = join(augment_folderpath, 'tables', 'learningData.csv')
save_info = DatamartJobUtilNYU.save_datamart_file(\
augment_folderpath,
response,
expected_filepath=dest_filepath)
if not save_info.success:
return err_resp(save_info.err_msg)
save_info = save_info.result_obj
# -----------------------------------------
# Retrieve preview rows and return response
# -----------------------------------------
# preview rows
#
preview_info = read_file_rows(save_info[dm_static.KEY_DATA_PATH],
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
# Format/return reponse
#
info_dict = DatamartJobUtilNYU.format_materialize_response(\
datamart_id,
dm_static.DATAMART_NYU_NAME,
save_info[dm_static.KEY_DATA_PATH],
preview_info,
**save_info)
return ok_resp(info_dict)
def datamart_materialize(user_workspace, search_result):
"""Materialize an NYU dataset!"""
LOGGER.info('-- atttempt to materialize NYU dataset --')
if not isinstance(user_workspace, UserWorkspace):
return err_resp('user_workspace must be a UserWorkspace')
if not isinstance(search_result, dict):
return err_resp('search_result must be a python dictionary')
print('\nsearch_result', search_result)
print('\nsearch_result.keys()', search_result.keys())
if not dm_static.KEY_NYU_DATAMART_ID in search_result:
user_msg = (f'"search_result" did not contain'
f' "{dm_static.KEY_NYU_DATAMART_ID}" key')
return err_resp(user_msg)
# -----------------------------------------
# Build the folder path where the .zip will
# be unbundled
# -----------------------------------------
LOGGER.info('(1) build path')
datamart_id = search_result[dm_static.KEY_NYU_DATAMART_ID]
dest_folderpath_info = DatamartJobUtilNYU.get_output_folderpath(\
user_workspace,
datamart_id,
dir_type=dm_static.KEY_MATERIALIZE)
# Failed to get/create the output folder
#
if not dest_folderpath_info.success:
return err_resp(dest_folderpath_info.err_msg)
# Set the output folder
#
dest_folderpath = dest_folderpath_info.result_obj
# Set the output file path
#
dest_filepath = join(dest_folderpath, 'tables', 'learningData.csv')
LOGGER.info('(2) Download file')
# -----------------------------------------
# Has the file already been downloaded?
# -----------------------------------------
print('dest_filepath', dest_filepath)
LOGGER.info('(2a) Has the file already been downloaded?')
if isfile(dest_filepath):
LOGGER.info('Yes, already downloaded')
# Get preview rows
#
preview_info = read_file_rows(dest_filepath,
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
info_dict = DatamartJobUtilNYU.format_materialize_response(\
datamart_id, dm_static.DATAMART_NYU_NAME,
dest_filepath, preview_info)
return ok_resp(info_dict)
# -----------------------------------------
# Download the file
# -----------------------------------------
LOGGER.info('(2b) File not yet downloaded. Attempting download')
if not 'id' in search_result:
user_msg = f'search_result did not contain the key "id"'
return err_resp(user_msg)
download_url = (f'{get_nyu_url()}/download/'
f'{search_result[dm_static.KEY_NYU_DATAMART_ID]}')
# ----------------------------
# Behavioral logging
# ----------------------------
log_data = dict(feature_id=f'GET|{download_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_DOWNLOAD,
path=download_url)
LogEntryMaker.create_datamart_entry(user_workspace, log_data)
# ----------------------------
# Download the file!
# ----------------------------
try:
response = requests.get(\
download_url,
params={'format': 'd3m'},
verify=False,
stream=True,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' % err_obj)
if response.status_code != 200:
user_msg = (f'Materialize failed. Status code:'
f' {response.status_code}. response: {response.text}')
return err_resp(user_msg)
save_info = DatamartJobUtilNYU.save_datamart_file(\
dest_folderpath,
response,
expected_filepath=dest_filepath)
if not save_info.success:
return err_resp(save_info.err_msg)
save_info = save_info.result_obj
# ----------------------------
# Get preview rows
# ----------------------------
preview_info = read_file_rows(save_info[dm_static.KEY_DATA_PATH],
dm_static.NUM_PREVIEW_ROWS)
if not preview_info.success:
user_msg = (f'Failed to retrieve preview rows.'
f' {preview_info.err_msg}')
return err_resp(user_msg)
info_dict = DatamartJobUtilNYU.format_materialize_response( \
datamart_id,
dm_static.DATAMART_NYU_NAME,
dest_filepath,
preview_info,
**save_info)
return ok_resp(info_dict)
def datamart_search(query_dict=None, dataset_path=None, **kwargs):
"""Search the NYU datamart"""
if query_dict is None and dataset_path is None:
return err_resp('Either a query or dataset path must be supplied.')
if query_dict is not None and not isinstance(query_dict, dict):
user_msg = ('There is something wrong with the search parameters.'
' Please try again. (expected a dictionary)')
return err_resp(user_msg)
search_url = get_nyu_url() + '/search'
# --------------------------------
# Behavioral logging
# --------------------------------
if 'user' in kwargs:
log_data = dict(feature_id=f'POST|{search_url}',
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_SEARCH,
path=search_url)
LogEntryMaker.create_datamart_entry(kwargs['user'], log_data)
# --------------------------------
# --------------------------------
# Query the datamart
# --------------------------------
if dataset_path:
try:
with open(dataset_path, 'rb') as dataset_p:
try:
response = requests.post(
search_url,
json=query_dict,
files=dict(data=dataset_p),
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp(
'Request timed out. responded with: %s' % err_obj)
except IOError as err_obj:
user_msg = (f'Failed to search with the dataset file.'
f' Technical: {err_obj}')
return err_resp(user_msg)
else:
try:
response = requests.post(
search_url,
json=query_dict,
stream=True,
timeout=settings.DATAMART_LONG_TIMEOUT)
except requests.exceptions.Timeout as err_obj:
return err_resp('Request timed out. responded with: %s' %
err_obj)
if response.status_code != 200:
print(str(response))
print(response.text)
return err_resp(('NYU Datamart internal server error.'
' status_code: %s') % response.status_code)
json_results = response.json()['results']
if not json_results:
return err_resp('No datasets found. (%s)' % \
(get_timestamp_string_readable(time_only=True),))
# print('num results: ', len(json_results))
return ok_resp(json_results)