def retrieve_examine_workspace(self):
"""Was the workspace set. If not, see if it can be retrieved"""
if self.has_error():
return
# If the workspace is set, make sure it's the right type
#
if self.user_workspace:
if not isinstance(self.user_workspace, UserWorkspace):
self.add_err_msg(('user_workspace must be a UserWorkspace'
' object or None'))
return
# Looks like we have a user workspace!
# If there's not a user workspace, see if we can get it
# via the request_obj
#
if self.request_obj and not self.user_workspace:
ws_info = ws_util.get_latest_user_workspace(self.request_obj)
if ws_info.success:
# Got one!
self.user_workspace = ws_info.result_obj
# Also use the d3m_config from it
# This overrides a manually set d3m_config
self.d3m_config = self.user_workspace.d3m_config
# See if there's a default d3m_config
#
if not self.d3m_config:
self.d3m_config = get_latest_d3m_config()
def view_pebbles_home(request):
"""Serve up the workspace, the current home page.
Include global js settings"""
app_config = AppConfiguration.get_config()
if app_config is None:
return HttpResponseRedirect(reverse('view_no_domain_config_error'))
# Is this D3M Mode? If so, make sure:
# (1) there is D3M config information
# (2) user is logged in
#
if app_config.is_d3m_domain():
# (1) Is there a valid D3M config?
d3m_config = get_latest_d3m_config()
if not d3m_config:
return HttpResponseRedirect(\
reverse('view_d3m_config_error'))
# (2) Is the user authenticated?
if not request.user.is_authenticated():
return HttpResponseRedirect(\
reverse('login'))
session_key = get_session_key(request)
dinfo = dict(title='TwoRavens',
session_key=session_key,
app_config=app_config.convert_to_dict())
return render(request, 'index.html', dinfo)
def view_get_problem_data_info(request, d3m_config_id=None):
"""Get info on train data and target files, if they exist"""
if d3m_config_id is None:
d3m_config = get_latest_d3m_config()
else:
d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first()
if d3m_config is None:
user_msg = 'Config not found! (view_get_problem_data_info)'
return JsonResponse(get_json_error(user_msg))
is_pretty = request.GET.get('pretty', False)
train_data_info = get_train_data_info(d3m_config)
if not train_data_info.success:
resp_dict = get_json_error(train_data_info.err_msg)
else:
resp_dict = get_json_success('It worked',
data=train_data_info.result_obj)
if is_pretty is not False: # return this as a formatted string?
config_str = '<pre>%s<pre>' % \
(json.dumps(resp_dict,
indent=4))
return HttpResponse(config_str)
return JsonResponse(resp_dict)
def view_d3m_config_error(request):
"""Show this when the app is in D3M mode
but there's no config info available"""
# Only show this if:
# (a) in D3M mode
#
app_config = AppConfiguration.get_config()
if app_config is None:
return HttpResponseRedirect(reverse('view_no_domain_config_error'))
if not app_config.is_d3m_domain():
return HttpResponseRedirect(reverse('home'))
# and (b) not D3M config info is in the db
#
d3m_config_info = get_latest_d3m_config()
#get_latest_d3m_user_config_by_request(request)
if d3m_config_info:
return HttpResponseRedirect(reverse('home'))
dinfo = dict(title='D3M configuration error')
return render(request,
'content_pages/no_config_error.html',
dinfo)
def view_get_problem_data_info(request, d3m_config_id=None):
"""Get info on train data and target files, if they exist"""
if d3m_config_id is None:
d3m_config = get_latest_d3m_config()
else:
d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first()
if d3m_config is None:
raise Http404('Config not found!')
is_pretty = request.GET.get('pretty', False)
info_dict, err_msg = get_train_data_info(d3m_config)
if err_msg:
resp_dict = dict(success=False, message=err_msg)
else:
resp_dict = dict(success=True, data=info_dict)
if is_pretty is not False: # return this as a formatted string?
config_str = '<pre>%s<pre>' % \
(json.dumps(resp_dict,
indent=4))
return HttpResponse(config_str)
return JsonResponse(resp_dict)
def delete_user_workspaces(user):
"""Used to reset UserWorkspace objects based on a user/problem"""
if not isinstance(user, User):
return err_resp('user must be a "User" object, not: "%s"' % user)
d3m_config = get_latest_d3m_config()
if not d3m_config:
return err_resp('No default D3MConfiguration set.')
params = dict(user=user)
workspaces = UserWorkspace.objects.filter(**params)
cnt = workspaces.count()
workspaces.delete()
return ok_resp('Workspaces cleared. %d deleted' % cnt)
def view_d3m_details_json_latest(request, as_eval_dict=False):
"""Return the "latest" D3m configuration as JSON.
"latest" may be most recently added or a "default"
of some kind"""
is_pretty = request.GET.get('pretty', False)
# Is there a default config?
d3m_config = get_latest_d3m_config()
if not d3m_config:
raise Http404('no configs available')
if is_pretty is not False: # return this as a formatted string?
config_str = '<pre>%s<pre>' % \
(json.dumps(d3m_config.to_dict(as_eval_dict),
indent=4))
return HttpResponse(config_str)
# return as JSON!
return JsonResponse(d3m_config.to_dict(as_eval_dict))
def view_get_config_file(request, config_key, d3m_config_id=None):
"""Get contents of a file specified in the config"""
if not config_key in D3M_FILE_ATTRIBUTES:
user_msg = (f'Config key "{config_key}" not found!'
f' (view_get_config_file)')
return JsonResponse(get_json_error(user_msg))
if d3m_config_id is None:
d3m_config = get_latest_d3m_config()
else:
d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first()
if d3m_config is None:
user_msg = 'Config not found!'
return JsonResponse(get_json_error(user_msg))
# Make sure the config has a value.
# For example the D3MPROBLEMPATH may be blank
#
if not getattr(d3m_config, config_key):
user_msg = f'Sorry! The config does not have a "{config_key}" value!'
return JsonResponse(get_json_error(user_msg))
filepath_info = get_d3m_filepath(d3m_config, config_key)
if not filepath_info.success:
user_msg = f'{filepath_info.err_msg} (view_get_config_file)'
return JsonResponse(get_json_error(user_msg))
# Relatively small files...
# response = FileResponse(open(filepath_info.result_obj, 'rb'))
# return response
fcontent = open(filepath_info.result_obj, 'r').read()
json_info = json_loads(fcontent)
if not json_info.success:
user_msg = f'{json_info.err_msg} (view_get_config_file)'
return JsonResponse(get_json_error(user_msg))
return JsonResponse(get_json_success(\
'Success!',
data=json_info.result_obj))
def view_get_config_file(request, config_key, d3m_config_id=None):
"""Get contents of a file specified in the config"""
if not config_key in D3M_FILE_ATTRIBUTES:
raise Http404('config_key not found!')
if d3m_config_id is None:
d3m_config = get_latest_d3m_config()
else:
d3m_config = D3MConfiguration.objects.filter(id=d3m_config_id).first()
if d3m_config is None:
raise Http404('Config not found!')
filepath, err_msg_or_None = get_d3m_filepath(d3m_config, config_key)
if err_msg_or_None is not None:
return JsonResponse(dict(success=False, message=err_msg_or_None))
response = FileResponse(open(filepath, 'rb'))
return response
def get_user_workspaces(user, create_if_not_found=True):
"""Get UserWorkspace list based on the active D3M config
- This is trickier now.
- An augmented workspace may have the active D3M config as
the original_workspace
"""
if not isinstance(user, User):
return err_resp('user must be a "User" object, not: "%s"' % user)
d3m_config = get_latest_d3m_config()
print('d3m_config', d3m_config)
if not d3m_config:
return err_resp('No default D3MConfiguration set.')
params = dict(user=user)
params = get_default_workspace_params(**params)
params['d3m_config'] = d3m_config
params_original_ws = dict(original_workspace__d3m_config=d3m_config)
workspaces = UserWorkspace.objects.filter(\
Q(**params) |
Q(**params_original_ws))
print('workspaces', workspaces)
# Make sure the list has a current workspace
#
has_current_workspace = [
ws for ws in workspaces if ws.is_current_workspace
]
if (not has_current_workspace) or (workspaces.count() == 0):
if create_if_not_found:
ws_info = create_new_user_workspace(user, d3m_config)
if not ws_info.success:
return err_resp('%s (get_user_workspaces)' %\
(ws_info.err_msg))
return ok_resp([ws_info.result_obj])
return err_resp('No workspaces found for the User and default config')
return ok_resp(list(workspaces))
def get_latest_d3m_user_config(user, create_if_not_found=True, **kwargs):
"""Find the lastest UserWorkspace and return the attached d3m_config
return_full_workspace = True : return UserWorkspace instead of D3MConfiguration
"""
if not isinstance(user, User):
return err_resp('user must be a "User" object, not: "%s"' % user)
return_full_workspace = kwargs.get('return_full_workspace', False)
d3m_config = get_latest_d3m_config()
if not d3m_config:
return err_resp('No default D3MConfiguration set.')
params = dict(
user=user,
is_current_workspace=True,
)
params = get_default_workspace_params(**params)
latest_workspace = UserWorkspace.objects.filter(**params).first()
if latest_workspace:
if return_full_workspace:
return ok_resp(latest_workspace)
return ok_resp(latest_workspace.d3m_config)
if create_if_not_found:
ws_info = create_new_user_workspace(user, d3m_config)
if not ws_info.success:
return err_resp('%s (get_latest_d3m_user_config)' %\
(ws_info.err_msg))
new_workspace = ws_info.result_obj
if return_full_workspace:
return ok_resp(new_workspace)
return ok_resp(new_workspace.d3m_config)
return err_resp('No workspace found for the User and default config')
def execute_pipeline(info_str=None):
"""Ask a TA2 to ListPipelines via gRPC
This call is a bit different b/c it writes part of the data to a file
and places that file uri into the original request
Success: (updated request str, grpc json response)
Failure: (None, error message)
"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for PipelineListResult is None'
return None, get_failed_precondition_response(err_msg)
if info_str.find(VAL_DATA_URI) == -1:
err_msg = ('Expected to see place holder for file uri.'
' Placeholder is "%s"') % VAL_DATA_URI
return None, get_failed_precondition_response(err_msg)
d3m_config = get_latest_d3m_config()
if not d3m_config:
err_msg = ('The D3M configuration is not available.'
' Therefore, there is no "temp_storage_root" directory to'
' write the data.')
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return None, get_failed_precondition_response(err_msg)
if not KEY_DATA in info_dict:
err_msg = ('The JSON request did not contain a "%s" key.') % KEY_DATA
return None, get_failed_precondition_response(err_msg)
file_uri, err_msg = write_data_for_execute_pipeline(
d3m_config, info_dict[KEY_DATA])
if err_msg is not None:
return None, get_failed_precondition_response(err_msg)
# Reformat the original content
#
# (1) remove the data key
if KEY_DATA in info_dict:
del info_dict[KEY_DATA]
# (2) convert it back to a JSON string
info_str = json.dumps(info_dict)
# (3) replace the VAL_DATA_URI with the file_uri
info_str_formatted = info_str.replace(VAL_DATA_URI, file_uri)
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str_formatted, core_pb2.PipelineExecuteRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return None, get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
#return info_str_formatted,\
# get_grpc_test_json('test_responses/execute_results_1pipe_ok.json',
# dict())
#---
template_info = get_predict_file_info_dict()
template_str = get_grpc_test_json(
'test_responses/execute_results_1pipe_ok.json', template_info)
# These next lines embed file uri content into the JSON
embed_util = FileEmbedUtil(template_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
test_note = ('Test. An actual result would be the test JSON with'
' the "data" section removed and DATA_URI replaced'
' with a file path to where the "data" section was'
' written.')
return json.dumps(dict(note=test_note)), embed_util.get_final_results()
#---
#return info_str_formatted,\
# get_grpc_test_json('test_responses/execute_results_1pipe_ok.json',
# dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.ExecutePipeline(req)
except Exception as ex:
return None, get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it on
# --------------------------------
results = map(MessageToJson, reply)
result_str = '[' + ', '.join(results) + ']'
embed_util = FileEmbedUtil(result_str)
if embed_util.has_error:
return get_failed_precondition_response(embed_util.error_message)
return info_str_formatted, embed_util.get_final_results()
def export_pipeline(info_str=None, call_entry=None):
"""Ask a TA2 to ExportPipeline via gRPC"""
if info_str is None:
info_str = get_test_info_str()
if info_str is None:
err_msg = 'UI Str for ExportPipeline is None'
return get_failed_precondition_response(err_msg)
if info_str.find(VAL_EXECUTABLE_URI) == -1:
err_msg = ('Expected to see place holder for executable uri.'
' Placeholder is "%s"') % VAL_EXECUTABLE_URI
return None, get_failed_precondition_response(err_msg)
d3m_config = get_latest_d3m_config()
if not d3m_config:
err_msg = ('The D3M configuration is not available.'
' Therefore, there is no "executables_root" directory to'
' write the data.')
return None, get_failed_precondition_response(err_msg)
# --------------------------------
# Is this valid JSON?
# --------------------------------
try:
info_dict = json.loads(info_str, object_pairs_hook=OrderedDict)
except json.decoder.JSONDecodeError as err_obj:
err_msg = 'Failed to convert UI Str to JSON: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
# --------------------------------
# Construct and set a write directory for the executable
# --------------------------------
# get the pipeline id
pipeline_id, err_msg = get_pipeline_id(info_dict)
if err_msg:
return get_failed_precondition_response(err_msg)
# dir = d3m_config.executables_root + pipeline_id
executable_write_dir = join('file://%s' % d3m_config.executables_root,
pipeline_id)
# update the dict + info_str
info_dict[KEY_PIPELINE_EXEC_URI] = executable_write_dir
if KEY_PIPELINE_EXEC_URI_FROM_UI in info_dict:
del info_dict[KEY_PIPELINE_EXEC_URI_FROM_UI]
try:
info_str = json.dumps(info_dict)
except TypeError as ex_obj:
err_msg = 'Failed to PipelineExportRequest info to JSON: %s' % ex_obj
return get_failed_precondition_response(err_msg)
#print('info_str', info_str)
if call_entry:
call_entry.request_msg = info_str
# --------------------------------
# convert the JSON string to a gRPC request
# --------------------------------
try:
req = Parse(info_str, core_pb2.PipelineExportRequest())
except ParseError as err_obj:
err_msg = 'Failed to convert JSON to gRPC: %s' % (err_obj)
return get_failed_precondition_response(err_msg)
if settings.TA2_STATIC_TEST_MODE:
return get_grpc_test_json('test_responses/export_pipeline_ok.json',
dict())
# --------------------------------
# Get the connection, return an error if there are channel issues
# --------------------------------
core_stub, err_msg = TA2Connection.get_grpc_stub()
if err_msg:
return get_failed_precondition_response(err_msg)
# --------------------------------
# Send the gRPC request - returns a stream
# --------------------------------
try:
reply = core_stub.ExportPipeline(req)
except Exception as ex:
return get_failed_precondition_response(str(ex))
# --------------------------------
# Convert the reply to JSON and send it back
# --------------------------------
return MessageToJson(reply)
def get_ta2_run_command(self):
"""Return the Docker run command"""
if self.has_error():
return self.error_message
ta2_info = self.get_ta2_info()
if not ta2_info:
return
# The new config has already been sent, so get env variables related to it
d3m_config = get_latest_d3m_config()
if not d3m_config:
print('d3m_config not found! (get_ta2_run_command)')
return
env_str = d3m_config.get_docker_env_settings()
if env_str is None:
env_str = ''
image_name = ta2_info[1]
additional_options = ta2_info[2]
print('INPUT', self.data_input_dir)
print('OUTPUT', self.data_output_dir)
# sudo is not necessary if just trying to run a local image?
# If trying to download and run into an issue, then use sudo
docker_cmd = ('docker run --rm'
' --name ta2_server'
' {4}'
' {2}'
' -v {0}:/input'
' -v {1}:/output'
' -v /ravens_volume:/ravens_volume'
' {3}'
'').format(self.data_input_dir,
self.data_output_dir,
additional_options,
image_name,
env_str)
print('docker_cmd', docker_cmd)
xdocker_cmd = ('docker run --rm'
' --name ta2_server'
' -e D3MTIMEOUT=60'
' -e D3MINPUTDIR=/input'
' -e D3MOUTPUTDIR=/output'
' -e D3MRUN=ta2ta3'
' {2}'
' -v {0}:/input'
' -v {1}:/output'
' -v /ravens_volume:/ravens_volume'
' {3}'
'').format(self.data_input_dir,
self.data_output_dir,
additional_options,
image_name)
return docker_cmd
def view_pebbles_home(request):
"""Serve up the workspace, the current home page.
Include global js settings"""
if not request.user.is_authenticated:
return HttpResponseRedirect(reverse('login'))
app_config = AppConfiguration.get_config()
if app_config is None:
return HttpResponseRedirect(reverse('view_no_domain_config_error'))
user_info = get_authenticated_user(request)
if not user_info.success:
return JsonResponse(get_json_error(user_info.err_msg))
user = user_info.result_obj
# Is this D3M Mode? If so, make sure:
# (1) there is D3M config information
# (2) user is logged in
#
if app_config.is_d3m_domain():
# (1) Is there a valid D3M config?
d3m_config_info = get_latest_d3m_config()
if not d3m_config_info:
return HttpResponseRedirect(reverse('view_list_dataset_choices_html'))
# return HttpResponseRedirect(reverse('view_d3m_config_error'))
session_key = get_session_key(request)
else:
session_key = '(event-data-no-session-key)'
dinfo = dict(title='TwoRavens',
session_key=session_key,
DEBUG=settings.DEBUG,
ALLOW_SOCIAL_AUTH=settings.ALLOW_SOCIAL_AUTH,
CSRF_COOKIE_NAME=settings.CSRF_COOKIE_NAME,
app_config=app_config.convert_to_dict(),
#
TA2_STATIC_TEST_MODE=settings.TA2_STATIC_TEST_MODE,
TA2_TEST_SERVER_URL=settings.TA2_TEST_SERVER_URL,
#
TA2_D3M_SOLVER_ENABLED=pybool_to_js(settings.TA2_D3M_SOLVER_ENABLED),
TA2_WRAPPED_SOLVERS=settings.TA2_WRAPPED_SOLVERS,
#
TA3_GRPC_USER_AGENT=settings.TA3_GRPC_USER_AGENT, TA3TA2_API_VERSION=TA3TA2Util.get_api_version(),
DISPLAY_DATAMART_UI=settings.DISPLAY_DATAMART_UI,
WEBSOCKET_PREFIX=settings.WEBSOCKET_PREFIX)
log_data = dict(session_key=session_key,
feature_id=bl_static.FID_START_RAVENS_PEBBLES_PAGE,
activity_l1=bl_static.L1_DATA_PREPARATION,
activity_l2=bl_static.L2_DATA_OPEN)
LogEntryMaker.create_system_entry(user, log_data)
#print('-' * 40)
#print(dinfo['app_config'])
return render(request,
'index.html',
dinfo)
