def getMLServices(credentials):
proj_credentials = WMLUtil.checkCredentials(credentials)
currentproject = Project(
None,
project_id=proj_credentials['project_id'],
project_access_token=proj_credentials['access_token'])
compute_entities = currentproject.get_metadata()['entity']['compute']
ml_services = [
entity for entity in compute_entities
if entity['type'] == 'machine_learning'
]
return ml_services
def add_file_path_as_data_asset_cpd25(file_path: str, asset_name: str = None) -> None:
"""Add a data file to the Watson Studio project.
Applies to CPDv2.5. Works for any file. Allows the file to be viewed and downloaded from Watson Studio UI.
Needs to be called after the file has been saved regularly in the file system.
Typically, that would be in `/project_data/data_asset/`.
Ensures the file is visible in the Data Assets of the Watson Studio UI.
Args:
file_path (str): full file path, including the file name and extension
asset_name (str): name of data asset. Default is None. If None, the asset_name will be extracted from the file_path.
Usage::
# Write some file as an example:
file_path = '/project_data/data_asset/myfile.csv'
with open(file_path, 'w+') as f:
f.write("Hello World")
# Add file as a data asset:
add_file_as_data_asset_cpd25(file_path)
Beware that the same data now exists in 2 different places:
* In the Cloud Object Storage (COS)
* As a file in `/project_data/data_asset/`
Changing any of the 2 independently can cause inconsistencies.
"""
if asset_name is None:
asset_name = os.path.basename(file_path)
with open(file_path, 'rb') as f:
from project_lib import Project
project = Project.access()
project.save_data(file_name=asset_name, data=f, overwrite=True)
def add_file_path_as_data_asset_wsc(file_path: str, asset_name: str = None, project=None) -> None:
"""Add a data file to the Watson Studio project.
Applies to WS Cloud and CPDv2.5. Works for any file. Allows the file to be viewed and downloaded from Watson Studio UI.
Needs to be called after the file has been saved regularly in the file system.
Typically, that would be in:
* CPDv2.5: `/project_data/data_asset/`
* WS Cloud: `/home/dsxuser/work/`, or `os.environ['PWD']`, or `./`, or no path
Ensures the file is visible in the Data Assets of the Watson Studio UI.
Args:
project (project_lib.Project): required for WS Cloud
file_path (str): full file path, including the file name and extension
asset_name (str): name of data asset. Default is None. If None, the asset_name will be extracted from the file_path.
Usage::
# Write some file as an example:
file_path = '/project_data/data_asset/myfile.csv'
with open(file_path, 'w+') as f:
f.write("Hello World")
# Add file as a data asset:
add_file_as_data_asset_cpd25(file_path)
"""
if project is None:
from project_lib import Project
project = Project.access()
if asset_name is None:
asset_name = os.path.basename(file_path)
with open(file_path, 'rb') as f:
project.save_data(file_name=asset_name, data=f, overwrite=True)
def add_file_as_data_asset_cpd25(file_name: str) -> None:
"""Adds a file located in `/project_data/data_asset/` as a Data Asset to the Watson Studio project.
So that it appears in the UI and can be exported.
"""
file_path = os.path.join('/project_data/data_asset/', file_name)
with open(file_path, 'rb') as f:
from project_lib import Project
project = Project.access()
project.save_data(file_name=file_name, data=f, overwrite=True)
def write_do_model_to_file(self, do_model_name: str, response: requests.Response) -> str:
import os
export_file_name = DOModelExporter._get_export_file_name(do_model_name)
file_path = os.path.join(self.export_directory, export_file_name)
with open(file_path, 'wb') as f:
f.write(response.content)
# If CPD2.5 then also add as asset:
if self.export_directory == '/project_data/data_asset/':
print("Adding export file to Data Assets of this project.")
with open(file_path, 'rb') as f:
from project_lib import Project
project = Project.access()
project.save_data(file_name=export_file_name, data=f, overwrite=True)
return file_path
def write_data_asset_as_file_cpd25(asset_name: str, path: str = '/project_data/data_asset/') -> str:
"""Writes a named data asset to file.
Assumes a data asset with `asset_name` exists.
Makes the file accessible for things like:
* Load from disk
* Pip install
* Module import
Args:
asset_name (str): name of the asset
path (str, Optional): Default is '/project_data/data_asset/'. Use path='' for current directory.
"""
from project_lib import Project
project = Project.access()
file_path = os.path.join(path, asset_name)
with open(file_path, "wb") as f:
f.write(project.get_file(asset_name).read())
return file_path
def add_data_file_to_project_s(file_path: str,
file_name: Optional[str] = None) -> None:
"""Add a data file to the Watson Studio project.
Applies to CP4Dv2.5.
Needs to be called after the file has been saved regularly in the file system in `/project_data/data_asset/`.
Ensures the file is visible in the Data Assets of the Watson Studio UI.
Args:
file_path (str): full file path, including the file name and extension
file_name (str): name of data asset. Default is None. If None, the file-name will be extracted from the file_path.
"""
# Add to Project
if file_name is None:
file_name = os.path.basename(file_path)
with open(file_path, 'rb') as f:
from project_lib import Project
project = Project.access()
project.save_data(file_name=file_name, data=f, overwrite=True)
def get_dd_client(self):
"""Return the Client managing the DO scenario.
Returns: new dd_scenario.Client
"""
from dd_scenario import Client
if self.project is not None:
pc = self.project.project_context
return Client(pc=pc)
elif (self.project_id is not None) and (self.project_access_token is not None):
# When in WS Cloud:
from project_lib import Project
# The do_optimization project token is an authorization token used to access project resources like data sources, connections, and used by platform APIs.
project = Project(project_id=self.project_id,
project_access_token=self.project_access_token)
pc = project.project_context
return Client(pc=pc)
else:
# In WSL/CPD:
return Client()
def write_data_asset_as_file_wsc(asset_name: str, path: str = '/home/dsxuser/work/', project=None) -> str:
"""Writes a named data asset to file (for WS Cloud).
Assumes a data asset with `asset_name` exists.
Makes the file accessible for things like:
* Load from disk
* Pip install
* Module import
Args:
asset_name (str): name of the asset
path (str, Optional): Default (for WS Cloud) is '/home/dsxuser/work/'. Use path='' for current directory.
project (project_lib.Project): required for WS Cloud. For CPD, leave as None.
"""
if project is None:
from project_lib import Project
project = Project.access()
file_path = os.path.join(path, asset_name)
with open(file_path, "wb") as f:
f.write(project.get_file(asset_name).read())
return file_path
def get_project_space(credentials):
"""Returns the notebooks project space GUID.
Argument:
credentials -- the credentials to be used to connect to WML
Call it only in the notebook where the topology is created, not at Streams runtime.
Models and deployments are placed in projects space if no other space is given at
their creation time.
The space GUID is needed to instantiate a WMLOnlineScoring object.
"""
from project_lib import Project
wml_client = APIClient(copy.copy(credentials))
spaces = wml_client.spaces.get_details()["resources"]
project = Project.access()
project_guid = project.get_metadata()["metadata"]["guid"]
# get the space associated with the project
project_space=None
for space in spaces:
for tag in space["entity"]["tags"]:
if tag["value"]=="dsx-project."+project_guid:
project_space = space["metadata"]["id"]
return project_space
# coding: utf-8 # In[1]: # @hidden_cell # The project token is an authorization token that is used to access project resources like data sources, connections, and used by platform APIs. from project_lib import Project project = Project(project_id='bfe99726-6cd1-4f12-b8e8-3e10f2ce7d52', project_access_token='p-2dda694dc92cbb2f1bc19052d6e747ec77d2059e') pc = project.project_context # In[2]: get_ipython().system(u'pip install turicreate') # In[3]: get_ipython().magic(u'load_ext autoreload') get_ipython().magic(u'autoreload 2') import pandas as pd import numpy as np import time import turicreate as tc from sklearn.model_selection import train_test_split
name = os.path.abspath(args.name);
if(not args.change):
while True:
try:
os.makedirs(name);
os.mkdir(os.path.join(name, "log"));
os.mkdir(os.path.join(name, "left"));
os.mkdir(os.path.join(name, "lstatistics"));
os.mkdir(os.path.join(name, "right"));
os.mkdir(os.path.join(name, "rstatistics"));
os.mkdir(os.path.join(name, "sam"));
os.mkdir(os.path.join(name, "output"));
os.mkdir(os.path.join(name, "plots"));
if(args.gmode):
os.mkdir(os.path.join(name, "gmode"));
break;
except:
name = os.path.abspath(raw_input("directory with this name is currently exists, please give another name: "))
p = Project(os.path.basename(name), args.reads, args.ref, args.index, args.system, args.mode, args.gmode, args.explore);
wf = open(os.path.join(gconst.proot, p.name + ".txt"), 'w');
wf.write("python " + " ".join(sys.argv) + "\n\n");
wf.write(p.info())
wf.close
mh = open(os.path.join(name, "Makefile"), 'w');
mh.write(p.makefile());
mh.close();