def __init__(self, dspath, dsdata, ppath, prob_desc=None, pfiles=None):
"""
inputs:
dspath - the path to the dataset root
dsdata - a dictionary containing the dataset metadata
ppath - the path the the prediction results file(s) directory root
prob_desc - the path to the problem description schema file that describes the prediction
"""
D3MDataset.__init__(self, dspath, dsdata)
LSPrediction.__init__(self, dspath, ppath, prob_desc, pfiles)
logger.debug("Initializing D3M prediction dataset")
def from_json(fpath):
"""
A static constructor of this class given a jsonified file
"""
if isinstance(fpath, str):
if path.exists(fpath):
#Get dataset path from json path
with open(fpath, 'r') as f:
ds_json = json.load(f)
else:
# logger.error("Found no dataset json at path: %s" % str(fpath))
raise Exception("Found no dataset json at path: %s" %
str(fpath))
elif isinstance(fpath, IOBase):
logger.debug("Loading dataset json from open file")
ds_json = json.load(fpath)
else:
# logger.error("Found no dataset json at path: %s" % str(fpath))
raise Exception("Found no dataset json at path: %s" % str(fpath))
logger.debug("got dataset json: %s" % str(ds_json))
json_doc = {
'about': ds_json['about'],
'dataResources': ds_json['dataResources']
}
dpath = ds_json['dataset_info']['root_path']
dname = ds_json['dataset_info']['dataset_dir'].rsplit("_", 1)[0]
if isinstance(fpath, str):
logger.debug("Creating D3mDataset with fpath: %s\nMetadata: %s" %
(dpath, str(ds_json)))
# ds = D3MDataset(fpath, ds_json['about'])
ds = D3MDataset(dpath, ds_json)
elif isinstance(fpath, IOBase):
logger.debug("Creating D3mDataset with fpath: %s\nMetadata: %s" %
(dpath, str(ds_json)))
# ds = D3MDataset(fpath.name, ds_json['about'])
ds = D3MDataset(dpath, ds_json)
logger.debug("Creating problem description")
logger.debug("Got default problem: %s" %
str(ProblemDesc.get_default_problem(ds)))
prob_desc = ProblemDesc.from_json(ProblemDesc.get_default_problem(ds))
# prob_desc = ProblemDesc.from_json(
# LSPrediction.get_default_problem(ds_json['dataset_info']['root_path']))
return D3MPrediction(dpath,
json_doc,
path.join(dpath,
ds_json['dataset_info']['dataset_dir'],
'output'),
prob_desc=prob_desc,
pfiles=ds_json['pred_info']['pred_files'])
def from_json(fpath):
"""
A static constructor of this class given a dataset json
"""
if isinstance(fpath, str):
if path.exists:
with open(fpath, 'r') as f:
ds_json = json.load(f)
else:
logger.error("Found no dataset json at path: %s" % str(fpath))
raise Exception("Found no dataset json at path: %s" %
str(fpath))
elif isinstance(fpath, IOBase):
logger.debug("Loading dataset json from open file")
ds_json = json.load(fpath)
else:
logger.error("Found no dataset json at path: %s" % str(fpath))
raise Exception("Found no dataset json at path: %s" % str(fpath))
dpath = ds_json['dataset_info']['root_path']
dname = ds_json['dataset_info']['dataset_dir'].rsplit("_", 1)[0]
if isinstance(fpath, str):
logger.debug("Creating D3mDataset with fpath: %s\nMetadata: %s" %
(str(fpath), str(ds_json['about'])))
# ds = D3MDataset(fpath, ds_json['about'])
ds = D3MDataset(dpath, ds_json)
elif isinstance(fpath, IOBase):
logger.debug("Creating D3mDataset with fpath: %s\nMetadata: %s" %
(fpath.name, str(ds_json['about'])))
# ds = D3MDataset(fpath.name, ds_json['about'])
ds = D3MDataset(dpath, ds_json)
logger.debug("Creating problem description")
logger.debug("Got default problem: %s" %
str(ProblemDesc.get_default_problem(ds)))
prob_desc = ProblemDesc.from_json(ProblemDesc.get_default_problem(ds))
# prob_desc = ProblemDesc.from_json(
# prob_desc = ProblemDesc.from_json(
# ProblemDesc.get_default_problem(D3MDataset(fpath, ds_json['about']))
# )
# LSPrediction.get_default_problem(ds_json['dataset_info']['root_path']))
return LSPrediction(ds_json['dataset_info']['root_path'],
ds_json['pred_info']['pred_root'],
prob_desc=prob_desc,
pfiles=ds_json['pred_info']['pred_files'])
# Get service fongif file
service_config = AppServiceSettings()
# Setup Logging
setup_logging(config)
logger = logging.getLogger('d3m_dataset_augmenter')
### Begin Script ###
logger.info(
"Generating an interface to allow user to query for a augmentation dataset"
)
logger.debug("Running D3M Dataset Augmenter with arguments: %s" %
str(args))
# Open dataset json
ds = D3MDataset.from_component_out_file(args.file0)
logger.debug("Dataset json parse: %s" % str(ds))
# Get the Problem Doc to forulate the Pipeline request
logger.debug("Problem input: %s" % args.file1)
prob = ProblemDesc.from_file(args.file1)
logger.debug("Got Problem Description: %s" % prob.print())
# Get URL of App Service
# host_url = os.environ["HOST_URL"]
# service_subdomain = os.environ["D3M_SERVICE_SUBDOMAIN"]
# service_url = service_subdomain + "." + host_url + ":9002"
service_url = service_config.get_service_url() + ":9002"
logger.info("connecting to service at url: %s" % service_url)
query_info = {"id": "1"}
setup_logging(config)
logger = logging.getLogger('datasets_importer')
### Begin Script ###
logger.info("Importing List of available datasets")
logger.debug("Running Dataset Importer with arguments: %s" % str(args))
# Read in the dataset json
ds_root = config.get_dataset_path()
datasets = set()
for root, dirs, files in os.walk(ds_root):
for f in files:
if f == 'datasetDoc.json':
logger.debug("Found dataset in directory: %s" % root)
try:
ds = D3MDataset.from_dataset_json(path.join(root, f))
if ds.name not in datasets:
logger.info("Found dataset name: %s\nAt path: %s" %
(ds.name, ds.dpath))
datasets.add(ds.name)
except:
# Don't choke on unsupported dataset jsons
logger.warning("Encountered unsupported dataset: %s" %
str(path.join(root, f)))
logger.debug("Found datasets: %s" % str(datasets))
# # Write dataset info to output file
out_file_path = path.join(args.workingDir,
config.get('Dataset', 'out_file'))
logger.info("Writing dataset list of %i datasets to file: %s" %