def load(self, path=None): if path is None: path = os.getcwd() for pname in self.part_names: filename = os.path.join(path, '%s.yaml' % pname) if not fsclient.is_s3_path(filename): filename = os.path.abspath(filename) if fsclient.is_file_exists(filename): self.parts[pname] = self._load(filename) self.is_loaded = True
def _path_to_credentials(self): if self.ctx.config.get('path_to_credentials'): creds_path = os.path.abspath( self.ctx.config.get('path_to_credentials')) elif os.environ.get('%s_CREDENTIALS_PATH' % self.provider.upper()): creds_path = os.environ.get('%s_CREDENTIALS_PATH' % self.provider.upper()) else: cur_path = os.getcwd() if self.ctx.config.path: cur_path = self.ctx.config.path if fsclient.is_file_exists( os.path.join(cur_path, "%s.json" % self.provider)): creds_path = cur_path else: creds_path = os.path.abspath( '%s/.a2ml' % os.environ.get('HOME', os.getcwd())) return creds_path
def verify(data_source_file, config_path=None): if urllib.parse.urlparse(data_source_file).scheme in ['http', 'https']: return data_source_file, False if not fsclient.is_s3_path(data_source_file): if config_path is None: config_path = os.getcwd() data_source_file = os.path.join(config_path, data_source_file) if not fsclient.is_s3_path(data_source_file): data_source_file = os.path.abspath(data_source_file) filename, file_extension = os.path.splitext(data_source_file) if not file_extension in SUPPORTED_FORMATS: raise AugerException( 'Source file has to be one of the supported fomats: %s' % ', '.join(SUPPORTED_FORMATS)) if not fsclient.is_file_exists(data_source_file): raise AugerException( 'Can\'t find file to import: %s' % data_source_file) return data_source_file, True
def _load(self, name): part = SerializableConfigYaml() if fsclient.is_file_exists(name): part.load_from_file(name) return part
def test_save_prediction(self): model_path = 'tests/fixtures/test_predict_by_model/iris' options = fsclient.read_json_file( os.path.join(model_path, "options.json")) prediction_id = "123" prediction_date = "today" results_file_path = os.path.join( model_path, "predictions", prediction_date + '_' + prediction_id + "_results.feather.zstd") predicted_file_path = os.path.join( model_path, "predictions", "iris_test_" + prediction_id + "_" + options.get('uid') + "_predicted.csv") ds = DataFrame.create_dataframe( os.path.join(model_path, "iris_test.csv")) fsclient.remove_file(results_file_path) self.assertFalse(fsclient.is_file_exists(results_file_path)) fsclient.remove_file(predicted_file_path) self.assertFalse(fsclient.is_file_exists(predicted_file_path)) res = ModelHelper.save_prediction(ds, prediction_id, support_review_model=True, json_result=False, count_in_result=False, prediction_date=prediction_date, model_path=model_path, model_id=options.get('uid')) self.assertEqual(res, predicted_file_path) self.assertTrue(fsclient.is_file_exists(predicted_file_path)) self.assertTrue(fsclient.is_file_exists(results_file_path)) ds = DataFrame.create_dataframe( os.path.join(model_path, "iris_test.csv")) fsclient.remove_file(results_file_path) self.assertFalse(fsclient.is_file_exists(results_file_path)) fsclient.remove_file(predicted_file_path) self.assertFalse(fsclient.is_file_exists(predicted_file_path)) res = ModelHelper.save_prediction(ds, prediction_id, support_review_model=True, json_result=True, count_in_result=False, prediction_date=prediction_date, model_path=model_path, model_id=options.get('uid')) res = json.loads(res) self.assertEqual(res['columns'], ds.columns) self.assertEqual(len(res['data']), 6) ds = DataFrame.create_dataframe( os.path.join(model_path, "iris_test.csv")) fsclient.remove_file(results_file_path) self.assertFalse(fsclient.is_file_exists(results_file_path)) fsclient.remove_file(predicted_file_path) self.assertFalse(fsclient.is_file_exists(predicted_file_path)) ds.options['data_path'] = None res = ModelHelper.save_prediction(ds, prediction_id, support_review_model=False, json_result=False, count_in_result=False, prediction_date=prediction_date, model_path=model_path, model_id=options.get('uid')) self.assertEqual(type(res[0]), dict) self.assertEqual(res[0][options['targetFeature']], 'setosa') ds = DataFrame.create_dataframe( os.path.join(model_path, "iris_test.csv")) fsclient.remove_file(results_file_path) self.assertFalse(fsclient.is_file_exists(results_file_path)) fsclient.remove_file(predicted_file_path) self.assertFalse(fsclient.is_file_exists(predicted_file_path)) ds.options['data_path'] = None ds.loaded_columns = ds.columns res = ModelHelper.save_prediction(ds, prediction_id, support_review_model=False, json_result=False, count_in_result=False, prediction_date=prediction_date, model_path=model_path, model_id=options.get('uid')) self.assertEqual(res['columns'], ds.columns) self.assertEqual(len(res['data']), 6) self.assertEqual(type(res['data'][0]), list)
def verify_local_model(self, model_id): model_path = os.path.join(self.ctx.config.get_path(), 'models') model_name = os.path.join(model_path, 'model-%s.zip' % model_id) return fsclient.is_file_exists(model_name), model_path, model_name
def verify_local_model(self, model_id): model_path = os.path.join(self.ctx.config.get_model_path(model_id), 'model.pkl.gz') return fsclient.is_file_exists(model_path), model_path