def test_transform_csv(self, popen_mock, run_and_monitor_mock):
mlmagic.ml(
line='transform --prefix my_prefix --shuffle --cloud',
cell="""\
output: my_out_dir
analysis: my_analyze_dir
batch_size: 123
training_data:
csv: file*.csv
cloud_config:
project_id: my_id
num_workers: 987
worker_machine_type: BLUE
job_name: RED""")
cmd_list = run_and_monitor_mock.call_args[0][0]
# cmd_list = [u'python', u'transform.py', u'--output', 'path/my_out_dir',
# u'--analysis', 'path/my_analyze_dir', u'--prefix', 'my_prefix',
# u'--shuffle', u'--batch-size', '100', u'--csv=/path/file*.csv'
# ...
self.assertEqual('python', cmd_list[0])
self.assertEqual('transform.py', cmd_list[1])
self.assertIn('--shuffle', cmd_list)
self.assertTrue(find_key_endswith(cmd_list, '--output', 'my_out_dir'))
self.assertTrue(find_key_endswith(cmd_list, '--analysis', 'my_analyze_dir'))
self.assertTrue(find_key_value(cmd_list, '--prefix', 'my_prefix'))
self.assertTrue(find_key_value(cmd_list, '--batch-size', '123'))
self.assertTrue(find_startswith_endswith(cmd_list, '--csv=', 'file*.csv'))
self.assertTrue(find_key_value(cmd_list, '--project-id', 'my_id'))
self.assertTrue(find_key_value(cmd_list, '--num-workers', '987'))
self.assertTrue(find_key_value(cmd_list, '--worker-machine-type', 'BLUE'))
self.assertTrue(find_key_value(cmd_list, '--job-name', 'RED'))
def test_train_csv(self, popen_mock, submit_training_mock,
package_and_copy_mock, _show_job_link_mock):
mlmagic.ml(
line='train --cloud',
cell="""\
output: gs://my_out_dir
analysis: my_analyze_dir
training_data:
csv: file*.csv
evaluation_data:
transformed: file*.tfrecords.gz
model_args:
key: value
cloud_config:
job_name: job1
project_id: id""")
job_request = submit_training_mock.call_args[0][0]
cmd_list = job_request['args']
self.assertEqual(job_request['project_id'], 'id')
self.assertEqual(job_request['job_dir'], 'gs://my_out_dir')
self.assertEqual(job_request['python_module'], 'trainer.task')
self.assertEqual(job_request['package_uris'], ['gs://my_out_dir/staging/trainer.tar.gz'])
self.assertTrue(find_key_value(cmd_list, '--job-dir', 'gs://my_out_dir'))
self.assertTrue(find_key_endswith(cmd_list, '--analysis', 'my_analyze_dir'))
self.assertTrue(find_startswith_endswith(cmd_list, '--train=', 'file*.csv'))
self.assertIn('--transform', cmd_list)
self.assertTrue(find_startswith_endswith(cmd_list, '--eval=', '*.tfrecords.gz'))
self.assertTrue(find_key_value(cmd_list, '--key', 'value'))
def test_analyze_csv_local(self, popen_mock, run_and_monitor_mock):
mlmagic.ml(
line='dataset create',
cell="""\
format: csv
train: ./taxi/train.csv
eval: ./taxi/eval.csv
name: taxi_data
schema:
- name: unique_key
type: STRING
- name: fare
type: FLOAT"""
)
mlmagic.ml(
line='analyze',
cell="""\
output: my_out_dir
data: taxi_data
features: dummy_features""")
cmd_list = run_and_monitor_mock.call_args[0][0]
# cmd_list = [u'python', u'analyze.py', u'--output', 'path/my_out_dir',
# u'--csv=path/file*.csv', u'--schema', u'/path/schema.json',
# u'--features', u'path/features.json']
self.assertEqual('python', cmd_list[0])
self.assertEqual('analyze.py', cmd_list[1])
self.assertIn('--schema', cmd_list)
self.assertIn('--features', cmd_list)
self.assertTrue(find_key_endswith(cmd_list, '--output', 'my_out_dir'))
self.assertTrue(find_startswith_endswith(cmd_list, '--csv=', 'train.csv'))
def test_train_csv(self, popen_mock, submit_training_mock,
package_and_copy_mock, _show_job_link_mock):
mlmagic.ml(
line='dataset create',
cell="""\
format: transformed
train: ./taxi/train_tfrecord.tar.gz
eval: ./taxi/eval_tfrecord.tar.gz
name: taxi_data_transformed"""
)
mlmagic.ml(
line='train --cloud',
cell="""\
output: gs://my_out_dir
analysis: my_analyze_dir
data: $taxi_data_transformed
model_args:
key: value
cloud_config:
job_name: job1
project_id: id""")
job_request = submit_training_mock.call_args[0][0]
cmd_list = job_request['args']
self.assertEqual(job_request['project_id'], 'id')
self.assertEqual(job_request['job_dir'], 'gs://my_out_dir')
self.assertEqual(job_request['python_module'], 'trainer.task')
self.assertEqual(job_request['package_uris'], ['gs://my_out_dir/staging/trainer.tar.gz'])
self.assertTrue(find_key_value(cmd_list, '--job-dir', 'gs://my_out_dir'))
self.assertTrue(find_key_endswith(cmd_list, '--analysis', 'my_analyze_dir'))
self.assertTrue(find_startswith_endswith(cmd_list, '--train=', 'train_tfrecord.tar.gz'))
self.assertTrue(find_startswith_endswith(cmd_list, '--eval=', 'eval_tfrecord.tar.gz'))
self.assertTrue(find_key_value(cmd_list, '--key', 'value'))
def test_predict_csv(self, popen_mock, get_prediction_results_mock, display_mock):
# Don't run prediction on a real graph, just return 1 value.
df = pd.DataFrame({'col1': ['key1'], 'col2': ['value1'], 'col3': ['value2']})
get_prediction_results_mock.return_value = df
mlmagic.ml(
line='predict --cloud',
cell="""\
model: model.version
headers: col1,col2,col3
image_columns: col3
prediction_data:
- key1,value1,value2""")
# Check it was printed
self.assertEqual(1, display_mock.call_count)
def _create_text_test_data(self):
"""Create text model."""
test_data = """1,sour green round,lime
2,melon green long,cucumber
3,sweet round red,apple"""
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
analyze_dir = os.path.join(self._test_dir, 'analysistxt')
train_dir = os.path.join(self._test_dir, 'traintxt')
mlmagic.ml(line='analyze',
cell="""\
output: %s
training_data:
csv: %s
schema:
- name: key
type: INTEGER
- name: text
type: STRING
- name: target
type: STRING
features:
key:
transform: key
text:
transform: bag_of_words
target:
transform: target""" % (analyze_dir, train_csv))
mlmagic.ml(line='train',
cell="""\
output: %s
analysis: %s
training_data:
csv: %s
evaluation_data:
csv: %s
model_args:
model: linear_classification
top-n: 0
max-steps: 300""" %
(train_dir, analyze_dir, train_csv, train_csv))
def test_transform_csv(self, popen_mock, run_and_monitor_mock):
mlmagic.ml(
line='dataset create',
cell="""\
format: csv
train: ./taxi/train.csv
eval: ./taxi/eval.csv
name: taxi_data
schema:
- name: unique_key
type: STRING
- name: fare
type: FLOAT"""
)
mlmagic.ml(
line='transform --shuffle --cloud',
cell="""\
output: my_out_dir
analysis: my_analyze_dir
batch_size: 123
data: taxi_data
cloud_config:
project_id: my_id
num_workers: 987
worker_machine_type: BLUE
job_name: RED""")
cmd_list = run_and_monitor_mock.call_args[0][0]
# cmd_list = [u'python', u'transform.py', u'--output', 'path/my_out_dir',
# u'--analysis', 'path/my_analyze_dir', u'--prefix', 'my_prefix',
# u'--shuffle', u'--batch-size', '100', u'--csv=/path/file*.csv'
# ...
self.assertEqual('python', cmd_list[0])
self.assertEqual('transform.py', cmd_list[1])
self.assertIn('--shuffle', cmd_list)
self.assertTrue(find_key_endswith(cmd_list, '--output', 'my_out_dir'))
self.assertTrue(find_key_endswith(cmd_list, '--analysis', 'my_analyze_dir'))
self.assertTrue(find_key_value(cmd_list, '--prefix', 'train') or
find_key_value(cmd_list, '--prefix', 'eval'))
self.assertTrue(find_key_value(cmd_list, '--batch-size', '123'))
self.assertTrue(find_startswith_endswith(cmd_list, '--csv=', 'train.csv') or
find_startswith_endswith(cmd_list, '--csv=', 'eval.csv'))
self.assertTrue(find_key_value(cmd_list, '--project-id', 'my_id'))
self.assertTrue(find_key_value(cmd_list, '--num-workers', '987'))
self.assertTrue(find_key_value(cmd_list, '--worker-machine-type', 'BLUE'))
self.assertTrue(find_key_value(cmd_list, '--job-name', 'RED'))
def test_analyze_csv_local(self, popen_mock, run_and_monitor_mock):
mlmagic.ml(
line='analyze',
cell="""\
output: my_out_dir
training_data:
csv: file*.csv
schema: dummy_schema
features: dummy_features""")
cmd_list = run_and_monitor_mock.call_args[0][0]
# cmd_list = [u'python', u'analyze.py', u'--output', 'path/my_out_dir',
# u'--csv=path/file*.csv', u'--schema', u'/path/schema.json',
# u'--features', u'path/features.json']
self.assertEqual('python', cmd_list[0])
self.assertEqual('analyze.py', cmd_list[1])
self.assertIn('--schema', cmd_list)
self.assertIn('--features', cmd_list)
self.assertTrue(find_key_endswith(cmd_list, '--output', 'my_out_dir'))
self.assertTrue(find_startswith_endswith(cmd_list, '--csv=', 'file*.csv'))
def _create_tabular_test_data(self):
"""Create tabular model with text."""
test_data = """1,5.0,monday,word1 word2 word3,true
2,3.2,tuesday,word1 word3,true
3,-1.1,friday,word1,false"""
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
df = pd.read_csv(train_csv,
names=['key', 'num', 'weekday', 'garbage', 'target'])
analyze_dir = os.path.join(self._test_dir, 'analysistab')
train_dir = os.path.join(self._test_dir, 'traintab')
mlmagic.ml(line='dataset create',
cell="""\
format: csv
name: mytabular
schema:
- name: key
type: INTEGER
- name: num
type: FLOAT
- name: weekday
type: STRING
- name: garbage
type: STRING
- name: target
type: STRING
train: %s
eval: %s""" % (train_csv, train_csv))
mlmagic.ml(line='analyze',
cell="""\
output: %s
data: mytabular
features:
key:
transform: key
num:
transform: scale
weekday:
transform: one_hot
garbage:
transform: bag_of_words
target:
transform: target""" % (analyze_dir))
mlmagic.ml(line='train',
cell="""\
output: %s
analysis: %s
data: mytabular
notb: true
model_args:
model: linear_classification
top-n: 0
max-steps: 300""" % (train_dir, analyze_dir))
return df
def test_batch_predict_csv(self, popen_mock, submit_batch_prediction_mock,
default_mock, _show_job_link_mock):
default_mock.return_value = mock.Mock(project_id='my_project_id')
mlmagic.ml(
line='batch_predict --cloud',
cell="""\
model: my_model.my_version
output: gs://output
format: json
batch_size: 10
prediction_data:
csv: %s""" % os.path.abspath(__file__))
job_args = submit_batch_prediction_mock.call_args[0][0]
self.assertEqual(job_args['input_paths'], [os.path.abspath(__file__)])
self.assertEqual(
job_args['version_name'],
'projects/my_project_id/models/my_model/versions/my_version')
self.assertEqual(job_args['output_path'], 'gs://output')
self.assertEqual(job_args['data_format'], 'TEXT')
def _create_text_test_data(self):
"""Create text model."""
test_data = """1,sour green round,lime
2,melon green long,cucumber
3,sweet round red,apple"""
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
analyze_dir = os.path.join(self._test_dir, 'analysistxt')
train_dir = os.path.join(self._test_dir, 'traintxt')
mlmagic.ml(
line='dataset create',
cell="""\
format: csv
name: mytext
schema:
- name: key
type: INTEGER
- name: text
type: STRING
- name: target
type: STRING
train: %s
eval: %s""" % (train_csv, train_csv))
mlmagic.ml(
line='analyze',
cell="""\
output: %s
data: mytext
features:
key:
transform: key
text:
transform: bag_of_words
target:
transform: target""" % (analyze_dir))
mlmagic.ml(
line='train',
cell="""\
output: %s
analysis: %s
data: mytext
notb: true
model_args:
model: linear_classification
top-n: 0
max-steps: 300""" % (train_dir, analyze_dir))
def _create_image_test_data(self):
image_path1 = os.path.join(self._test_dir, 'img1.jpg')
image_path2 = os.path.join(self._test_dir, 'img2.jpg')
image_path3 = os.path.join(self._test_dir, 'img3.jpg')
Image.new('RGB', size=(128, 128),
color=(155, 211, 64)).save(image_path1, "JPEG")
Image.new('RGB', size=(64, 64),
color=(111, 21, 86)).save(image_path2, "JPEG")
Image.new('RGB', size=(16, 16),
color=(255, 21, 1)).save(image_path3, "JPEG")
test_data = """1,1.2,word1 word2,%s,true
2,3.2,word2 word3,%s,false
5,-2.1,word3 word4,%s,true""" % (image_path1, image_path2, image_path3)
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
analyze_dir = os.path.join(self._test_dir, 'analysisimg')
transform_dir = os.path.join(self._test_dir, 'transformimg')
train_dir = os.path.join(self._test_dir, 'trainimg')
# Download inception checkpoint. Note that gs url doesn't work because
# we may not have gcloud signed in when running the test.
url = ('https://storage.googleapis.com/cloud-ml-data/img/' +
'flower_photos/inception_v3_2016_08_28.ckpt')
checkpoint_path = os.path.join(self._test_dir, "checkpoint")
response = urlopen(url)
with open(checkpoint_path, 'wb') as f:
f.write(response.read())
mlmagic.ml(line='dataset create',
cell="""\
format: csv
name: myds
schema:
- name: key
type: INTEGER
- name: num
type: FLOAT
- name: text
type: STRING
- name: img_url
type: STRING
- name: target
type: STRING
train: %s
eval: %s""" % (train_csv, train_csv))
mlmagic.ml(line='analyze',
cell="""\
output: %s
data: myds
features:
key:
transform: key
num:
transform: scale
text:
transform: bag_of_words
img_url:
transform: image_to_vec
checkpoint: %s
target:
transform: target""" % (analyze_dir, checkpoint_path))
mlmagic.ml(line='transform',
cell="""\
output: %s
analysis: %s
data: myds""" % (transform_dir, analyze_dir))
mlmagic.ml(line='dataset create',
cell="""\
format: transformed
name: transformed_ds
train: %s/train-*
eval: %s/eval-*""" % (transform_dir, transform_dir))
mlmagic.ml(line='train',
cell="""\
output: %s
analysis: %s
data: transformed_ds
notb: true
model_args:
model: linear_classification
top-n: 0
max-steps: 200""" % (train_dir, analyze_dir))
def _create_tabular_test_data(self):
"""Create tabular model with text."""
test_data = """1,5.0,monday,word1 word2 word3,true
2,3.2,tuesday,word1 word3,true
3,-1.1,friday,word1,false"""
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
df = pd.read_csv(train_csv, names=['key', 'num', 'weekday', 'garbage', 'target'])
analyze_dir = os.path.join(self._test_dir, 'analysistab')
train_dir = os.path.join(self._test_dir, 'traintab')
mlmagic.ml(
line='dataset create',
cell="""\
format: csv
name: mytabular
schema:
- name: key
type: INTEGER
- name: num
type: FLOAT
- name: weekday
type: STRING
- name: garbage
type: STRING
- name: target
type: STRING
train: %s
eval: %s""" % (train_csv, train_csv))
mlmagic.ml(
line='analyze',
cell="""\
output: %s
data: mytabular
features:
key:
transform: key
num:
transform: scale
weekday:
transform: one_hot
garbage:
transform: bag_of_words
target:
transform: target""" % (analyze_dir))
mlmagic.ml(
line='train',
cell="""\
output: %s
analysis: %s
data: mytabular
notb: true
model_args:
model: linear_classification
top-n: 0
max-steps: 300""" % (train_dir, analyze_dir))
return df
def _create_image_test_data(self):
image_path1 = os.path.join(self._test_dir, 'img1.jpg')
image_path2 = os.path.join(self._test_dir, 'img2.jpg')
image_path3 = os.path.join(self._test_dir, 'img3.jpg')
Image.new('RGB', size=(128, 128), color=(155, 211, 64)).save(image_path1, "JPEG")
Image.new('RGB', size=(64, 64), color=(111, 21, 86)).save(image_path2, "JPEG")
Image.new('RGB', size=(16, 16), color=(255, 21, 1)).save(image_path3, "JPEG")
test_data = """1,1.2,word1 word2,%s,true
2,3.2,word2 word3,%s,false
5,-2.1,word3 word4,%s,true""" % (image_path1, image_path2, image_path3)
train_csv = os.path.join(self._test_dir, 'train.csv')
with open(train_csv, 'w') as f:
f.write(test_data)
analyze_dir = os.path.join(self._test_dir, 'analysisimg')
transform_dir = os.path.join(self._test_dir, 'transformimg')
train_dir = os.path.join(self._test_dir, 'trainimg')
# Download inception checkpoint. Note that gs url doesn't work because
# we may not have gcloud signed in when running the test.
url = ('https://storage.googleapis.com/cloud-ml-data/img/' +
'flower_photos/inception_v3_2016_08_28.ckpt')
checkpoint_path = os.path.join(self._test_dir, "checkpoint")
response = urlopen(url)
with open(checkpoint_path, 'wb') as f:
f.write(response.read())
mlmagic.ml(
line='dataset create',
cell="""\
format: csv
name: myds
schema:
- name: key
type: INTEGER
- name: num
type: FLOAT
- name: text
type: STRING
- name: img_url
type: STRING
- name: target
type: STRING
train: %s
eval: %s""" % (train_csv, train_csv))
mlmagic.ml(
line='analyze',
cell="""\
output: %s
data: myds
features:
key:
transform: key
num:
transform: scale
text:
transform: bag_of_words
img_url:
transform: image_to_vec
checkpoint: %s
target:
transform: target""" % (analyze_dir, checkpoint_path))
mlmagic.ml(
line='transform',
cell="""\
output: %s
analysis: %s
data: myds""" % (transform_dir, analyze_dir))
mlmagic.ml(
line='dataset create',
cell="""\
format: transformed
name: transformed_ds
train: %s/train-*
eval: %s/eval-*""" % (transform_dir, transform_dir))
mlmagic.ml(
line='train',
cell="""\
output: %s
analysis: %s
data: transformed_ds
notb: true
model_args:
model: linear_classification
top-n: 0
max-steps: 200""" % (train_dir, analyze_dir))