def initialize():
train_channel = cv.Channel(name="train", required=True)
train_channel.add("csv", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
train_channel.add("csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
train_channel.add("libsvm", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
train_channel.add("libsvm", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
train_channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
train_channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
train_channel.add("text/libsvm", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
train_channel.add("text/libsvm", cv.Channel.FILE_MODE,
cv.Channel.REPLICATED)
validation_channel = cv.Channel(name="validation", required=False)
validation_channel.add("csv", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
validation_channel.add("csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
validation_channel.add("libsvm", cv.Channel.FILE_MODE, cv.Channel.SHARDED)
validation_channel.add("libsvm", cv.Channel.FILE_MODE,
cv.Channel.REPLICATED)
validation_channel.add("text/csv", cv.Channel.FILE_MODE,
cv.Channel.SHARDED)
validation_channel.add("text/csv", cv.Channel.FILE_MODE,
cv.Channel.REPLICATED)
validation_channel.add("text/libsvm", cv.Channel.FILE_MODE,
cv.Channel.SHARDED)
validation_channel.add("text/libsvm", cv.Channel.FILE_MODE,
cv.Channel.REPLICATED)
# new for script mode/algorithm mode toggle
code_channel = cv.Channel(name="code", required=False)
code_channel.add("text/python", cv.Channel.FILE_MODE,
cv.Channel.REPLICATED)
return cv.Channels(train_channel, validation_channel, code_channel)
def initialize():
train_channel = cv.Channel(name="train", required=True)
for ct in VALID_CONTENT_TYPES:
train_channel.add(ct, cv.Channel.FILE_MODE, cv.Channel.SHARDED)
train_channel.add(ct, cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
for ct in VALID_PIPED_CONTENT_TYPES:
train_channel.add(ct, cv.Channel.PIPE_MODE, cv.Channel.SHARDED)
train_channel.add(ct, cv.Channel.PIPE_MODE, cv.Channel.REPLICATED)
validation_channel = cv.Channel(name="validation", required=False)
for ct in VALID_CONTENT_TYPES:
validation_channel.add(ct, cv.Channel.FILE_MODE, cv.Channel.SHARDED)
validation_channel.add(ct, cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
for ct in VALID_PIPED_CONTENT_TYPES:
validation_channel.add(ct, cv.Channel.PIPE_MODE, cv.Channel.SHARDED)
validation_channel.add(ct, cv.Channel.PIPE_MODE, cv.Channel.REPLICATED)
# new for script mode/algorithm mode toggle
code_channel = cv.Channel(name="code", required=False)
code_channel.add("text/python", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
data_channels = cv.Channels(train_channel, validation_channel, code_channel)
data_channels.set_default_content_type("text/libsvm")
return data_channels
def test_simple_supported(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
channels.validate({
"train": {
"ContentType": "text/csv",
"TrainingInputMode": "File",
"S3DistributionType": "FullyReplicated",
"RecordWrapperType": "None"
}
})
def test_simple_supported(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
channels.validate({
"train": {
cv.CONTENT_TYPE: "text/csv",
cv.TRAINING_INPUT_MODE: "File",
cv.S3_DIST_TYPE: "FullyReplicated",
"RecordWrapperType": "None"
}
})
def test_simple_format(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
result = {
"Name": "train",
"Description": "train",
"IsRequired": True,
"SupportedContentTypes": ["text/csv"],
"SupportedInputModes": ["File"]
}
self.assertEqual(channel.format(), result)
self.assertEqual(channels.format(), [result])
def test_simple_extra(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
with self.assertRaises(exc.UserError):
channels.validate({
"train": {
"ContentType": "text/csv",
"TrainingInputMode": "File",
"S3DistributionType": "FullyReplicated",
"RecordWrapperType": "None"
},
"extra": {}
})
def test_simple_extra(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
with self.assertRaises(exc.UserError):
channels.validate({
"train": {
cv.CONTENT_TYPE: "text/csv",
cv.TRAINING_INPUT_MODE: "File",
cv.S3_DIST_TYPE: "FullyReplicated",
"RecordWrapperType": "None"
},
"extra": {}
})
def test_default_content_type(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
channels.set_default_content_type("text/csv")
test_user_channels = {
"train": {
cv.TRAINING_INPUT_MODE: "File",
cv.S3_DIST_TYPE: "FullyReplicated",
"RecordWrapperType": "None"
}
}
channels.validate(test_user_channels)
self.assertEqual(test_user_channels["train"][cv.CONTENT_TYPE],
"text/csv")
def test_simple_required(self):
channel = cv.Channel(name="train", required=True)
channel.add("text/csv", cv.Channel.FILE_MODE, cv.Channel.REPLICATED)
channels = cv.Channels(channel)
with self.assertRaises(exc.UserError):
channels.validate({"sorry": {}})