def __init__(self, config):
deployConfig = jobs_configure.init_config(config)
self.deployConfig = deployConfig
self.alarm_interface = AlarmInterface(
deployConfig['defaultConfig']['region'])
self.iot_interface = IotInterface(
deployConfig['defaultConfig']['region'])
self.clean_up(deployConfig)
def __init__(self):
thingArnList, deviceCount, thingNameList = parse_thingList(thingListFilePath)
if deviceCount < 1:
raise Exception('thing list should not be empty')
self.thingArnList = thingArnList
self.deviceCount = deviceCount
self.thingNameList = thingNameList
self.thingListUpdateTime = 0
self.iot_interface = IotInterface(region)
def setUp(self):
self.iot_interface = IotInterface(REGION)
self.iot_interface.client.create_stream = Mock(return_value=None)
self.iot_interface.client.create_job = Mock(return_value=None)
self.iot_interface.client.describe_job = Mock(
return_value=TEST_EXPTECTED_DSB_JOB_RSP)
self.iot_interface.client.describe_job_execution = Mock(
return_value=TEST_EXPTECTED_DSB_JOB_EXEC_RSP)
self.iot_interface.client.describe_stream = Mock(
return_value=TEST_EXPTECTED_DSB_STREAM_RSP)
self.iot_interface.client.cancel_job = Mock(return_value=None)
self.iot_interface.client.delete_job = Mock(return_value=None)
self.iot_interface.client.delete_stream = Mock(return_value=None)
class ota_deployment_tool():
def __init__(self, config):
deployConfig = jobs_configure.init_config(config)
self.deployConfig = deployConfig
self.alarm_interface = AlarmInterface(
deployConfig['defaultConfig']['region'])
self.iot_interface = IotInterface(
deployConfig['defaultConfig']['region'])
self.clean_up(deployConfig)
def clean_up(self, deployConfig):
cleanUpCfg = deployConfig['defaultConfig']['cleanUpCfg']
streamId = deployConfig['defaultConfig']['streamId']
jobId = deployConfig['defaultConfig']['jobId']
if cleanUpCfg:
logging.info('deleting old job')
self.iot_interface.delete_job(jobId)
logging.info('deleting old stream')
self.iot_interface.delete_stream(streamId)
logging.info('deleting old alarms')
if 'alarmConfigs' in deployConfig:
self.alarm_interface.delete_alarms(
deployConfig['alarmConfigs'])
else:
logging.info('skipping clean_up due to cleanUpCfg set to false')
def schedule_jobs(self, deployConfig):
logging.info('creating new stream')
streamId = deployConfig['defaultConfig']['streamId']
fileId = deployConfig['defaultConfig']['fileId']
bucket = deployConfig['defaultConfig']['bucket']
binFileKey = deployConfig['defaultConfig']['binFileKey']
roleArn = deployConfig['defaultConfig']['roleArn']
jobId = deployConfig['defaultConfig']['jobId']
thingArnList = deployConfig['defaultConfig']['thingArnList']
jobDocumentSrc = deployConfig['defaultConfig']['jobDocumentSrc']
deviceCount = deployConfig['defaultConfig']['deviceCount']
defaultDelay = deployConfig['defaultConfig']['defaultDelay']
rounds = deployConfig['defaultConfig']['rounds']
if 'alarmConfigs' in deployConfig:
status, err = self.alarm_interface.create_alarms(
deployConfig['alarmConfigs'])
if not status:
logging.error(err)
return
status, err = self.iot_interface.create_stream(streamId, fileId,
bucket, binFileKey,
roleArn)
if not status:
logging.error(err)
return
status, err = self.iot_interface.create_job(deployConfig)
if not status:
logging.error(err)
return
job_complete_counter = 0
while job_complete_counter < rounds:
job_dsb, err = self.iot_interface.get_job_info(jobId)
status = job_dsb.get('status')
if err:
logging.error(err)
return
if status == 'COMPLETED':
numberOfSucceededThings = job_dsb.get(
'jobProcessDetails', {}).get('numberOfSucceededThings')
if numberOfSucceededThings == deviceCount:
job_complete_counter = job_complete_counter + 1
logging.info(
'deviceCount: %d matches numberOfSucceededThings: %d job completed with success',
deviceCount, numberOfSucceededThings)
logging.info(
'jobId : %s completed , job_complete_counter: %d',
jobId, job_complete_counter)
logging.info('jobId %s completed', jobId)
self.clean_up(deployConfig)
if job_complete_counter < rounds:
logging.info('creating new stream')
status, err = self.iot_interface.create_stream(
streamId, fileId, bucket, binFileKey, roleArn)
if not status:
logging.error(err)
return
logging.info('creating new job, thingArnList: %s',
thingArnList)
status, err = self.iot_interface.create_job(
deployConfig)
if not status:
logging.error(err)
return
else:
logging.info(
'deviceCount: %d does not matches numberOfSucceededThings: %d job completed with failure',
deviceCount, numberOfSucceededThings)
break
elif status == 'IN_PROGRESS':
logging.info('jobId: %s IN_PROGRESS: ', jobId)
elif status == 'CANCELED' or status == 'DELETION_IN_PROGRESS' or status == 'DELETION_IN_PROGRESS':
logging.info('unexpected failure with status: %s', status)
break
else:
logging.info('unexpected status: %s', status)
break
time.sleep(defaultDelay)
class IotInterfaceTests(unittest.TestCase):
def setUp(self):
self.iot_interface = IotInterface(REGION)
self.iot_interface.client.create_stream = Mock(return_value=None)
self.iot_interface.client.create_job = Mock(return_value=None)
self.iot_interface.client.describe_job = Mock(
return_value=TEST_EXPTECTED_DSB_JOB_RSP)
self.iot_interface.client.describe_job_execution = Mock(
return_value=TEST_EXPTECTED_DSB_JOB_EXEC_RSP)
self.iot_interface.client.describe_stream = Mock(
return_value=TEST_EXPTECTED_DSB_STREAM_RSP)
self.iot_interface.client.cancel_job = Mock(return_value=None)
self.iot_interface.client.delete_job = Mock(return_value=None)
self.iot_interface.client.delete_stream = Mock(return_value=None)
def test_create_stream(self):
self.iot_interface.client.create_stream = Mock(
return_value=TEST_EXPTECTED_DSB_JOB_RSP)
# Given these inputs and expected outputs
createStreamArgs = {
'streamId': STREAM_ID,
'fileId': FILE_ID,
'bucket': BUCKET,
'key': KEY,
'role_arn': ROLE_ARN
}
iotCreateStreamArgs = {
'streamId':
STREAM_ID,
'description':
'testing',
'files': [{
"fileId": FILE_ID,
"s3Location": {
"bucket": BUCKET,
"key": KEY
}
}],
'roleArn':
ROLE_ARN
}
# When this happens
status, err = self.iot_interface.create_stream(**createStreamArgs)
# Expect these behaviors
self.assertTrue(status)
self.iot_interface.client.create_stream.assert_called_with(
**iotCreateStreamArgs)
def test_create_stream_exception(self):
self.iot_interface.client.create_stream = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
createStreamArgs = {
'streamId': STREAM_ID,
'fileId': FILE_ID,
'bucket': BUCKET,
'key': KEY,
'role_arn': ROLE_ARN
}
iotCreateStreamArgs = {
'streamId':
STREAM_ID,
'description':
'testing',
'files': [{
"fileId": FILE_ID,
"s3Location": {
"bucket": BUCKET,
"key": KEY
}
}],
'roleArn':
ROLE_ARN
}
# When this happens
status, err = self.iot_interface.create_stream(**createStreamArgs)
# Expect these behaviors
self.assertFalse(status)
self.iot_interface.client.create_stream.assert_called_with(
**iotCreateStreamArgs)
def test_create_job(self):
# Given these inputs and expected outputs
deployConfig = {}
deployConfig['defaultConfig'] = configMap['defaultConfig']
kwargs = {
'jobId': TEST_DEFAULT_CONFIG['jobId'],
'targets': TEST_DEFAULT_CONFIG['thingArnList'],
'documentSource': TEST_DEFAULT_CONFIG['jobDocumentSrc'],
'targetSelection': TEST_DEFAULT_CONFIG['targetSelection']
}
configList = [
'presignedUrlConfig', 'jobExecutionsRolloutConfig', 'abortConfig',
'timeoutConfig'
]
# When this happens
status, err = self.iot_interface.create_job(deployConfig)
# Expect these behaviors
self.assertTrue(status)
self.iot_interface.client.create_job.assert_called_with(**kwargs)
# add nore configs into test
for configName in configList:
with self.subTest(configName=configName):
deployConfig[configName] = configMap[configName]
kwargs[configName] = configMap[configName]
# When this happens
status, err = self.iot_interface.create_job(deployConfig)
# Expect these behaviors
self.assertTrue(status)
assert not err
self.iot_interface.client.create_job.assert_called_with(
**kwargs)
def test_create_job_exception(self):
self.iot_interface.client.create_job = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
deployConfig = {}
deployConfig['defaultConfig'] = configMap['defaultConfig']
kwargs = {
'jobId': TEST_DEFAULT_CONFIG['jobId'],
'targets': TEST_DEFAULT_CONFIG['thingArnList'],
'documentSource': TEST_DEFAULT_CONFIG['jobDocumentSrc'],
'targetSelection': TEST_DEFAULT_CONFIG['targetSelection']
}
configList = [
'presignedUrlConfig', 'jobExecutionsRolloutConfig', 'abortConfig',
'timeoutConfig'
]
for configName in configList:
deployConfig[configName] = configMap[configName]
kwargs[configName] = configMap[configName]
# When this happens
status, err = self.iot_interface.create_job(deployConfig)
# Expect these behaviors
self.assertFalse(status)
self.iot_interface.client.create_job.assert_called_with(**kwargs)
def test_get_job_info(self):
# Given these inputs and expected outputs
jobId = JOD_ID
# When this happens
job_dsb, err = self.iot_interface.get_job_info(jobId=jobId)
# Expect these behaviors
assert job_dsb == TEST_EXPTECTED_DSB_JOB_RSP['job']
assert not err
self.iot_interface.client.describe_job.assert_called_with(jobId=jobId)
def test_get_job_info_exception(self):
self.iot_interface.client.describe_job = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
jobId = JOD_ID
# When this happens
job_dsb, err = self.iot_interface.get_job_info(jobId=jobId)
# Expect these behaviors
assert job_dsb is None
self.iot_interface.client.describe_job.assert_called_with(jobId=jobId)
def test_get_job_exe_info(self):
# Given these inputs and expected outputs
jobId = JOD_ID
thingName = THING_NAME
# When this happens
job_exe_dsb, err = self.iot_interface.get_job_exe_info(
jobId=jobId, thingName=thingName)
# Expect these behaviors
assert job_exe_dsb == TEST_EXPTECTED_DSB_JOB_EXEC_RSP['execution']
assert not err
self.iot_interface.client.describe_job_execution.assert_called_with(
jobId=jobId, thingName=thingName)
def test_get_job_exe_info_exception(self):
self.iot_interface.client.describe_job_execution = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
jobId = JOD_ID
thingName = THING_NAME
# When this happens
job_exe_dsb, err = self.iot_interface.get_job_exe_info(
jobId=jobId, thingName=thingName)
# Expect these behaviors
assert job_exe_dsb is None
self.iot_interface.client.describe_job_execution.assert_called_with(
jobId=jobId, thingName=thingName)
def test_cancel_job(self):
self.iot_interface.client.describe_job = Mock(side_effect=[
TEST_EXPTECTED_DSB_JOB_RSP_IN_PROGRESS,
TEST_EXPTECTED_DSB_JOB_RSP_IN_PROGRESS, TEST_EXPTECTED_DSB_JOB_RSP
])
# Given these inputs and expected outputs
jobId = JOD_ID
expectedCalls = [
call(jobId=jobId),
call(jobId=jobId),
call(jobId=jobId)
]
# When this happens
status, err = self.iot_interface.cancel_job(jobId=jobId)
# Expect these behaviors
self.assertTrue(status)
assert not err
self.iot_interface.client.cancel_job.assert_called_with(jobId=jobId)
self.iot_interface.client.describe_job.assert_has_calls(expectedCalls)
def test_cancel_job_exception(self):
self.iot_interface.client.cancel_job = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
jobId = JOD_ID
# When this happens
status, _ = self.iot_interface.cancel_job(jobId=jobId)
# Expect these behaviors
self.assertFalse(status)
self.iot_interface.client.cancel_job.assert_called_with(jobId=jobId)
def test_job_is_deleting_while_cancel_job_is_called(self):
self.iot_interface.client.describe_job = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
jobId = JOD_ID
# When this happens
status, err = self.iot_interface.cancel_job(jobId=jobId)
# Expect these behaviors
self.assertTrue(status)
assert not err
self.iot_interface.client.cancel_job.assert_called_with(jobId=jobId)
self.iot_interface.client.describe_job.assert_called_once_with(
jobId=jobId)
def test_delete_job(self):
self.iot_interface.client.describe_job = Mock(side_effect=[
TEST_EXPTECTED_DSB_JOB_RSP, TEST_EXPTECTED_DSB_JOB_RSP,
ClientError({}, 'test')
])
# Given these inputs and expected outputs
jobId = JOD_ID
expectedCalls = [
call(jobId=jobId),
call(jobId=jobId),
call(jobId=jobId)
]
# When this happens
status, err = self.iot_interface.delete_job(jobId=jobId)
# Expect these behaviors
self.assertTrue(status)
assert not err
self.iot_interface.client.delete_job.assert_called_with(jobId=jobId,
force=True)
self.iot_interface.client.describe_job.assert_has_calls(expectedCalls)
def test_delete_job_exception(self):
self.iot_interface.client.delete_job = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
jobId = JOD_ID
# When this happens
status, _ = self.iot_interface.delete_job(jobId=jobId)
# Expect these behaviors
self.assertFalse(status)
self.iot_interface.client.delete_job.assert_called_with(jobId=jobId,
force=True)
def test_delete_stream(self):
self.iot_interface.client.describe_stream = Mock(side_effect=[
TEST_EXPTECTED_DSB_STREAM_RSP, TEST_EXPTECTED_DSB_STREAM_RSP,
ClientError({}, 'test')
])
# Given these inputs and expected outputs
streamId = STREAM_ID
expectedCalls = [
call(streamId=streamId),
call(streamId=streamId),
call(streamId=streamId)
]
# When this happens
status, err = self.iot_interface.delete_stream(streamId=streamId)
# Expect these behaviors
self.assertTrue(status)
assert not err
self.iot_interface.client.delete_stream.assert_called_with(
streamId=streamId)
self.iot_interface.client.describe_stream.assert_has_calls(
expectedCalls)
def test_delete_stream_exception(self):
self.iot_interface.client.delete_stream = Mock(
side_effect=ClientError({}, 'test'))
# Given these inputs and expected outputs
streamId = STREAM_ID
# When this happens
status, _ = self.iot_interface.delete_stream(streamId=streamId)
# Expect these behaviors
self.assertFalse(status)
self.iot_interface.client.delete_stream.assert_called_with(
streamId=streamId)
class monitor_tool():
def __init__(self):
thingArnList, deviceCount, thingNameList = parse_thingList(thingListFilePath)
if deviceCount < 1:
raise Exception('thing list should not be empty')
self.thingArnList = thingArnList
self.deviceCount = deviceCount
self.thingNameList = thingNameList
self.thingListUpdateTime = 0
self.iot_interface = IotInterface(region)
def monitor_start(self):
isParseList = False
job_complete_counter = 0
while True:
fileUpdateTime = path.getmtime(thingListFilePath)
if fileUpdateTime > self.thingListUpdateTime or isParseList is True:
thingArnList, deviceCount, thingNameList = parse_thingList(thingListFilePath)
self.thingListUpdateTime = path.getmtime(thingListFilePath)
isParseList = False
if len(thingArnList) < 1 or deviceCount < 1:
raise Exception('thing list should not be empty')
if deviceCount > self.deviceCount:
self.thingArnList = thingArnList
self.deviceCount = deviceCount
self.thingNameList = thingNameList
job_dsb, err = self.iot_interface.get_job_info(jobId)
if job_dsb:
status = job_dsb.get('status')
else:
continue
if err:
logging.error(err)
return
if status == 'COMPLETED':
isParseList = True
numberOfSucceededThings = job_dsb.get('jobProcessDetails', {}).get('numberOfSucceededThings')
if numberOfSucceededThings >= self.deviceCount:
job_complete_counter = job_complete_counter + 1
logging.info('deviceCount: %d matches numberOfSucceededThings: %d job completed with success', self.deviceCount, numberOfSucceededThings)
logging.info('jobId : %s completed , job_complete_counter: %d', jobId, job_complete_counter)
logging.info('jobId %s completed', jobId)
else:
logging.info('deviceCount: %d does not matches numberOfSucceededThings: %d job completed with failure', self.deviceCount, numberOfSucceededThings)
break
elif status == 'IN_PROGRESS':
logging.info('jobId %s IN_PROGRESS', jobId)
thingNameList = self.thingNameList
for thingName in thingNameList:
job_exe_dsb, err = self.iot_interface.get_job_exe_info(jobId, thingName)
if job_exe_dsb:
thing_status = job_exe_dsb.get('status')
if thing_status == 'SUCCEEDED':
self.thingNameList.remove(thingName)
else:
logging.info('thing name: %s status: %s statusDetails %s: ', thingName, thing_status, job_exe_dsb['statusDetails'])
elif status == 'CANCELED' or status == 'DELETION_IN_PROGRESS' or status == 'DELETION_IN_PROGRESS':
logging.info('unexpected failure with status: %s', status)
isParseList = True
else:
logging.info('unexpected status: %s', status)
break
time.sleep(defaultDelay)