def test_get_job_metrics(self, http_mock):
response = {
"user-task-accumulators": [{
"name":
"__metricscontainers",
"type":
"MetricsAccumulator",
"value":
"{\"metrics\": {\"attempted\": [{\"urn\": "
"\"metric_urn\", \"type\": \"beam:metrics:sum_int64:v1\", "
"\"payload\": \"AA==\", \"labels\": "
"{\"PTRANSFORM\": \"ptransform_id\"}}]}}"
}]
}
http_mock.get('http://flink/v1/jobs/flink_job_id/accumulators',
json=response)
options = pipeline_options.FlinkRunnerOptions()
job_server = flink_uber_jar_job_server.FlinkUberJarJobServer(
'http://flink', options)
job = flink_uber_jar_job_server.FlinkBeamJob('http://flink', None,
'job_id', 'job_name',
None, options)
job._flink_job_id = 'flink_job_id'
job_server._jobs['job_id'] = job
request = beam_job_api_pb2.GetJobMetricsRequest(job_id='job_id')
expected = beam_job_api_pb2.GetJobMetricsResponse(
metrics=beam_job_api_pb2.MetricResults(attempted=[{
"urn": "metric_urn",
"type": "beam:metrics:sum_int64:v1",
"payload": b'\000',
"labels": {
"PTRANSFORM": "ptransform_id"
}
}]))
actual = job_server.GetJobMetrics(request)
self.assertEqual(actual, expected)
def test_end_to_end(self, http_mock):
with temp_name(suffix='fake.jar') as fake_jar:
# Create the jar file with some trivial contents.
with zipfile.ZipFile(fake_jar, 'w') as zip:
with zip.open('FakeClass.class', 'w') as fout:
fout.write(b'[original_contents]')
options = pipeline_options.FlinkRunnerOptions()
options.flink_job_server_jar = fake_jar
job_server = flink_uber_jar_job_server.FlinkUberJarJobServer(
'http://flink', options)
# Prepare the job.
prepare_response = job_server.Prepare(
beam_job_api_pb2.PrepareJobRequest(
job_name='job', pipeline=beam_runner_api_pb2.Pipeline()))
channel = grpc.insecure_channel(
prepare_response.artifact_staging_endpoint.url)
retrieval_token = beam_artifact_api_pb2_grpc.ArtifactStagingServiceStub(
channel).CommitManifest(
beam_artifact_api_pb2.CommitManifestRequest(
staging_session_token=prepare_response.
staging_session_token,
manifest=beam_artifact_api_pb2.Manifest())
).retrieval_token
channel.close()
# Now actually run the job.
http_mock.post('http://flink/v1/jars/upload',
json={'filename': '/path/to/jar/nonce'})
http_mock.post('http://flink/v1/jars/nonce/run',
json={'jobid': 'some_job_id'})
job_server.Run(
beam_job_api_pb2.RunJobRequest(
preparation_id=prepare_response.preparation_id,
retrieval_token=retrieval_token))
# Check the status until the job is "done" and get all error messages.
http_mock.get('http://flink/v1/jobs/some_job_id/execution-result',
[{
'json': {
'status': {
'id': 'IN_PROGRESS'
}
}
}, {
'json': {
'status': {
'id': 'IN_PROGRESS'
}
}
}, {
'json': {
'status': {
'id': 'COMPLETED'
}
}
}])
http_mock.get('http://flink/v1/jobs/some_job_id',
json={'state': 'FINISHED'})
http_mock.delete('http://flink/v1/jars/nonce')
state_stream = job_server.GetStateStream(
beam_job_api_pb2.GetJobStateRequest(
job_id=prepare_response.preparation_id))
self.assertEqual([s.state for s in state_stream], [
beam_job_api_pb2.JobState.RUNNING,
beam_job_api_pb2.JobState.DONE
])
http_mock.get('http://flink/v1/jobs/some_job_id/exceptions',
json={
'all-exceptions': [{
'exception': 'exc_text',
'timestamp': 0
}]
})
message_stream = job_server.GetMessageStream(
beam_job_api_pb2.JobMessagesRequest(
job_id=prepare_response.preparation_id))
self.assertEqual([m for m in message_stream], [
beam_job_api_pb2.JobMessagesResponse(
message_response=beam_job_api_pb2.JobMessage(
message_id='message0',
time='0',
importance=beam_job_api_pb2.JobMessage.
MessageImportance.JOB_MESSAGE_ERROR,
message_text='exc_text')),
beam_job_api_pb2.JobMessagesResponse(
state_response=beam_job_api_pb2.GetJobStateResponse(
state=beam_job_api_pb2.JobState.DONE)),
])
def test_flink_version(self, http_mock):
http_mock.get('http://flink/v1/config',
json={'flink-version': '3.1.4.1'})
job_server = flink_uber_jar_job_server.FlinkUberJarJobServer(
'http://flink', pipeline_options.FlinkRunnerOptions())
self.assertEqual(job_server.flink_version(), "3.1")
def test_end_to_end(self, http_mock):
with temp_name(suffix='fake.jar') as fake_jar:
# Create the jar file with some trivial contents.
with zipfile.ZipFile(fake_jar, 'w') as zip:
with zip.open('FakeClass.class', 'w') as fout:
fout.write(b'[original_contents]')
options = pipeline_options.FlinkRunnerOptions()
options.flink_job_server_jar = fake_jar
job_server = flink_uber_jar_job_server.FlinkUberJarJobServer(
'http://flink', options)
plan = TestJobServicePlan(job_server)
# Prepare the job.
prepare_response = plan.prepare(beam_runner_api_pb2.Pipeline())
plan.stage(beam_runner_api_pb2.Pipeline(),
prepare_response.artifact_staging_endpoint.url,
prepare_response.staging_session_token)
# Now actually run the job.
http_mock.post('http://flink/v1/jars/upload',
json={'filename': '/path/to/jar/nonce'})
http_mock.post('http://flink/v1/jars/nonce/run',
json={'jobid': 'some_job_id'})
_, message_stream, state_stream = plan.run(
prepare_response.preparation_id)
# Check the status until the job is "done" and get all error messages.
http_mock.get('http://flink/v1/jobs/some_job_id/execution-result',
[{
'json': {
'status': {
'id': 'IN_PROGRESS'
}
}
}, {
'json': {
'status': {
'id': 'IN_PROGRESS'
}
}
}, {
'json': {
'status': {
'id': 'COMPLETED'
}
}
}])
http_mock.get('http://flink/v1/jobs/some_job_id',
json={'state': 'FINISHED'})
http_mock.delete('http://flink/v1/jars/nonce')
self.assertEqual([s.state for s in state_stream], [
beam_job_api_pb2.JobState.STOPPED,
beam_job_api_pb2.JobState.RUNNING,
beam_job_api_pb2.JobState.RUNNING,
beam_job_api_pb2.JobState.DONE
])
http_mock.get('http://flink/v1/jobs/some_job_id/exceptions',
json={
'all-exceptions': [{
'exception': 'exc_text',
'timestamp': 0
}]
})
def get_item(x):
if x.HasField('message_response'):
return x.message_response
else:
return x.state_response.state
self.assertEqual([get_item(m) for m in message_stream], [
beam_job_api_pb2.JobState.STOPPED,
beam_job_api_pb2.JobState.RUNNING,
beam_job_api_pb2.JobMessage(
message_id='message0',
time='0',
importance=beam_job_api_pb2.JobMessage.MessageImportance.
JOB_MESSAGE_ERROR,
message_text='exc_text'),
beam_job_api_pb2.JobState.DONE,
])