def execute(self, context):
requester = Requester(self.datastore_url, self.access_token)
response = requester.get(constants.PROJECT_ID_LIST_URL)
loaded_project_ids = response.json()
response = requester.get(constants.TRACE_ID_LIST_URL)
loaded_trace_ids = response.json()
trace_ids = {d["trace_id"]: d["id"] for d in loaded_trace_ids}
project_ids = {d["project_id"]: d["id"] for d in loaded_project_ids}
raw_matches = pd.read_csv(self.filename, dtype=str).to_dict('records')
data = []
row_count = 0
for match in raw_matches:
trace_id = trace_ids.get(match["trace_id"], None)
project_id = project_ids.get(match["project_id"], None)
if trace_id is not None and project_id is not None:
data.append({
"trace_id": trace_id,
"project_id": project_id,
})
if len(data) >= self.bulk_size:
response = requester.upload_chunk(constants.PROJECT_TRACE_MAPPING_BULK_UPSERT_VERBOSE_URL, data)
if response.status_code < 200 or response.status_code >= 300:
raise RuntimeError('Bad response attempting to upsert')
row_count += len(data)
logging.info("Loaded {} proj-trace maps, {} so far".format(len(data), row_count))
data = []
# leftovers
if len(data) > 0:
response = requester.upload_chunk(constants.PROJECT_TRACE_MAPPING_BULK_UPSERT_VERBOSE_URL, data)
if response.status_code < 200 or response.status_code >= 300:
raise RuntimeError('Bad response attempting to upsert')
row_count += len(data)
logging.info("Loaded {} proj-trace maps, {} so far".format(len(data), row_count))
logging.info("Completed loading {} proj-trace map records".format(row_count))
def loaded_project_ids():
requester = Requester(config.oeem.url, config.oeem.access_token)
response = requester.get(constants.PROJECT_ID_LIST_URL)
return response.json()
def loaded_trace_ids():
requester = Requester(config.oeem.url, config.oeem.access_token)
response = requester.get(constants.TRACE_ID_LIST_URL)
return response.json()