def execute(self, context): requester = Requester(self.datastore_url, self.access_token) response = requester.get(constants.PROJECT_ID_LIST_URL) loaded_project_ids = response.json() response = requester.get(constants.TRACE_ID_LIST_URL) loaded_trace_ids = response.json() trace_ids = {d["trace_id"]: d["id"] for d in loaded_trace_ids} project_ids = {d["project_id"]: d["id"] for d in loaded_project_ids} raw_matches = pd.read_csv(self.filename, dtype=str).to_dict('records') data = [] row_count = 0 for match in raw_matches: trace_id = trace_ids.get(match["trace_id"], None) project_id = project_ids.get(match["project_id"], None) if trace_id is not None and project_id is not None: data.append({ "trace_id": trace_id, "project_id": project_id, }) if len(data) >= self.bulk_size: response = requester.upload_chunk(constants.PROJECT_TRACE_MAPPING_BULK_UPSERT_VERBOSE_URL, data) if response.status_code < 200 or response.status_code >= 300: raise RuntimeError('Bad response attempting to upsert') row_count += len(data) logging.info("Loaded {} proj-trace maps, {} so far".format(len(data), row_count)) data = [] # leftovers if len(data) > 0: response = requester.upload_chunk(constants.PROJECT_TRACE_MAPPING_BULK_UPSERT_VERBOSE_URL, data) if response.status_code < 200 or response.status_code >= 300: raise RuntimeError('Bad response attempting to upsert') row_count += len(data) logging.info("Loaded {} proj-trace maps, {} so far".format(len(data), row_count)) logging.info("Completed loading {} proj-trace map records".format(row_count))
def loaded_project_ids(): requester = Requester(config.oeem.url, config.oeem.access_token) response = requester.get(constants.PROJECT_ID_LIST_URL) return response.json()
def loaded_trace_ids(): requester = Requester(config.oeem.url, config.oeem.access_token) response = requester.get(constants.TRACE_ID_LIST_URL) return response.json()