def _check_portal_job(self, etcd, fnames, portal_manifest, job_id): etcd_key = common.portal_job_etcd_key(portal_manifest.name, job_id) data = etcd.get_data(etcd_key) self.assertIsNotNone(data) portal_job = text_format.Parse(data, dp_pb.DataPortalJob()) self.assertEqual(job_id, portal_job.job_id) self.assertFalse(portal_job.finished) fnames.sort() fpaths = [os.path.join(portal_manifest.input_base_dir, f) for f in fnames if fnmatch(f, portal_manifest.input_file_wildcard)] self.assertEqual(len(fpaths), len(portal_job.fpaths)) for index, fpath in enumerate(fpaths): self.assertEqual(fpath, portal_job.fpaths[index])
def _update_processing_job(self, job): self._processing_job = None etcd_key = common.portal_job_etcd_key(self._portal_name, job.job_id) self._etcd.set_data(etcd_key, text_format.MessageToString(job)) self._processing_job = job
def _sync_portal_job(self, job_id): etcd_key = common.portal_job_etcd_key(self._portal_name, job_id) data = self._etcd.get_data(etcd_key) if data is not None: return text_format.Parse(data, dp_pb.DataPortalJob()) return None