def test_get_db_credentials_exceptions(env_prefix, skipos, user, password, defaults, expected): if not skipos: os.environ[env_prefix + 'USER'] = user os.environ[env_prefix + 'PASSWORD'] = password try: with pytest.raises(qu.CredentialsException): qu.get_db_credentials(env_prefix=env_prefix, **defaults) finally: if not skipos: os.environ.pop(env_prefix + 'USER') os.environ.pop(env_prefix + 'PASSWORD')
def set_segmentation_run_id(self, data, **kwargs): """if segmentation_run_id not provided, query the database using the other provided args """ # If we already have a prod manifest no need to query DB if "prod_segmentation_run_manifest" in data: return data if "segmentation_run_id" not in data: if not all( i in data for i in ["ophys_experiment_id", "ophys_segmentation_commit_hash"]): raise TransformPipelineException( "if omitting arg segmentation_run_id, must " "include args ophys_experiment_id and " "ophys_segmentation_commit_hash") db_credentials = query_utils.get_db_credentials( env_prefix="LABELING_", **query_utils.label_defaults) db_connection = query_utils.DbConnection(**db_credentials) query_string = ( "SELECT id FROM segmentation_runs WHERE " f"ophys_experiment_id={data['ophys_experiment_id']} AND " "ophys_segmentation_commit_hash=" f"'{data['ophys_segmentation_commit_hash']}'") entries = db_connection.query(query_string) if len(entries) != 1: raise TransformPipelineException( f"{query_string} did not return exactly 1 result") data['segmentation_run_id'] = entries[0]['id'] return data
def test_get_db_credentials(env_prefix, user, password, defaults, expected): os.environ[env_prefix + 'USER'] = user os.environ[env_prefix + 'PASSWORD'] = password try: creds = qu.get_db_credentials(env_prefix=env_prefix, **defaults) assert creds == expected finally: os.environ.pop(env_prefix + 'USER') os.environ.pop(env_prefix + 'PASSWORD')
# LIMS is a little weird for not storing the actual movie path # check to be sure. raise SegmentationManifestException( f"for experiment {result['id']} LIMS has " f"nframes = {result['nframes']} " f"but the found video file {video_path} " f"has nframes = {nframes_h5}") manifest['manifest'].append({ 'experiment_id': result['id'], 'nbinned': int(result['nframes'] / self.args['bin_size']), 'input_video': str(video_path) }) self.output(manifest, indent=2) self.logger.info(f"wrote {self.args['output_json']}") if __name__ == "__main__": # pragma: no cover lims_credentials = qu.get_db_credentials( env_prefix="LIMS_", **qu.lims_defaults) lims_connection = qu.DbConnection(**lims_credentials) label_credentials = qu.get_db_credentials( env_prefix="LABELING_", **qu.label_defaults) label_connection = qu.DbConnection(**label_credentials) sm = SegmentationManifest() sm.run(lims_connection, label_connection)
manifest['full-video-source-ref'] = str(full_video_path) manifest['video-source-ref'] = str(sub_video_path) manifest['max-source-ref'] = str(max_proj_path) manifest['avg-source-ref'] = str(avg_proj_path) manifest['trace-source-ref'] = str(trace_path) manifest['full-outline-source-ref'] = str(full_outline_path) if 'output_manifest' in self.args: manifests.append(manifest) else: insert_str = insert_str_template.format( json.dumps(manifest), os.environ['TRANSFORM_HASH'], roi.roi_id) insert_statements.append(insert_str) if 'output_manifest' in self.args: with open(self.args['output_manifest'], "w") as fp: jsonlines.Writer(fp).write_all(manifests) else: db_conn.bulk_insert(insert_statements) if __name__ == "__main__": # pragma: no cover db_credentials = query_utils.get_db_credentials( env_prefix="LABELING_", **query_utils.label_defaults) db_connection = query_utils.DbConnection(**db_credentials) pipeline = TransformPipeline() pipeline.run(db_connection)