def from_results_file_in_drive(
drive_service, protocol, cfg, results_folder_id, filename
):
"""Initialize processing results from just the name of the results file in drive"""
with drive.get_file_by_name(
drive_service,
results_folder_id,
filename,
drive.FindMode.MOST_RECENTLY_MODIFIED,
) as fh:
ProcessingResults.from_results_file(fh=fh, protocol=protocol)
def test_put_guess_mimetype(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_put_guess_mimetype.txt",
):
"""Puts a file, guessing the mimetype from the filename, and ensure we can
see the file."""
put_request = put_file(gdrive_service, gdrive_folder.id, filename)
with put_request as fh:
fh.write("hello world")
# find the file and retrieve the contents.
with get_file_by_name(gdrive_service, gdrive_folder.id, filename) as fh:
assert fh.read() == "hello world"
def test_put_read_after(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_put_read_after.txt",
):
"""Puts a file and ensure we can read what was written to the file."""
put_request = put_file(gdrive_service, gdrive_folder.id, filename)
with put_request as fh:
fh.write("hello world")
# find the file and retrieve the contents.
with get_file_by_name(gdrive_service, gdrive_folder.id, filename) as fh:
assert fh.read() == "hello world"
with put_request as fh:
assert fh.read() == "hello world"
def set_calls(self):
"""
Grabs the associated results.csv and stores accession data
"""
if self.qPCR_processing_complete:
# If processing complete get results file
with drive.get_file_by_name(
self.drive_service,
self.results_folder_id,
self.results_filename,
drive.FindMode.MOST_RECENTLY_MODIFIED,
) as fh:
self.calls = PCRResultsTracker.get_accession_to_call_mapping_from_results_csv(
fh
)
if self.calls:
logger.info(f"Found results file: {self.results_filename}")
def test_put_enforce_binary(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_put_enforce_binary.txt",
):
"""Puts a file, and ensure we can see the file."""
put_request = put_file(gdrive_service,
gdrive_folder.id,
filename,
binary=True)
with put_request as fh:
fh.write(b"hello world")
# find the file and retrieve the contents. because the content-type is
# guessed, we still get back a text file.
with get_file_by_name(gdrive_service, gdrive_folder.id, filename) as fh:
assert fh.read() == "hello world"
def test_get_put_has_name(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_get_put_has_name.txt",
):
"""Puts and gets files. Ensure they both have sane 'name' fields for the file
handle."""
put_request = put_file(gdrive_service, gdrive_folder.id, filename)
with put_request as fh:
fh.write("hello world")
assert fh.name is not None
assert put_request.fh.name is not None
# find the file and retrieve the contents.
with get_file_by_name(gdrive_service, gdrive_folder.id, filename) as fh:
assert fh.read() == "hello world"
assert fh.name is not None
def test_put_override_content_type(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_put_override_content_type.txt",
):
"""Puts a file, and ensure we can see the file."""
put_request = put_file(
gdrive_service,
gdrive_folder.id,
filename,
content_type="application/i-made-this-up",
)
with put_request as fh:
fh.write(b"hello world")
# find the file and retrieve the contents. because the content-type is
# set to a binary one, we should get back binary data.
with get_file_by_name(gdrive_service, gdrive_folder.id, filename) as fh:
assert fh.read() == b"hello world"
def test_get_put_http_client(
gdrive_service: DriveService,
gdrive_folder: DriveObject,
filename="test_get_put_http_client.txt",
):
"""Puts a file. Then retrieve the file using a custom HTTP client."""
put_request = put_file(gdrive_service, gdrive_folder.id, filename)
with put_request as fh:
fh.write("hello world")
assert fh.name is not None
assert put_request.fh.name is not None
# instantiate a new http client with the same credentials.
http = new_http_client_from_service(gdrive_service)
# find the file and retrieve the contents.
with get_file_by_name(gdrive_service,
gdrive_folder.id,
filename,
http=http) as fh:
assert fh.read() == "hello world"
assert fh.name is not None
def test_processing(gdrive_service: DriveService, gdrive_folder: DriveObject):
"""Test to validate the gdrive processing pipeline. This test has four phases:
1. Create directories on a test gdrive space for the files that are staged.
2. Stage the files so that it mimics the production environment. This includes
the logs, the plate layout files, and the control layout files.
3. Run the main processsing loop.
4. Verify that the output csv files are correct.
5. Verify that the expected markers and PDFs are present.
If this test fails, verify that all the data necessary to run the pipeline is staged
correctly."""
####################################################################################
# STEP 1: Create directories on a test gdrive space for the files that are staged.
# set up the test space
cfg = AlternateGDriveConfig(gdrive_folder.name)
# make the necessary input folders
logs_folder_id = mkdir_recursive(gdrive_service, "root",
cfg.PCR_LOGS_FOLDER)
plate_layout_folder_id = mkdir_recursive(gdrive_service, "root",
cfg.PLATE_LAYOUT_FOLDER)
# make the necessary output folders
mkdir_recursive(gdrive_service, "root", cfg.CSV_RESULTS_FOLDER)
cb_csv_folder_id = mkdir_recursive(gdrive_service, "root",
cfg.CHINA_BASIN_CSV_REPORTS_FOLDER)
final_results_folder_id = mkdir_recursive(gdrive_service, "root",
cfg.FINAL_REPORTS_FOLDER)
markers_folder_id = mkdir_recursive(gdrive_service, "root",
cfg.PCR_MARKERS_FOLDER)
####################################################################################
# STEP 2: Stage the files.
# copy all the files to the appropriate places
for filename in LOG_FILES:
mode = f"r{file_mode(filename)}"
with (EXAMPLE_FILE_DIR / filename).open(mode) as src_fh, put_file(
gdrive_service, logs_folder_id, filename) as dst_fh:
dst_fh.write(src_fh.read())
for filename in PLATE_LAYOUT_FILES:
mode = f"r{file_mode(filename)}"
with (EXAMPLE_FILE_DIR / filename).open(mode) as src_fh, put_file(
gdrive_service, plate_layout_folder_id, filename) as dst_fh:
dst_fh.write(src_fh.read())
####################################################################################
# STEP 3: Run the processing pipeline
processing(cfg, credentials_for_tests())
####################################################################################
# STEP 4: Verify the csv files.
for remote_filename, local_filename in EXPECTED_CSV_FILES.items():
with (EXAMPLE_FILE_DIR /
local_filename).open("r") as t1, get_file_by_name(
gdrive_service, cb_csv_folder_id, remote_filename) as t2:
rdr1 = csv.reader(t1)
rdr2 = csv.reader(t2)
for row1, row2, in zip(rdr1, rdr2):
assert (
row1 == row2
), f"mismatch between {local_filename} and {remote_filename}"
####################################################################################
# STEP 5: Verify that markers and PDFs were created and uploaded
marker_folder_contents = get_contents_by_folder_id(gdrive_service,
markers_folder_id,
only_files=True)
marker_set = {
marker_folder_entry.name
for marker_folder_entry in marker_folder_contents
}
assert marker_set == EXPECTED_MARKERS
pdf_folder_contents = get_contents_by_folder_id(gdrive_service,
final_results_folder_id,
only_files=True)
pdf_report_set = {
pdf_folder_entry.name
for pdf_folder_entry in pdf_folder_contents
}
assert pdf_report_set == EXPECTED_PDFS
