def find_runs_in_directory(directory):
"""Find and validate all runs the specified directory.
Filters out any runs in the directory that have already
been uploaded. The filter is silent, so no warnings are emitted
if there is an uploaded run that's found in the directory.
Arguments:
directory -- the directory to find sequencing runs
Returns: a list of populated sequencing run objects found
in the directory, ready to be uploaded.
"""
logging.info("looking for sample sheet in {}".format(directory))
sample_sheets = find_file_by_name(directory=directory,
name_pattern='^SampleSheet.csv$',
depth=2)
logging.info("found sample sheets: {}".format(", ".join(sample_sheets)))
# filter directories that have been completely uploaded
sheets_to_upload = filter(
lambda dir: not run_is_uploaded(path.dirname(dir)), sample_sheets)
logging.info("filtered sample sheets: {}".format(
", ".join(sheets_to_upload)))
sequencing_runs = [
process_sample_sheet(sheet) for sheet in sheets_to_upload
]
send_message(DirectoryScannerTopics.finished_run_scan)
return sequencing_runs
def validate_run(sequencing_run):
"""Do the validation on a run, its samples, and files.
This function is kinda yucky because the validators should be raising
exceptions instead of returning validation objects...
Arguments:
sequencing_run -- the run to validate
"""
sample_sheet = sequencing_run.sample_sheet
validation = validate_sample_sheet(sequencing_run.sample_sheet)
if not validation.is_valid():
send_message(sequencing_run.offline_validation_topic,
run=sequencing_run,
errors=validation.get_errors())
raise SampleSheetError(
'Sample sheet {} is invalid. Reason:\n {}'.format(
sample_sheet, validation.get_errors()),
validation.error_list())
validation = validate_sample_list(sequencing_run.sample_list)
if not validation.is_valid():
raise SampleError(
'Sample sheet {} is invalid. Reason:\n {}'.format(
sample_sheet, validation.get_errors()),
validation.error_list())
def process_sample_sheet(sample_sheet):
"""Create a SequencingRun object for the specified sample sheet.
Arguments:
sample_sheet -- a `SampleSheet.csv` file that refers to an Illumina
MiSeq sequencing run.
Returns: an individual SequencingRun object for the sample sheet,
ready to be uploaded.
"""
try:
logging.info("going to parse metadata")
run_metadata = parse_metadata(sample_sheet)
logging.info("going to parse samples")
samples = complete_parse_samples(sample_sheet)
logging.info("going to build sequencing run")
sequencing_run = SequencingRun(run_metadata, samples, sample_sheet)
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
send_message(DirectoryScannerTopics.run_discovered, run=sequencing_run)
return sequencing_run
except SampleSheetError, e:
logging.exception("Failed to parse sample sheet.")
send_message(DirectoryScannerTopics.garbled_sample_sheet,
sample_sheet=sample_sheet,
error=e)
def send_sequence_files(self, samples_list, upload_id=1):
"""
send sequence files found in each sample in samples_list
the files to be sent is in sample.get_files()
this function iterates through the samples in samples_list and send
them to _send_sequence_files() which actually makes the connection
to the api in order to send the data
arguments:
samples_list -- list containing Sample object(s)
upload_id -- the run to send the files to
returns a list containing dictionaries of the result of post request.
"""
json_res_list = []
file_size_list = self.get_file_size_list(samples_list)
self.size_of_all_seq_files = sum(file_size_list)
self.total_bytes_read = 0
self.start_time = time()
for sample in samples_list:
try:
json_res = self._send_sequence_files(sample, upload_id)
json_res_list.append(json_res)
except Exception, e:
logging.error(
"The upload failed for unexpected reasons, informing the UI."
)
send_message(sample.upload_failed_topic, exception=e)
raise
def _directory_chooser(self, event):
"""Open a directory chooser to select a directory other than default."""
logging.info("Going to open a folder chooser")
dir_dialog = wx.DirDialog(self, style=wx.DD_DIR_MUST_EXIST)
response = dir_dialog.ShowModal()
if response == wx.ID_OK:
logging.info("User selected directory [{}]".format(dir_dialog.GetPath()))
send_message(UploaderAppFrame.directory_selected_topic, directory=dir_dialog.GetPath())
def _on_close(self, evt=None):
pub.unsubscribe(self._field_changed,
SettingsDialog.field_changed_topic)
# only inform the UI that settings have changed if we're not being invoked
# to set up the initial configuration settings (i.e., the first run)
if type(evt) is wx.CommandEvent and not self._first_run:
logging.info(
"Closing settings dialog and informing UI to scan for sample sheets."
)
send_message(SettingsDialog.settings_closed_topic)
evt.Skip()
def __init__(self, parent=None, default_post_process=""):
wx.Panel.__init__(self, parent)
self._sizer = wx.BoxSizer(wx.HORIZONTAL)
task_label = wx.StaticText(self,
label="Task to run on successful upload")
task_label.SetToolTipString(
"Post-processing job to run after a run has been successfully uploaded to IRIDA."
)
self._sizer.Add(task_label,
flag=wx.ALIGN_CENTER_VERTICAL | wx.RIGHT,
border=5,
proportion=0)
task = wx.TextCtrl(self)
task.Bind(
wx.EVT_KILL_FOCUS,
lambda evt: send_message(SettingsDialog.field_changed_topic,
field_name="completion_cmd",
field_value=task.GetValue()))
task.SetValue(default_post_process)
self._sizer.Add(task, flag=wx.EXPAND, proportion=1)
self.SetSizerAndFit(self._sizer)
self.Layout()
def __init__(self, parent, sample, run, api):
"""Initialize the SamplePanel.
Args:
parent: the parent Window for this panel.
sample: the sample that should be displayed.
run: the run that this sample belongs to.
api: an initialized instance of an API for interacting with IRIDA.
"""
wx.Panel.__init__(self, parent)
self._sample = sample
self._timer = wx.Timer(self)
self._progress_value = 0
self._last_timer_progress = 0
self._progress = None
self._progress_max = sample.get_files_size()
self._sizer = wx.BoxSizer(wx.HORIZONTAL)
self._label = wx.StaticText(self, label=self._sample.get_id())
self._sizer.Add(self._label, proportion=2)
self._status_label = wx.StaticText(self, label="Validating...")
self._sizer.Add(self._status_label)
self.SetSizer(self._sizer)
self.Bind(wx.EVT_TIMER, self._update_progress_timer, self._timer)
self.Layout()
pub.subscribe(self._upload_completed, sample.upload_completed_topic)
pub.subscribe(self._upload_progress, sample.upload_progress_topic)
if not sample.already_uploaded:
pub.subscribe(self._validation_results, sample.online_validation_topic)
pub.subscribe(self._upload_started, sample.upload_started_topic)
pub.subscribe(self._upload_failed, sample.upload_failed_topic)
threading.Thread(target=project_exists, kwargs={"api": api, "project_id": sample.get_project_id(), "message_id": sample.online_validation_topic}).start()
else:
# this sample is already uploaded, so inform the run panel to move
# it down to the bottom of the list.
send_message(sample.upload_completed_topic, sample=sample)
self._status_label.Destroy()
self._upload_completed()
def __init__(self,
parent=None,
default_directory="",
monitor_directory=False):
wx.Panel.__init__(self, parent)
self._sizer = wx.BoxSizer(wx.HORIZONTAL)
directory_label = wx.StaticText(self, label="Default directory")
directory_label.SetToolTipString(
"Default directory to scan for uploads")
self._sizer.Add(directory_label,
flag=wx.ALIGN_CENTER_VERTICAL | wx.RIGHT,
border=5,
proportion=0)
directory = wx.DirPickerCtrl(self, path=default_directory)
self.Bind(
wx.EVT_DIRPICKER_CHANGED,
lambda evt: send_message(SettingsDialog.field_changed_topic,
field_name="default_dir",
field_value=directory.GetPath()))
self._sizer.Add(directory, flag=wx.EXPAND, proportion=1)
monitor_checkbox = wx.CheckBox(self,
label="Monitor directory for new runs?")
monitor_checkbox.SetValue(monitor_directory)
monitor_checkbox.SetToolTipString(
"Monitor the default directory for when the Illumina Software indicates that the analysis is complete and ready to upload (when CompletedJobInfo.xml is written to the directory)."
)
monitor_checkbox.Bind(
wx.EVT_CHECKBOX,
lambda evt: send_message(SettingsDialog.field_changed_topic,
field_name="monitor_default_dir",
field_value=str(monitor_checkbox.GetValue(
))))
self._sizer.Add(monitor_checkbox,
flag=wx.ALIGN_CENTER_VERTICAL | wx.LEFT,
border=5,
proportion=0)
self.SetSizerAndFit(self._sizer)
self.Layout()
def _online_validation(api, sequencing_run):
"""Do online validation for the specified sequencing run.
Arguments:
api -- the API object to use for interacting with the server
sequencing_run -- the run to validate
Publishes messages:
start_online_validation -- when running online validation
online_validation_failure (params: project_id, sample_id) -- when the online validation fails
"""
send_message("start_online_validation")
for sample in sequencing_run.sample_list:
if not project_exists(api, sample.get_project_id()):
send_message("online_validation_failure",
project_id=sample.get_project_id(),
sample_id=sample.get_id())
raise ProjectError(
"The Sample_Project: {pid} doesn't exist in IRIDA for Sample_Id: {sid}"
.format(sid=sample.get_id(), pid=sample.get_project_id()))
def _send_file(filename, parameter_name, bytes_read=0):
"""This function is a generator that yields a multipart form-data
entry for the specified file. This function will yield `read_size`
bytes of the specified file name at a time as the generator is called.
This function will also terminate generating data when the field
`self._stop_upload` is set.
Args:
filename: the file to read and yield in `read_size` chunks to
the server.
parameter_name: the form field name to send to the server.
bytes_read: used for sending messages to the UI layer indicating
the total number of bytes sent when sending the sample
to the server.
"""
# Send the boundary header section for the file
logging.info(
"Sending the boundary header section for {}".format(filename))
yield (
"\r\n--{boundary}\r\n"
"Content-Disposition: form-data; name=\"{parameter_name}\"; filename=\"{filename}\"\r\n"
"\r\n").format(boundary=boundary,
parameter_name=parameter_name,
filename=filename.replace("\\", "/"))
# Send the contents of the file, read_size bytes at a time until
# we've either read the entire file, or we've been instructed to
# stop the upload by the UI
logging.info("Starting to send the file {}".format(filename))
with open(filename, "rb", read_size) as fastq_file:
data = fastq_file.read(read_size)
while data and not self._stop_upload:
bytes_read += len(data)
send_message(sample.upload_progress_topic,
progress=bytes_read)
yield data
data = fastq_file.read(read_size)
logging.info("Finished sending file {}".format(filename))
if self._stop_upload:
logging.info("Halting upload on user request.")
def __init__(self, parent, sheets_directory):
"""Initalize InvalidSampleSheetsPanel.
Args:
parent: the owning Window
sheets_directory: the parent directory for searching sample sheets. This
argument is used in the error message that's displayed to the user to
tell them where to look to fix any issues.
"""
wx.Panel.__init__(self, parent)
self._sizer = wx.BoxSizer(wx.VERTICAL)
self.SetSizer(self._sizer)
header = wx.StaticText(
self, label=u"✘ Looks like some sample sheets are not valid.")
header.SetFont(wx.Font(18, wx.DEFAULT, wx.NORMAL, wx.BOLD))
header.SetForegroundColour(wx.Colour(255, 0, 0))
header.Wrap(350)
self._sizer.Add(header,
flag=wx.TOP | wx.BOTTOM | wx.ALIGN_CENTER,
border=5)
self._sizer.Add(wx.StaticText(
self,
label=wordwrap((
"I found the following sample sheets in {}, but I couldn't understand "
"their contents. Check these sample sheets in an editor outside "
"of the uploader, then click the 'Scan Again' button below."
).format(sheets_directory), 350, wx.ClientDC(self))),
flag=wx.ALIGN_CENTER | wx.TOP | wx.BOTTOM,
border=5)
self._errors_tree = wx.TreeCtrl(
self,
style=wx.TR_DEFAULT_STYLE | wx.TR_FULL_ROW_HIGHLIGHT
| wx.TR_LINES_AT_ROOT | wx.TR_HIDE_ROOT)
self._errors_tree_root = self._errors_tree.AddRoot("")
self._sizer.Add(self._errors_tree, flag=wx.EXPAND, proportion=1)
scan_again_button = wx.Button(self, label="Scan Again")
self.Bind(
wx.EVT_BUTTON,
lambda evt: send_message(SettingsDialog.settings_closed_topic),
id=scan_again_button.GetId())
self._sizer.Add(scan_again_button,
flag=wx.ALIGN_CENTER | wx.TOP | wx.BOTTOM,
border=5)
pub.subscribe(self._sample_sheet_error,
DirectoryScannerTopics.garbled_sample_sheet)
pub.subscribe(self._sample_sheet_error,
DirectoryScannerTopics.missing_files)
def on_created(self, event):
"""Overrides `on_created` in `FileSystemEventHandler` to filter on
file creation events for `CompletedJobInfo.xml`."""
if isinstance(event, FileCreatedEvent) and event.src_path.endswith(
'CompletedJobInfo.xml'):
logging.info(
"Observed new run in {}, telling the UI to start uploading it."
.format(event.src_path))
directory = os.path.dirname(event.src_path)
# tell the UI to clean itself up before observing new runs
send_message(DirectoryMonitorTopics.new_run_observed)
# this will send a bunch of events that the UI is listening for, but
# unlike the UI (which runs this in a separate thread), we're going to do this
# in our own thread and block on it so we can tell the UI to start
# uploading once we've finished discovering the run
find_runs_in_directory(directory)
# now tell the UI to start
send_message(DirectoryMonitorTopics.finished_discovering_run)
else:
logging.debug("Ignoring file event [{}] with path [{}]".format(
str(event), event.src_path))
def project_exists(api, project_id, message_id=None):
try:
proj_list = api.get_projects()
project = next(proj for proj in proj_list
if proj.get_id() == project_id)
if message_id:
send_message(message_id, project=project)
return True
except ConnectionError:
if message_id:
send_message(message_id, project=None)
except StopIteration:
if message_id:
send_message(message_id, project=None)
return False
def run(self):
"""Initiate upload. The upload happens serially, one run at a time."""
for run in self._runs:
upload_run_to_server(api=self._api, sequencing_run=run)
# once the run uploads are complete, we can launch the post-processing
# command
if self._post_processing_task:
send_message(RunUploaderTopics.started_post_processing)
logging.info("About to launch post-processing command: {}".format(
self._post_processing_task))
# blocks until the command is complete
try:
exit_code = os.system(self._post_processing_task)
except:
exit_code = 1
if not exit_code:
logging.info("Post-processing command is complete.")
send_message(RunUploaderTopics.finished_post_processing)
else:
logging.error(
"The post-processing command is reporting failure")
send_message(RunUploaderTopics.failed_post_processing)
def _username_changed(self, evt=None):
send_message(SettingsDialog.field_changed_topic,
field_name="username",
field_value=self._username.GetValue())
self._status_label_user.Restart()
evt.Skip()
def _client_secret_changed(self, evt=None):
send_message(SettingsDialog.field_changed_topic,
field_name="client_secret",
field_value=self._client_secret.GetValue())
self._client_secret_status_label.Restart()
evt.Skip()
def _field_changed(self, evt=None):
send_message(SettingsDialog.field_changed_topic,
field_name="baseurl",
field_value=self._url.GetValue())
self._status_label.Restart()
evt.Skip()
logging.info("going to validate sequencing run")
validate_run(sequencing_run)
send_message(DirectoryScannerTopics.run_discovered, run=sequencing_run)
return sequencing_run
except SampleSheetError, e:
logging.exception("Failed to parse sample sheet.")
send_message(DirectoryScannerTopics.garbled_sample_sheet,
sample_sheet=sample_sheet,
error=e)
except SampleError, e:
logging.exception("Failed to parse sample.")
send_message(DirectoryScannerTopics.garbled_sample_sheet,
sample_sheet=sample_sheet,
error=e)
except SequenceFileError as e:
logging.exception("Failed to find files for sample sheet.")
send_message(DirectoryScannerTopics.missing_files,
sample_sheet=sample_sheet,
error=e)
return None
def validate_run(sequencing_run):
"""Do the validation on a run, its samples, and files.
This function is kinda yucky because the validators should be raising
exceptions instead of returning validation objects...
def connect_to_irida():
"""Connect to IRIDA for online validation.
Returns:
A configured instance of API.apiCalls.
"""
client_id = read_config_option("client_id")
client_secret = read_config_option("client_secret")
baseURL = read_config_option("baseURL")
username = read_config_option("username")
password = read_config_option("password")
try:
# Several threads might be attempting to connect at the same time, so lock
# the connection step, but **do not** block (acquire(False) means do not block)
# and just return if someone else is already trying to connect.
if lock.acquire(False):
logging.info("About to try connecting to IRIDA.")
api = ApiCalls(client_id, client_secret, baseURL, username, password)
send_message(APIConnectorTopics.connection_success_topic, api=api)
return api
else:
logging.info("Someone else is already trying to connect to IRIDA.")
except ConnectionError as e:
logging.info("Got a connection error when trying to connect to IRIDA.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {}. The server might be down. Make "
"sure that the connection address is correct (you can change the "
"address by clicking on the 'Open Settings' button below) and try"
" again, try again later, or contact an administrator."
).format(baseURL))
raise
except (SyntaxError, ValueError) as e:
logging.info("Connected, but the response was garbled.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {}. The server is up, but I "
"didn't understand the response. Make sure that the connection "
"address is correct (you can change the address by clicking on "
"the 'Open Settings' button below) and try again, try again"
" later, or contact an administrator."
).format(baseURL))
raise
except KeyError as e:
logging.info("Connected, but the OAuth credentials are wrong.", exc_info=True)
# this is credentials related, but let's try to figure out why the server
# is telling us that we can't log in.
message = str(e.message)
if "Bad credentials" in message:
topic = APIConnectorTopics.connection_error_user_credentials_topic
# if we're getting bad credentials, then that means the API is allowing
# us to try authenticate with a username and password, so our client id
# and secret are both correct:
send_message(APIConnectorTopics.connection_success_valid_client_secret)
elif "clientId does not exist" in message:
topic = APIConnectorTopics.connection_error_client_id_topic
elif "Bad client credentials" in message:
topic = APIConnectorTopics.connection_error_client_secret_topic
# if we're getting a bad client secret message, that means that the
# client ID is valid.
send_message(APIConnectorTopics.connection_success_valid_client_id)
else:
topic = APIConnectorTopics.connection_error_credentials_topic
send_message(topic, error_message=(
"We couldn't connect to IRIDA at {}. The server is up, but it's "
"reporting that your credentials are wrong. Click on the 'Open Settings'"
" button below and check your credentials, then try again. If the "
"connection still doesn't work, contact an administrator."
).format(baseURL))
# in spite of it all, this means that we're probably actually trying to connect
# to a real IRIDA server, so let the settings dialog know that it can render
# a success icon beside the URL
send_message(APIConnectorTopics.connection_success_valid_url)
raise
except URLError as e:
logging.info("Couldn't connect to IRIDA because the URL is invalid.", exc_info=True)
send_message(APIConnectorTopics.connection_error_url_topic, error_message=(
"We couldn't connect to IRIDA at {} because it isn't a valid URL. "
"Click on the 'Open Settings' button below to enter a new URL and "
"try again."
).format(baseURL))
raise
except:
logging.info("Some other kind of error happened.", exc_info=True)
send_message(APIConnectorTopics.connection_error_topic, error_message=(
"We couldn't connect to IRIDA at {} for an unknown reason. Click "
"on the 'Open Settings' button below to check the URL and your "
"credentials, then try again. If the connection still doesn't "
"work, contact an administrator."
).format(baseURL))
raise
finally:
try:
lock.release()
except:
pass
def Destroy(self):
self._upload_thread.join()
send_message(DirectoryMonitorTopics.shut_down_directory_monitor)
wx.Panel.Destroy(self)
def _send_sequence_files(self, sample, upload_id):
"""
post request to send sequence files found in given sample argument
raises error if either project ID or sample ID found in Sample object
doesn't exist in irida
arguments:
sample -- Sample object
upload_id -- the run to upload the files to
returns result of post request.
"""
json_res = {}
self._stop_upload = False
try:
project_id = sample.get_project_id()
proj_URL = self.get_link(self.base_URL, "projects")
samples_url = self.get_link(proj_URL,
"project/samples",
targ_dict={
"key": "identifier",
"value": project_id
})
except StopIteration:
raise ProjectError("The given project ID: " + project_id +
" doesn't exist")
try:
sample_id = sample.get_id()
seq_url = self.get_link(samples_url,
"sample/sequenceFiles",
targ_dict={
"key": "sampleName",
"value": sample_id
})
except StopIteration:
raise SampleError(
"The given sample ID: {} doesn't exist".format(sample_id), [
"No sample with name [{}] exists in project [{}]".format(
sample_id, project_id)
])
boundary = "B0undary"
read_size = 32768
def _send_file(filename, parameter_name, bytes_read=0):
"""This function is a generator that yields a multipart form-data
entry for the specified file. This function will yield `read_size`
bytes of the specified file name at a time as the generator is called.
This function will also terminate generating data when the field
`self._stop_upload` is set.
Args:
filename: the file to read and yield in `read_size` chunks to
the server.
parameter_name: the form field name to send to the server.
bytes_read: used for sending messages to the UI layer indicating
the total number of bytes sent when sending the sample
to the server.
"""
# Send the boundary header section for the file
logging.info(
"Sending the boundary header section for {}".format(filename))
yield (
"\r\n--{boundary}\r\n"
"Content-Disposition: form-data; name=\"{parameter_name}\"; filename=\"{filename}\"\r\n"
"\r\n").format(boundary=boundary,
parameter_name=parameter_name,
filename=filename.replace("\\", "/"))
# Send the contents of the file, read_size bytes at a time until
# we've either read the entire file, or we've been instructed to
# stop the upload by the UI
logging.info("Starting to send the file {}".format(filename))
with open(filename, "rb", read_size) as fastq_file:
data = fastq_file.read(read_size)
while data and not self._stop_upload:
bytes_read += len(data)
send_message(sample.upload_progress_topic,
progress=bytes_read)
yield data
data = fastq_file.read(read_size)
logging.info("Finished sending file {}".format(filename))
if self._stop_upload:
logging.info("Halting upload on user request.")
def _send_parameters(parameter_name, parameters):
"""This function is a generator that yields a multipart form-data
entry with additional file metadata.
Args:
parameter_name: the form field name to use to send to the server.
parameters: a JSON encoded object with the metadata for the file.
"""
logging.info(
"Going to send parameters for {}".format(parameter_name))
yield (
"\r\n--{boundary}\r\n"
"Content-Disposition: form-data; name=\"{parameter_name}\"\r\n"
"Content-Type: application/json\r\n\r\n"
"{parameters}\r\n").format(boundary=boundary,
parameter_name=parameter_name,
parameters=parameters)
def _finish_request():
"""This function is a generator that yields the terminal boundary
entry for a multipart form-data upload."""
yield "--{boundary}--".format(boundary=boundary)
def _sample_upload_generator(sample):
"""This function accepts the sample and composes a series of generators
that are used to send the file contents and metadata for the sample.
Args:
sample: the sample to send to the server
"""
bytes_read = 0
file_metadata = sample.get_sample_metadata()
file_metadata["miseqRunId"] = str(upload_id)
file_metadata_json = json.dumps(file_metadata)
if sample.is_paired_end():
# Compose a collection of generators to send both files of a paired-end
# file set and the corresponding metadata
return itertools.chain(
_send_file(filename=sample.get_files()[0],
parameter_name="file1"),
_send_file(filename=sample.get_files()[1],
parameter_name="file2",
bytes_read=path.getsize(sample.get_files()[0])),
_send_parameters(parameter_name="parameters1",
parameters=file_metadata_json),
_send_parameters(parameter_name="parameters2",
parameters=file_metadata_json),
_finish_request())
else:
# Compose a generator to send the single file from a single-end
# file set and the corresponding metadata.
return itertools.chain(
_send_file(filename=sample.get_files()[0],
parameter_name="file"),
_send_parameters(parameter_name="parameters",
parameters=file_metadata_json),
_finish_request())
if sample.is_paired_end():
logging.info("sending paired-end file")
url = self.get_link(seq_url, "sample/sequenceFiles/pairs")
else:
logging.info("sending single-end file")
url = seq_url
send_message(sample.upload_started_topic)
logging.info("Sending files to [{}]".format(url))
response = self.session.post(
url,
data=_sample_upload_generator(sample),
headers={
"Content-Type":
"multipart/form-data; boundary={}".format(boundary)
})
if self._stop_upload:
logging.info(
"Upload was halted on user request, raising exception so that server upload status is set to error state."
)
raise SequenceFileError("Upload halted on user request.", [])
if response.status_code == httplib.CREATED:
json_res = json.loads(response.text)
logging.info(
"Finished uploading sequence files for sample [{}]".format(
sample.get_id()))
send_message(sample.upload_completed_topic, sample=sample)
else:
err_msg = ("Error {status_code}: {err_msg}\n").format(
status_code=str(response.status_code), err_msg=response.reason)
logging.info("Got an error when uploading [{}]: [{}]".format(
sample.get_id(), err_msg))
logging.info(response.text)
send_message(sample.upload_failed_topic, exception=e)
raise SequenceFileError(err_msg, [])
return json_res
def _password_changed(self, evt=None):
send_message(SettingsDialog.field_changed_topic,
field_name="password",
field_value=self._password.GetValue())
self._status_label_pass.Restart()
evt.Skip()
def upload_run_to_server(api, sequencing_run):
"""Upload a single run to the server.
Arguments:
api -- the API object to use for interacting with the server
sequencing_run -- the run to upload to the server
Publishes messages:
start_online_validation -- when running an online validation (checking project ids) starts
online_validation_failure (params: project_id, sample_id) -- when the online validation fails
start_checking_samples -- when initially checking to see which samples should be created on the server
start_uploading_samples (params: sheet_dir) -- when actually initiating upload of the samples in the run
finished_uploading_samples (params: sheet_dir) -- when uploading the samples has completed
"""
filename = path.join(sequencing_run.sample_sheet_dir, ".miseqUploaderInfo")
def _handle_upload_sample_complete(sample=None):
"""Handle the event that happens when a sample has finished uploading.
"""
if sample is None:
raise Exception("sample is required!")
with open(filename, "rb") as reader:
uploader_info = json.load(reader)
logging.info(uploader_info)
if not 'uploaded_samples' in uploader_info:
uploader_info['uploaded_samples'] = list()
uploader_info['uploaded_samples'].append(sample.get_id())
with open(filename, 'wb') as writer:
json.dump(uploader_info, writer)
logging.info("Finished updating info file.")
pub.unsubscribe(_handle_upload_sample_complete,
sample.upload_completed_topic)
# do online validation first.
_online_validation(api, sequencing_run)
# then do actual uploading
if not path.exists(filename):
logging.info("Going to create a new sequencing run on the server.")
run_on_server = api.create_seq_run(sequencing_run.metadata)
run_id = run_on_server["resource"]["identifier"]
_create_miseq_uploader_info_file(sequencing_run.sample_sheet_dir,
run_id, "Uploading")
else:
logging.info("Resuming upload.")
with open(filename, "rb") as reader:
uploader_info = json.load(reader)
run_id = uploader_info['Upload ID']
send_message(RunUploaderTopics.start_checking_samples)
logging.info("Starting to check samples. [{}]".format(
len(sequencing_run.sample_list)))
# only send samples that aren't already on the server
samples_to_create = filter(lambda sample: not sample_exists(api, sample),
sequencing_run.sample_list)
logging.info("Sending samples to server: [{}].".format(", ".join(
[str(x) for x in samples_to_create])))
api.send_samples(samples_to_create)
for sample in sequencing_run.samples_to_upload:
pub.subscribe(_handle_upload_sample_complete,
sample.upload_completed_topic)
send_message("start_uploading_samples",
sheet_dir=sequencing_run.sample_sheet_dir,
skipped_sample_ids=[
sample.get_id()
for sample in sequencing_run.uploaded_samples
],
run_id=run_id)
send_message(sequencing_run.upload_started_topic)
logging.info("About to start uploading samples.")
try:
api.send_sequence_files(samples_list=sequencing_run.samples_to_upload,
upload_id=run_id)
send_message("finished_uploading_samples",
sheet_dir=sequencing_run.sample_sheet_dir)
send_message(sequencing_run.upload_completed_topic)
api.set_seq_run_complete(run_id)
_create_miseq_uploader_info_file(sequencing_run.sample_sheet_dir,
run_id, "Complete")
except Exception as e:
logging.exception(
"Encountered error while uploading files to server, updating status of run to error state."
)
api.set_seq_run_error(run_id)
raise
