def send_node_alert(self, node, msg=''): """ 发送节点错误信息 :param node: :param msg: :return: """ logger.error("{} {}".format(node.id, msg)) self_node = models.Node.objects.filter(agent=node.agent, role='self').first() if not self_node or not self_node.notify_when_lose_children: # "当下级失联时告警"设置为否时不告警 return title = "中心链接错误" node_type = '分中心' if node.type == 'sub_center' else '子中心' body = msg if msg else "下级{node_type}-{node_name}({node_ip})无法连接,数据和配置将无法同步,请及时处理".format( **{ "node_type": node_type, "node_name": node.name, "node_ip": node.ip }) userinfos = node.agent.userinfo_set.filter(is_admin=1) user_emails = [ui.user.email for ui in userinfos] for email in user_emails: send_msg(msg_type=4, to=email, content=body, title=title, agent=node.agent)
def exception_error(step, desc, conf): """Handle an exception. Arguments: step: error step desc: description of the section error conf: configuration object run_id: run_id """ exp_info = sys.exc_info() exception_msg = str(exp_info[0]) + ' (' + str(exp_info[1]) + ')' traceback_msg = traceback.format_exc(exp_info[2]) short_message = desc + ', catch exception ' + exception_msg message = desc + ', catch exception ' + exception_msg + \ '\n Stacktrace: \n' + traceback_msg if step == 'sync': sync_run.error(short_message, message, conf) elif step == 'demux': demux_run.error(short_message, message, conf) elif step == 'qc': qc_run.error(short_message, message, conf) else: # Log the exception common.log('SEVERE', 'Exception: ' + exception_msg, conf) common.log('WARNING', traceback_msg.replace('\n', ' '), conf) # Send a mail with the exception common.send_msg("[Aozan] Exception: " + exception_msg, traceback_msg, True, conf)
def send_command(name: CommandEnum, data: str = None) -> Optional[dict]: data = get_command(name, data) data = bytes(data, 'utf-8') # Публичный ключ передается в не зашифрованном виде if name != CommandEnum.SEND_PUBLIC_KEY: print(f'[*] Sending raw ({len(data)}): {data}') data = DATA['info_security'].encrypt(data) print(f'[+] Sending ({len(data)}): {data}') send_msg(sock, data) print('[+] Receiving...') response_data = recv_msg(sock) if not response_data: return print(f'[+] Response ({len(response_data)}): {response_data}') # AES ключ, зашифрованный публичным ключом, передается в не зашифрованном виде if name != CommandEnum.SEND_PUBLIC_KEY: response_data = DATA['info_security'].decrypt(response_data) print(f'[*] Response raw ({len(response_data)}): {response_data}') rs = json.loads(response_data, encoding='utf-8') command = CommandEnum[rs['command']] if command != name: raise Exception(f'Received response to another command. ') return rs
def send_command(command: str) -> str: with socket.socket() as sock: sock.connect((HOST, PORT)) data = bytes(command, 'utf-8') send_msg(sock, data) response_data = recv_msg(sock) return str(response_data, 'utf-8')
def handle(self): print('Connected:', self.client_address) data = recv_msg(self.request) print('Receiving ({}): {}'.format(len(data), data)) print('Sending') send_msg(self.request, data.upper()) print('Close\n')
def listen_client(self, client, address): while True: request = self._get_request(client) if request: response = self._generate_responce(request) send_msg(client, response) else: self.close_client(client) return False
def listen_client(self, client, address): while True: request = self._get_request(client) print("request: {}".format(request)) if request: response = self._generate_responce(request) send_msg(client, response) else: client.close() return False
def listenServer(self): while True: try: time.sleep(3) send_msg(self.sock, "ping *") recv_msg(self.sock) except: if self.sock._closed: return None self.changeStatus(connected=False) self.connectionRefused.emit() return None
def demux_run_standalone(run_id, input_run_data_path, fastq_output_dir, samplesheet_csv_path, nb_mismatch, conf): """ Demultiplexing the run with bcl2fastq on version parameter. Arguments: run_id: The run id input_run_data_path: input run data path to demultiplexing fastq_output_dir: fastq directory to save result on demultiplexing samplesheet_csv_path: samplesheet path in csv format, version used by bcl2fastq conf: configuration dictionary """ bcl2fastq_executable_path = conf[BCL2FASTQ_PATH_KEY] tmp = conf[TMP_PATH_KEY] run_id_msg = " for run " + run_id + ' on ' + common.get_instrument_name(run_id, conf) bcl2fastq_log_file = tmp + "/bcl2fastq_output_" + run_id + ".err" # Check if the bcl2fastq path is OK if os.path.isdir(bcl2fastq_executable_path): bcl2fastq_executable_path += '/bcl2fastq' elif not os.path.isfile(bcl2fastq_executable_path): error("Error while setting executable command file bcl2fastq" + run_id_msg + ", invalid bcl2fastq path: " + bcl2fastq_executable_path, "Error while setting executable command file bcl2fastq" + run_id_msg + ", invalid bcl2fastq path: " + bcl2fastq_executable_path, conf) return False cmd = create_bcl2fastq_command_line(run_id, bcl2fastq_executable_path, input_run_data_path, fastq_output_dir, samplesheet_csv_path, tmp, nb_mismatch, conf) common.log('INFO', 'Demultiplexing in standalone mode using the following command line: ' + str(cmd), conf) exit_code = os.system(cmd) if exit_code != 0: error("Error while executing bcl2fastq " + run_id_msg, 'Error while executing bcl2fastq (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd, conf) msg = 'Error while executing bcl2fastq ' + run_id_msg + ' (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd # Check if the log file has been generated if not os.path.exists(bcl2fastq_log_file): error("Error with bcl2fastq log for run " + run_id + ".", "No bcl2fastq log available", conf) common.send_msg('[Aozan] Failed demultiplexing ' + run_id_msg, msg, True, conf) else: msg += "\n\nPlease check the attached bcl2fastq output error file." common.send_msg_with_attachment('[Aozan] Failed demultiplexing ' + run_id_msg, msg, bcl2fastq_log_file, True, conf) return False return True
def _send_message(self): logging.info('message sender started.') while not self._stopped: try: remote_addr, msg = self._sendq.get() sock = self._agentsocks.get(remote_addr, None) if sock is None: logging.warn('agent socket not found by %s', remote_addr) else: logging.info('pub msg to agent %s', remote_addr) send_msg(sock, msg) except Exception as e: logging.exception('error while send message to %s', remote_addr) self.stop_agent(remote_addr) logging.info('message sender stopped')
def process_connect(conn, addr): print(f"[+] New connection from {addr}") try: while True: data = recv_msg(conn) if not data: break print(f'[+] Receiving ({len(data)}): {data}') # Проверка, что этот запрос уже не первый, т.к. то, что AES уже есть # и что, нужно расшифровавывать запрос is_existing_connect = conn in CONNECTION_BY_KEY if is_existing_connect: data = CONNECTION_BY_KEY[conn].decrypt(data) print(f'[*] Receiving raw ({len(data)}): {data}') rs = process_command(data, conn, addr) print(f'[*] Sending raw ({len(rs)}): {rs}') rs = CONNECTION_BY_KEY[conn].encrypt(rs) else: key_AES = rsa.decrypt(data, PRIVATE_KEY) print('key_AES:', key_AES) CONNECTION_BY_KEY[conn] = InfoSecurity(key_AES) rs = b'' print(f'[+] Sending ({len(rs)}): {rs}') send_msg(conn, rs) print() except: import traceback print(traceback.format_exc()) finally: conn.close() if conn in CONNECTION_BY_KEY: CONNECTION_BY_KEY.pop(conn) print(f"[+] Closed connection from {addr}") print()
def send_mail_if_critical_free_space_available(conf): """Check if disk free space is critical. If true send a mail. Arguments: conf: configuration dictionary """ for path in get_hiseq_data_paths(conf): if os.path.exists(path): df = common.df(path) free_space_threshold = long(conf[HISEQ_CRITICAL_MIN_SPACE_KEY]) if df < free_space_threshold: common.send_msg('[Aozan] Critical: Not enough disk space on sequencer storage for current run', 'There is only %.2f' % (df / (1024 * 1024 * 1024)) + ' Gb left for run storage in ' + path + '. ' + ' The current warning threshold is set to %.2f' % ( free_space_threshold / (1024 * 1024 * 1024)) + ' Gb.', False, conf)
def send_failed_run_message(run_id, secs, conf): """Send a mail to inform about a failed run. Arguments: conf: configuration dictionary """ run_path = hiseq_run.find_hiseq_run_path(run_id, conf) file_to_test = run_path + '/' + run_id + '/RTAComplete.txt' last = os.stat(file_to_test).st_mtime df = common.df(run_path) / (1024 * 1024 * 1024) du = common.du(run_path + '/' + run_id) / (1024 * 1024 * 1024) common.send_msg('[Aozan] Failed run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'A run (' + run_id + ') has failed on ' + common.get_instrument_name(run_id, conf) + ' at ' + common.time_to_human_readable(last) + '.\n' + 'Data for this run can be found at: ' + run_path + '\n\nFor this task %.2f GB has been used and %.2f GB still free.' % (du, df), False, conf)
def _do_write(self, sock): while not self._queue.empty(): try: msg = self._queue.get_nowait() except Q.Empty: logging.warn('Try to get msg from empty queue..') return msg.set_header(msg.H_SEND_AT, datetime.utcnow()) size, times = send_msg(sock, msg) logging.info('msg %s sent to %s use %d times', msg, self._master_addr, times) logging.debug('msg data = %s', msg.body)
def send_command(data: bytes = None, key: InfoSecurity = None) -> Optional[dict]: print(f'[+] Sending raw ({len(data)}): {data}') if key: data = key.encrypt(data) print(f'[+] Sending ({len(data)}): {data}') send_msg(sock, data) print('[+] Receiving...') response_data = recv_msg(sock) if not response_data: return print(f'[+] Response ({len(response_data)}): {response_data}') if key: response_data = key.decrypt(response_data) print(f'[*] Response raw ({len(response_data)}): {response_data}') rs = json.loads(response_data, encoding='utf-8') return rs
def send_mail_if_recent_run(run_id, secs, conf): """Send an email to inform that a new run is finished. Arguments: run_id: run id secs: maximum delay since the end of the run conf: configuration object """ run_path = hiseq_run.find_hiseq_run_path(run_id, conf) if run_path is False: return last = hiseq_run.check_end_run_since(run_id, secs, conf) if last > 0: df = common.df(run_path) / (1024 * 1024 * 1024) du = common.du(run_path + '/' + run_id) / (1024 * 1024 * 1024) common.send_msg('[Aozan] Ending run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'A new run (' + run_id + ') is finished on ' + common.get_instrument_name(run_id, conf) + ' at ' + common.time_to_human_readable( last) + '.\n' + 'Data for this run can be found at: ' + run_path + '\n\nFor this task %.2f GB has been used and %.2f GB still free.' % (du, df), False, conf)
def sendRequest(self, request): send_msg(self.sock, request) response = recv_msg(self.sock) return response
if __name__ == '__main__': parser = opt_parser() opt = parser.parse_args() config = load_config(opt.net_config) #pprint(config) # Start server here server = Server(host=opt.host, port=opt.port, size=len(config)) server.start() time.sleep(0.1) fetch_msg = encode(dict(type="extract", host=opt.host, port=opt.port)) for node in config.values(): send_msg(fetch_msg, host=node['host'], port=node['port']) #print(node) time.sleep(0.1) # wait for server here server.join() id_table = dict() while not server.responses.empty(): resp = server.responses.get() print(repr(resp)) id_table[resp['id']] = resp graph = as_nx(id_table) nx.draw(graph, with_labels=True) plt.draw()
def demux(run_id, conf): """Add a processed run id to the list of the run ids. Arguments: run_id: The run id conf: configuration dictionary """ start_time = time.time() common.log('INFO', 'Demux step: Starting', conf) reports_data_base_path = conf[REPORTS_DATA_PATH_KEY] reports_data_path = common.get_report_run_data_path(run_id, conf) samplesheet_filename = build_samplesheet_filename(run_id, conf) bcl2fastq_samplesheet_path = conf[TMP_PATH_KEY] + '/' + samplesheet_filename + '.csv' input_run_data_path = common.get_input_run_data_path(run_id, conf) if input_run_data_path is None: return False fastq_output_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id basecall_stats_prefix = 'basecall_stats_' basecall_stats_file = basecall_stats_prefix + run_id + '.tar.bz2' # Check if root input bcl data directory exists if not os.path.exists(input_run_data_path): error("Basecalling data directory does not exist", "Basecalling data directory does not exist: " + str(input_run_data_path), conf) # return False # Check if root input fastq data directory exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False # Check if bcl2fastq samplesheets path exists if not common.is_dir_exists(BCL2FASTQ_SAMPLESHEETS_PATH_KEY, conf): error("Bcl2fastq samplesheet directory does not exist", "Bcl2fastq samplesheet directory does not exist: " + conf[BCL2FASTQ_SAMPLESHEETS_PATH_KEY], conf) return False # Check if bcl2fastq basedir path exists if not common.is_conf_value_equals_true(BCL2FASTQ_USE_DOCKER_KEY, conf): if not common.is_dir_exists(BCL2FASTQ_PATH_KEY, conf): error("Bcl2fastq directory does not exist", "Bcl2fastq directory does not exist: " + conf[BCL2FASTQ_PATH_KEY], conf) return False # Check if temporary directory exists if not common.is_dir_exists(TMP_PATH_KEY, conf): error("Temporary directory does not exist", "Temporary directory does not exist: " + conf[TMP_PATH_KEY], conf) return False # Check if reports_data_path exists if not os.path.exists(reports_data_base_path): error("Report directory does not exist", "Report directory does not exist: " + reports_data_base_path, conf) return False # Create if not exist report directory for the run if not os.path.exists(reports_data_path): os.mkdir(reports_data_path) # Check if basecall stats archive exists if os.path.exists(reports_data_path + '/' + basecall_stats_file): error('Basecall stats archive already exists for run ' + run_id, 'Basecall stats archive already exists for run ' + run_id + ': ' + basecall_stats_file, conf) return False # Check if the output directory already exists if os.path.exists(fastq_output_dir): error("FASTQ output directory already exists for run " + run_id, 'FASTQ output directory already exists for run ' + run_id + ': ' + fastq_output_dir, conf) return False # Compute disk usage and disk free to check if enough disk space is available input_path_du = common.du(input_run_data_path) output_df = common.df(conf[FASTQ_DATA_PATH_KEY]) du_factor = float(conf[DEMUX_SPACE_FACTOR_KEY]) space_needed = input_path_du * du_factor common.log("WARNING", "Demux step: input disk usage: " + str(input_path_du), conf) common.log("WARNING", "Demux step: output disk free: " + str(output_df), conf) common.log("WARNING", "Demux step: space needed: " + str(space_needed), conf) common.log("CONFIG", "Bcl2fastq Docker mode: " + str( common.is_conf_value_equals_true(Settings.BCL2FASTQ_USE_DOCKER_KEY, conf)), conf) # Check if free space is available if output_df < space_needed: error("Not enough disk space to perform demultiplexing for run " + run_id, "Not enough disk space to perform demultiplexing for run " + run_id + '.\n%.2f Gb' % (space_needed / 1024 / 1024 / 1024) + ' is needed (factor x' + str( du_factor) + ') on ' + fastq_output_dir + '.', conf) return False # Load RunInfo object run_info = RunInfo.parse(input_run_data_path + '/RunInfo.xml') # Load samplesheet samplesheet, original_samplesheet_path = load_samplesheet(run_id, input_run_data_path, samplesheet_filename, conf) if samplesheet is None: return False # Update samplesheet if not update_samplesheet(samplesheet, run_id, run_info.getFlowCellLaneCount(), conf): return False # Check samplesheet check_result, samplesheet_warnings = check_samplesheet(samplesheet, run_id, run_info.getFlowCell(), conf) if not check_result: return False # Get the number of mismatches nb_mismatch = get_bcl2fastq_mismatches(samplesheet, conf[BCL2FASTQ_MISMATCHES_KEY]) # Write final samplesheet if not write_bcl2fastq_samplesheet(samplesheet, bcl2fastq_samplesheet_path, conf): return False # Run demultiplexing if common.is_conf_value_equals_true(Settings.BCL2FASTQ_USE_DOCKER_KEY, conf): # With image docker if not demux_run_with_docker(run_id, input_run_data_path, fastq_output_dir, bcl2fastq_samplesheet_path, nb_mismatch, conf): return False else: if not demux_run_standalone(run_id, input_run_data_path, fastq_output_dir, bcl2fastq_samplesheet_path, nb_mismatch, conf): return False # Check if the output directory has been created if not os.path.exists(fastq_output_dir): error("Error while demultiplexing run " + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'Error while demultiplexing run ' + run_id + '.\n' + 'The output directory of bcl2fastq has been created: ' + fastq_output_dir, conf) return False # Check if the output directory has been created if os.path.isfile(fastq_output_dir): error("Error while demultiplexing run " + run_id + ' on ' + common.get_instrument_name(run_id, conf), 'Error while demultiplexing run ' + run_id + '.\n' + 'The output directory of bcl2fastq is a file instead of a directory: ' + fastq_output_dir, conf) return False # Copy bcl2fastq log to output directory cmd = 'cp ' + quote(conf[TMP_PATH_KEY]) + '/bcl2fastq_output_' + run_id + '.* ' + quote(fastq_output_dir) common.log("INFO", "exec: " + cmd, conf) if os.system(cmd) != 0: error("Error while copying bcl2fastq log to the output FASTQ directory" + run_id_msg, 'Error while copying bcl2fastq log to the output FASTQ directory.\nCommand line:\n' + cmd, conf) return False # The output directory must be read only if not common.chmod_files_in_dir(fastq_output_dir, ".fastq", conf): error("Error while setting the output FASTQ directory to read only" + run_id_msg, 'Error while setting the output FASTQ directory to read only.\nCommand line:\n' + cmd, conf) return False if not check_if_output_fastq_files_exists(fastq_output_dir): error("Error with bcl2fastq execution for run " + run_id, "Error with bcl2fastq execution for run " + run_id + " no FASTQ file found in " + fastq_output_dir, conf) return False # Copy samplesheet to output directory cmd = 'cp -p ' + quote(bcl2fastq_samplesheet_path) + ' ' + quote(fastq_output_dir + '/SampleSheet.csv') common.log("INFO", "exec: " + cmd, conf) if os.system(cmd) != 0: error("Error while copying samplesheet file to FASTQ directory for run " + run_id, 'Error while copying samplesheet file to FASTQ directory.\nCommand line:\n' + cmd, conf) return False # Create archives on demultiplexing statistics if not archive_demux_stat(run_id, fastq_output_dir, reports_data_path, basecall_stats_file, basecall_stats_prefix, bcl2fastq_samplesheet_path, conf): return False # Archive samplesheet if not archive_samplesheet(run_id, original_samplesheet_path, bcl2fastq_samplesheet_path, conf): return False # Remove temporary samplesheet files if os.path.exists(bcl2fastq_samplesheet_path): os.remove(bcl2fastq_samplesheet_path) # Create index.hml file common.create_html_index_file(conf, run_id, [Settings.HISEQ_STEP_KEY, Settings.DEMUX_STEP_KEY]) df_in_bytes = common.df(fastq_output_dir) du_in_bytes = common.du(fastq_output_dir) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024 * 1024) common.log("WARNING", "Demux step: output disk free after demux: " + str(df_in_bytes), conf) common.log("WARNING", "Demux step: space used by demux: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending demultiplexing with ' + nb_mismatch + ' mismatch(es) for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '.\n\n' + \ 'FASTQ files for this run ' + \ 'can be found in the following directory:\n ' + fastq_output_dir if samplesheet_warnings.size() > 0: msg += '\n\nSamplesheet warnings:' for warn in samplesheet_warnings: msg += "\n - " + warn # Add path to report if reports.url exists if common.is_conf_key_exists(REPORTS_URL_KEY, conf): msg += '\n\nRun reports can be found at following location:\n ' + conf[REPORTS_URL_KEY] + '/' + run_id msg += '\n\nFor this task %.2f GB has been used and %.2f GB still free.' % (du, df) common.send_msg('[Aozan] Ending demultiplexing for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'Demux step: successful in ' + common.duration_to_human_readable(duration), conf) return True
from common import send_msg, recv_msg PORT = 9090 with socket.socket() as sock: sock.bind(('', 9090)) sock.listen(1) print('Server: {}'.format(sock.getsockname())) while True: conn, addr = sock.accept() print('Connected:', addr) data = recv_msg(conn) print('Receiving ({}): {}'.format(len(data), data)) json_data = json.loads(data, encoding='utf-8') print('json_data:', json_data) json_data['title'] = 'updates' json_data['counter'] += 1 data = json.dumps(json_data) print('Sending: {}'.format(data)) rs = bytes(data, 'utf-8') send_msg(conn, rs) print('Close\n')
def sync(run_id, conf): """Synchronize a run. Arguments: run_id: the run id conf: configuration dictionary """ start_time = time.time() common.log('INFO', 'Sync step: Starting', conf) bcl_data_path = conf[BCL_DATA_PATH_KEY] reports_data_base_path = conf[REPORTS_DATA_PATH_KEY] output_path = bcl_data_path + '/' + run_id # check if rsync exists in PATH if not common.exists_in_path("rsync"): error("Can't find all needed commands in PATH env var", "Can't find all needed commands in PATH env var. Unable to find: rsync command.", conf) return False # Check if reports_data_path exists if not os.path.exists(reports_data_base_path): error("Report directory does not exist", "Report directory does not exist: " + reports_data_base_path, conf) return False # Check if enough space to store reports if common.df(reports_data_base_path) < 10 * 1024 * 1024 * 1024: error("Not enough disk space to store aozan reports for run " + run_id, "Not enough disk space to store aozan reports for run " + run_id + '.\nNeed more than 10 Gb on ' + reports_data_base_path + '.', conf) return False # Do the synchronization if not partial_sync(run_id, True, conf): return False # Rename partial sync directory to final run BCL directory if os.path.exists(output_path + '.tmp'): os.rename(output_path + '.tmp', output_path) # Check used and free space df_in_bytes = common.df(bcl_data_path) du_in_bytes = common.du(output_path) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024 * 1024) common.log("WARNING", "Sync step: output disk free after sync: " + str(df_in_bytes), conf) common.log("WARNING", "Sync step: space used by sync: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending synchronization for run ' + run_id + '.\n' + \ 'Job finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '.\n\n' + \ 'Run output files (without .cif files) can be found in the following directory:\n ' + output_path # Add path to report if reports.url exists if common.is_conf_key_exists(REPORTS_URL_KEY, conf): msg += '\n\nRun reports can be found at following location:\n ' + conf[REPORTS_URL_KEY] + '/' + run_id msg += '\n\nFor this task %.2f GB has been used and %.2f GB is still free.' % (du, df) common.send_msg('[Aozan] Ending synchronization for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'sync step: successful in ' + common.duration_to_human_readable(duration), conf) return True
print('Server: {}'.format(sock.getsockname())) while True: conn, addr = sock.accept() print('Connected:', addr) data = recv_msg(conn) print('Receiving {} bytes'.format(len(data))) img = Image.open(io.BytesIO(data)) print('Receiving image:', img) print('Transform image in thumbnail') # Transform in thumbnail img.thumbnail((75, 75)) print('Img:', img) # Write thumbnail in buffer data_io = io.BytesIO() img.save(data_io, 'jpeg') response_data = data_io.getvalue() print('Sending {} bytes'.format(len(response_data))) send_msg(conn, response_data) print('Close\n')
icred = gss.gssCred() icred.acquire(None, 3600, [gss.GSS_MECH_KRB5], gss.GSS_PY_INITIATE) target = gss.gssName() target.import_name('host@linuxbox', gss.GSS_NT_SERVICE_NAME) ictx = gss.gssContext() itoken = gss.GSS_PY_NO_BUFFER while (1): (cont_needed, otoken) = ictx.init( icred, target, gss.GSS_MECH_KRB5, gss.GSS_PY_MUTUAL_FLAG | gss.GSS_PY_CONF_FLAG | gss.GSS_PY_INTEG_FLAG, 0, itoken) if not cont_needed: break common.send_msg(s, otoken) itoken = common.recv_msg(s) print "\ncontext info" print "-=-=-=-=-=-=-=-=-" ctxinfo = ictx.inquire() (src_name, targ_name, time_rec, mech_type, context_flags) = ctxinfo print "initator: %s" % src_name.display() print "acceptor: %s" % targ_name.display() print "Context time: %d" % time_rec print "mech: %s" % common.b2a(mech_type) print "context flags: %04x" % context_flags # wrap test for message in holygrail.quotes.keys():
def run(self): while True: if not incoming_queue.empty(): msg = incoming_queue.get() logging.debug( 'Processing message: {}\nCurrent recvd: {}\nCurrent status: {}\nCoord so far: {}\nPort to coord: {}' .format(repr(msg), self.recd_reply, self.status, self.coord_so_far, self.port_to_coord)) logging.debug(''.format(self.recd_reply)) if msg['type'] == 'start': print('Activating in a second') time.sleep(2.0) self._heartbeat() elif msg['type'] == 'reconfig': self.recd_reply[self.port_to(int( msg['id']))] = 'no_contention' self._on_reconfig(msg['node_list'], int(msg['frag_id']), int(msg['id'])) elif msg['type'] == 'no_contention': if self.status == 'wait': self.recd_reply[self.port_to(int( msg['id']))] = 'no_contention' if len(self.recd_reply) == len(self._ports): self._on_everybody_responded() elif msg['type'] == 'accept': if self.status == 'wait': self.recd_reply[self.port_to(int( msg['id']))] = 'accepted' if len(self.recd_reply) == len(self._ports): self._on_everybody_responded() elif msg['type'] == 'stop': if self.status == 'wait': self._on_stop(int(msg['frag_id']), int(msg['id'])) elif msg['type'] == 'fail': # set status of the node # TODO: This might be an issue self.status = 'wait' self.coord_so_far = self.id self.port_to_coord = None # What to put here??? # remove the failed edge from our MST self.remove_edge(int(msg['id'])) # send reconfiguration request through all the ports for dest_id in set(self._ports.values()).difference( [int(msg['id'])]): self._con.send(id=dest_id, msg=dict(type='reconfig', node_list=[self.id], frag_id=self.id)) elif msg['type'] == 'extract': host = msg['host'] port = msg['port'] graph_msg = encode( dict(id=self.id, links=list(self._ports.values()), edges=[ self._ports[port_id] for port_id in self._edges ])) send_msg(msg=graph_msg, host=host, port=port)
def run(self): while True: if not incoming_queue.empty(): msg = incoming_queue.get() if msg['type'] == 'start': time.sleep(2.0) self._heartbeat() elif msg['type'] == 'reconfig': sender_id = int(msg['id']) frag_id = int(msg['frag_id']) if self.status == 'idle': self.status = 'wait' failed_id = int(msg['failed_node']) self.coord_so_far = sender_id self.port_to_coord = self.port_to(sender_id) can_communicate = list( set(self._ports.values()).difference( [sender_id, failed_id])) if len(can_communicate) == 0: self._con.send(sender_id, dict(type='no_contention')) else: for node_id in can_communicate: self.recd_reply[node_id] = None self._con.send( node_id, dict(type='reconfig', node_list=msg['node_list'] + [self.id], frag_id=msg['frag_id'], failed_node=failed_id)) else: e = self.port_to(sender_id) # Message is the copy of message recieved earlier if (frag_id == self.coord_so_far) and ( e not in self.get_port()): logging.debug('Message was recieved earlier') self._con.send(self._ports[e], dict(type='no_contention')) continue # Detected loop if self.id in msg['node_list']: logging.debug('Loop is detected') self._con.send(self._ports[e], dict(type='no_contention')) continue # Resolve contention if (self.coord_so_far > frag_id) or ( (self.coord_so_far == frag_id) and (self.id > sender_id)): logging.debug('Sending out stop message') self._con.send( sender_id, dict(type='stop', frag_id=self.coord_so_far)) else: self.coord_so_far = frag_id if self.port_to_coord is not None: logging.debug('Sending out stop message') self._con.send( self._ports[self.port_to_coord], dict(type='stop', frag_id=frag_id)) self.port_to_coord = self.port_to(sender_id) elif msg['type'] == 'no_contention': sender_id = int(msg['id']) if sender_id in self.recd_reply.keys(): self.recd_reply[sender_id] = 'no_contention' if not there_is_a_none(self.recd_reply): self._on_everybody_responded() elif msg['type'] == 'accepted': sender_id = int(msg['id']) if sender_id in self.recd_reply.keys(): self.recd_reply[sender_id] = 'accepted' if not there_is_a_none(self.recd_reply): self._on_everybody_responded() elif msg['type'] == 'stop': frag_id = msg['frag_id'] from_id = msg['id'] p = self.port_to(from_id) if frag_id > self.coord_so_far: self.coord_so_far = frag_id if self.port_to_coord is not None: self._con.send(self._ports[self.port_to_coord], dict(type='stop', frag_id=frag_id)) self.port_to_coord = p if frag_id == self.coord_so_far: if self.port_to_coord not in self.get_port(): if self.port_to_coord is not None: self._con.send(self._ports[self.port_to_coord], dict(type='no_contention')) self.recd_reply[ self.port_to_coord] = 'no_contention' if not there_is_a_none(self.recd_reply): self._on_everybody_responded() else: self._con.send(self._ports[self.port_to_coord], dict(type='stop', frag_id=frag_id)) self.port_to_coord = p if frag_id < self.coord_so_far: self._con.send( self._ports[p], dict(type='stop', frag_id=self.coord_so_far)) elif msg['type'] == 'fail': failed_node = int(msg['id']) self.remove_edge(failed_node) self.status = 'wait' self.coord_so_far = self.id self.port_to_coord = None for node_id in set(self._ports.values()).difference( [failed_node]): self.recd_reply[node_id] = None self._con.send( node_id, dict(type='reconfig', node_list=[self.id], frag_id=self.id, failed_node=failed_node)) elif msg['type'] == 'extract': host = msg['host'] port = msg['port'] graph_msg = encode( dict(id=self.id, links=list(self._ports.values()), edges=[ self._ports[port_id] for port_id in self._edges ])) send_msg(msg=graph_msg, host=host, port=port)
import socket import json from common import send_msg, recv_msg s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) host = socket.gethostname() port = 9999 s.connect((host, port)) msg = "abs" #.encode('utf-8') send_msg(s, msg) data = recv_msg(s) s.close() print(data)
def demux_run_with_docker(run_id, input_run_data_path, fastq_output_dir, samplesheet_csv_path, nb_mismatch, conf): """ Demultiplexing the run with bcl2fastq on version parameter with image Docker. Arguments: run_id: The run id input_run_data_path: input run data path to demultiplexing fastq_output_dir: fastq directory to save result on demultiplexing samplesheet_csv_path: samplesheet path in csv format, version used by bcl2fastq conf: configuration dictionary """ # In docker mount with input_run_data_path input_docker = '/data/input' input_run_data_path_in_docker = input_docker run_id_msg = " for run " + run_id + ' on ' + common.get_instrument_name(run_id, conf) # In docker mount with fastq_output_dir output_docker = '/data/output' fastq_data_path_in_docker = output_docker + '/' + os.path.basename(fastq_output_dir) tmp = conf[TMP_PATH_KEY] tmp_docker = '/tmp' bcl2fastq_log_file = tmp + "/bcl2fastq_output_" + run_id + ".err" samplesheet_csv_docker = tmp_docker + '/' + os.path.basename(samplesheet_csv_path) cmd = create_bcl2fastq_command_line(run_id, None, input_run_data_path_in_docker, fastq_data_path_in_docker, samplesheet_csv_docker, tmp_docker, nb_mismatch, conf) try: # Set working in docker on parent demultiplexing run directory. # Demultiplexing run directory will create by bcl2fastq docker = DockerCommand(conf[Settings.DOCKER_URI_KEY], ['/bin/bash', '-c', cmd], 'bcl2fastq2', common.BCL2FASTQ2_VERSION) common.log("CONFIG", "Demultiplexing using docker image from " + docker.getImageDockerName() + " with command line " + cmd, conf) common.log("CONFIG", "Bcl2fastq docker mount: " + str(os.path.dirname(fastq_output_dir)) + ":" + str(output_docker) + "; " + input_run_data_path + ":" + input_docker + "; " + tmp + ":" + tmp_docker, conf) # Mount input directory docker.addMountDirectory(input_run_data_path, input_docker) docker.addMountDirectory(os.path.dirname(fastq_output_dir), output_docker) docker.addMountDirectory(tmp, tmp_docker) docker.run() exit_code = docker.getExitValue() if exit_code != 0: error("Error while demultiplexing run " + run_id, 'Error while demultiplexing run (exit code: ' + str(exit_code) + ').\nCommand line:\n' + cmd, conf) msg = 'Error while executing bcl2fastq ' + run_id_msg + ' (exit code: ' + str( exit_code) + ').\nCommand line:\n' + cmd # Check if the log file has been generated if not os.path.exists(bcl2fastq_log_file): error("Error with bcl2fastq log for run " + run_id + ".", "No bcl2fastq log available " + bcl2fastq_log_file, conf) common.send_msg('[Aozan] Failed demultiplexing ' + run_id_msg, msg, True, conf) else: msg += "\n\nPlease check the attached bcl2fastq output error file." common.send_msg_with_attachment('[Aozan] Failed demultiplexing ' + run_id_msg, msg, bcl2fastq_log_file, True, conf) return False except Throwable, exp: error("Error while running Docker image", common.exception_msg(exp, conf), conf) return False
#!/usr/bin/env python3 import argparse from pprint import pprint from common import load_config from common import encode from common import send_msg def opt_parser(): parser = argparse.ArgumentParser( description='Network reconfiguration node') parser.add_argument('--net_config', default='config/sample_graph3.json', type=str) return parser if __name__ == '__main__': parser = opt_parser() opt = parser.parse_args() config = load_config(opt.net_config) pprint(config) start_msg = encode(dict(type="start")) for node in config.values(): send_msg(start_msg, host=node['host'], port=node['port']) print(node)
# -*- coding: utf-8 -*- __author__ = 'ipetrash' import socket import sys sys.path.append('..') from common import send_msg, recv_msg PORT = 9090 with socket.socket() as sock: sock.bind(('', PORT)) sock.listen() print('Server: {}'.format(sock.getsockname())) while True: conn, addr = sock.accept() print('Connected:', addr) data = recv_msg(conn) print('Receiving ({}): {}'.format(len(data), data)) print('Sending') send_msg(conn, data.upper()) print('Close\n')
sys.path.append('..') from common import send_msg, recv_msg HOST, PORT = "localhost", 9090 with socket.socket() as sock: sock.connect((HOST, PORT)) # Send file with open('img.jpg', 'rb') as f: data = f.read() print('Sending {} bytes'.format(len(data))) print() send_msg(sock, data) print('Receiving') response_data = recv_msg(sock) print('Response {} bytes'.format(len(response_data))) file_name = 'img_thumbnail.jpg' print('Save in ' + file_name) with open(file_name, 'wb') as f: f.write(response_data) print('Close\n')
def send_report(run_id, conf): """Send a mail with the first base report. Arguments: run_id: the run id conf: configuration dictionary """ # # Retrieve features the current run in RunInfos.xml file # run_info = hiseq_run.get_run_info(run_id, conf) if run_info is None: return False # TODO ?? add check sample-sheet if demux step enable # add warning in report if useful reads = run_info.getReads() error_cycles_per_read_not_indexes_count = 0 reads_indexed_count = 0 reads_not_indexed_count = 0 cycles_count = 0 cycles_per_read_not_indexed = 0 for read in reads: cycles_count += read.getNumberCycles() if read.isIndexedRead(): reads_indexed_count += 1 else: reads_not_indexed_count += 1 if cycles_per_read_not_indexed == 0: cycles_per_read_not_indexed = read.getNumberCycles() # Check same cycles count for each reads not indexed error_cycles_per_read_not_indexes_count = cycles_per_read_not_indexed != read.getNumberCycles() # Identification type run according to data in RunInfos.xml : SR or PE if reads_not_indexed_count == 1: type_run_estimated = "SR-" + str(cycles_per_read_not_indexed) + " with " + str( reads_indexed_count) + " index" if reads_indexed_count > 1: type_run_estimated += "es" elif reads_not_indexed_count == 2: type_run_estimated = "PE-" + str(cycles_per_read_not_indexed) + " with " + str( reads_indexed_count) + " index" if reads_indexed_count > 1: type_run_estimated += "es" else: type_run_estimated = "Undetermined run type (" + str(reads_not_indexed_count) + " reads with " + str( reads_indexed_count) + " index)" if reads_indexed_count > 1: type_run_estimated += "es" type_run_estimated += ")" description_run = "Informations about this run:\n" description_run += "\t- Sequencer: " + common.get_instrument_name(run_id, conf) + ".\n" description_run += "\t- " + str(run_info.getFlowCellLaneCount()) + " lanes with " + str( run_info.alignToPhix.size()) + " aligned to Phix.\n" description_run += "\t- " + str(reads_not_indexed_count) + " read" if reads_not_indexed_count > 1: description_run += "s" description_run += " and " + str(reads_indexed_count) + " index" if reads_indexed_count > 1: description_run += "es" description_run += ".\n" if error_cycles_per_read_not_indexes_count or cycles_per_read_not_indexed == 0: description_run += "\t- ERROR : cycles count per read different between reads (" + str( cycles_count) + " total cycles).\n" else: description_run += "\t- " + str(cycles_per_read_not_indexed) + " cycles per read (" + str( cycles_count) + " total cycles).\n" description_run += "\t- Estimated run type: " + type_run_estimated + ".\n" attachment_file = str(hiseq_run.find_hiseq_run_path(run_id, conf)) + '/' + run_id + '/' + common.FIRST_BASE_REPORT_FILE # If the First base report file exists, send it by email if common.is_file_readable(attachment_file): message = 'You will find attached to this message the first base report for the run ' + \ run_id + '.\n\n' + description_run common.send_msg_with_attachment('[Aozan] First base report for the run ' + type_run_estimated + ' ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), message, attachment_file, False, conf) else: # With other no attachment file message = 'You will find below the parameters of the run ' + run_id + '.\n\n' + description_run common.send_msg('[Aozan] New run ' + type_run_estimated + ' ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), message, False, conf) return True
def recompress(run_id, conf): """Proceed to recompression of a run. Arguments: run_id: The run id conf: configuration dictionary """ common.log('INFO', 'Recompress step: Starting', conf) # Check if input root fastq root data exists if not common.is_dir_exists(FASTQ_DATA_PATH_KEY, conf): error("FASTQ data directory does not exist", "FASTQ data directory does not exist: " + conf[FASTQ_DATA_PATH_KEY], conf) return False start_time = time.time() fastq_input_dir = conf[FASTQ_DATA_PATH_KEY] + '/' + run_id # initial du for comparing with ending disk usage previous_du_in_bytes = common.du(fastq_input_dir) # get information about compression type compression_type = conf[RECOMPRESS_COMPRESSION_KEY] compression_level = conf[RECOMPRESS_COMPRESSION_LEVEL_KEY] compression_info_tuple = get_info_from_file_type(compression_type, compression_level) if compression_info_tuple is None: error("Unknown compression type", "Unknown compression type: " + compression_type, conf) return False (compression_type_result, output_file_extension, output_compression_command, output_decompression_command, compression_level_argument) = compression_info_tuple # The following list contains the processed type of files to recompress types_to_recompress = ["fastq.gz", "fastq"] # list of program to check if exists in path before execution program_set = {"bash", "tee", "touch", "chmod", "md5sum", output_compression_command, output_decompression_command} # get list of file to process input_files = [] for extension in types_to_recompress: input_files.extend(list_files(fastq_input_dir, extension)) simple_extension = os.path.splitext(extension)[-1][1:] extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + extension, conf) return False program_set.add(extension_info_tuple[3]) # actual program list check for program in program_set: if not common.exists_in_path(program): error("Can't find all needed commands in PATH env var", "Can't find all needed commands in PATH env var. Unable to find: " + program + " command.", conf) return False # Create executor and for parallelization of processus executor = Executors.newFixedThreadPool(int(conf[RECOMPRESS_THREADS_KEY])) workers = [] # process each fastq and fastq.gz recursively in each fastq directory for input_file in input_files: simple_extension = os.path.splitext(input_file)[-1][1:] # get info about the type of input file extension_info_tuple = get_info_from_file_type(simple_extension) if extension_info_tuple is None: error("Unknown extension type", "Unknown extension type: " + simple_extension, conf) return False input_decompression_command = extension_info_tuple[3] # get file base name and create output_file name, if file is already .fastq its ready to be base_input_file base_input_file = input_file[0: input_file.index(".fastq") + 6] output_file = base_input_file + "." + output_file_extension # Skip if the output_file already exists if not os.path.exists(output_file): # Create worker then execute in thread worker = Worker(input_file, output_file, input_decompression_command, output_compression_command, output_decompression_command, compression_level_argument, common.is_conf_value_equals_true(RECOMPRESS_DELETE_ORIGINAL_FASTQ_KEY, conf)) workers.append(worker) executor.execute(worker) else: common.log("WARNING", "Recompress step: Omitting processing file " + input_file + ". The associated output file " + output_file + " already exists.", conf) # Wait for all thread to finish executor.shutdown() while not executor.isTerminated(): time.sleep(1) # Check if any worker is in error for worker in workers: if not worker.is_successful(): error(worker.get_error_message(), worker.get_long_error_message(), conf) return False # check new disk usage df_in_bytes = common.df(fastq_input_dir) du_in_bytes = common.du(fastq_input_dir) previous_du = previous_du_in_bytes / (1024 * 1024) df = df_in_bytes / (1024 * 1024 * 1024) du = du_in_bytes / (1024 * 1024) common.log("WARNING", "Recompress step: output disk free after step: " + str(df_in_bytes), conf) common.log("WARNING", "Recompress step: space previously used: " + str(previous_du_in_bytes), conf) common.log("WARNING", "Recompress step: space now used by step: " + str(du_in_bytes), conf) duration = time.time() - start_time msg = 'Ending recompression for run ' + run_id + '.' + \ '\nJob finished at ' + common.time_to_human_readable(time.time()) + \ ' without error in ' + common.duration_to_human_readable(duration) + '. ' msg += '\n\nAfter recompress step FASTQ folder is now %.2f MB (previously %.2f MB) and %.2f GB still free.' % ( du, previous_du, df) common.send_msg('[Aozan] Ending recompress for run ' + run_id + ' on ' + common.get_instrument_name(run_id, conf), msg, False, conf) common.log('INFO', 'Recompress step: successful in ' + common.duration_to_human_readable(duration), conf) return True