def run(parser, args):
# TODO: Move logging config to separate configuration file
# TODO: use setup_logger here instead?
# set up logging to file for DEBUG messages and above
logging.basicConfig(
level=logging.DEBUG,
# TODO: args.log_format
format="%(levelname)s %(asctime)s %(name)s %(message)s",
filename=args.log_file,
filemode="w",
)
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter(args.log_format)
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger("").addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
# logger = setup_logger(__name__, args.log_format, log_file=args.log_file, level=logging.INFO)
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
read_until_client = read_until.ReadUntilClient(
mk_host=args.host,
mk_port=args.port,
device=args.device,
# one_chunk=args.one_chunk,
filter_strands=True,
# TODO: test cache_type by passing a function here
cache_type=args.read_cache,
cache_size=args.cache_size,
)
analysis_worker = functools.partial(
simple_analysis,
client=read_until_client,
batch_size=args.batch_size,
throttle=args.throttle,
unblock_duration=args.unblock_duration,
)
results = run_workflow(
client=read_until_client,
partial_analysis_func=analysis_worker,
n_workers=args.workers,
run_time=args.run_time,
runner_kwargs={
# "min_chunk_size": args.min_chunk_size,
"first_channel": args.channels[0],
"last_channel": args.channels[-1],
},
)
def run(parser, args):
args.tomlfile = args.toml
args.toml = toml.load(args.toml)
print(args)
# TODO: Move logging config to separate configuration file
# set up logging to file
logging.basicConfig(
level=logging.DEBUG,
format='%(levelname)s::%(asctime)s::%(name)s::%(message)s',
filename=args.log_file,
filemode='w')
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-15s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
logger.info("Initialising iterAlign.")
logger.info("Setting up FastQ monitoring.")
#### Check if a run is active - if not, wait.
args.simulation = True
connection = None
if args.watch is None:
args.simulation = False
logger.info("Creating rpc connection for device {}.".format(
args.device))
try:
connection, messageport = get_rpc_connection(args.device)
except ValueError as e:
print(e)
sys.exit(1)
send_message(connection, "Iteralign Connected to MinKNOW",
Severity.WARN)
logger.info("Loaded RPC")
while parse_message(connection.acquisition.current_status()
)['status'] != "PROCESSING":
time.sleep(1)
#### Check if we know where data is being written to , if not... wait
args.watch = parse_message(connection.acquisition.get_acquisition_info(
))['config_summary']['reads_directory']
else:
messageport = ""
event_handler = FastqHandler(args, logging, messageport, connection)
# This block handles the fastq
observer = Observer()
observer.schedule(event_handler, path=args.watch, recursive=True)
observer.daemon = True
try:
observer.start()
logger.info("FastQ Monitoring Running.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Exiting - Will take a few seconds to clean up!")
observer.stop()
observer.join()
os._exit(0)
def run(parser, args):
# new code block: change the reference path within the args.toml file into the args.mindex path
d = toml.load(args.toml)
print(d["conditions"]["reference"])
args.tomlfile = args.toml
args.toml = toml.load(args.toml)
print(args)
# TODO: Move logging config to separate configuration file
# set up logging to file
logging.basicConfig(level=logging.DEBUG,
format='%(levelname)s::%(asctime)s::%(name)s::%(message)s',
filename=args.log_file,
filemode='w')
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter('%(name)-15s: %(levelname)-8s %(message)s')
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger('').addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
logger.info("Initialising iterAlign.")
logger.info("Setting up FastQ monitoring.")
#### Check if a run is active - if not, wait.
args.simulation = True
connection = None
#set default message severity level.
severity = 2
if args.watch is None:
args.simulation = False
logger.info("Creating rpc connection for device {}.".format(args.device))
try:
connection, messageport = get_rpc_connection(args.device)
except ValueError as e:
print(e)
sys.exit(1)
#send_message_port("Iteralign Connected to MinKNOW", args.host, messageport)
send_message(connection, "Iteralign Connected to MinKNOW.", Severity.WARN)
logger.info("Loaded RPC")
while parse_message(connection.acquisition.current_status())['status'] != "PROCESSING":
time.sleep(1)
### Check if we know where data is being written to , if not... wait
args.watch = parse_message(connection.acquisition.get_acquisition_info())['config_summary'][
'reads_directory']
else:
messageport = ""
event_handler = FastqHandler(args, logging, messageport, connection)
# This block handles the fastq
observer = Observer()
observer.schedule(event_handler, path=args.watch, recursive=True)
observer.daemon = True
try:
observer.start()
logger.info("FastQ Monitoring Running.")
while 1:
time.sleep(1)
except KeyboardInterrupt:
logger.info("Exiting - Will take a few seconds to clean up!")
observer.stop()
observer.join()
if args.keepfiles:
logging.info("The 'keepfiles' argument was set, files generated by classifier have been retained")
else:
if os.path.isdir(args.path):
for path, dirs, files in os.walk(args.path):
for f in files:
if f.startswith(args.prefix):
os.unlink(f)
logging.info("file removed: {}".format(f))
if os.path.isdir("./"):
for path, dirs, files in os.walk("./"):
for f in files:
if f.endswith(args.creport):
os.unlink(f)
logging.info("file removed: {}".format(f))
logging.info("All files generated by classifier have been removed.")
os._exit(0)
def main():
extra_args = (
(
"--toml",
dict(
metavar="TOML",
required=True,
help="TOML file specifying experimental parameters",
),
),
("--paf-log", dict(
help="PAF log",
default="paflog.log",
)),
("--chunk-log", dict(
help="Chunk log",
default="chunk_log.log",
)),
)
parser, args = get_parser(extra_args=extra_args, file=__file__)
# set up logging to file for DEBUG messages and above
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(name)s %(message)s",
filename=args.log_file,
filemode="w",
)
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter(args.log_format)
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger("").addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
# Setup chunk and paf logs
chunk_logger = setup_logger("DEC", log_file=args.chunk_log)
paf_logger = setup_logger("PAF", log_file=args.paf_log)
# Parse configuration TOML
# TODO: num_channels is not configurable here, should be inferred from client
run_info, conditions, reference, caller_kwargs = get_run_info(
args.toml, num_channels=512)
live_toml = Path("{}_live".format(args.toml))
# Load Minimap2 index
logger.info("Initialising minimap2 mapper")
mapper = CustomMapper(reference)
logger.info("Mapper initialised")
read_until_client = read_until.ReadUntilClient(
mk_host=args.host,
mk_port=args.port,
device=args.device,
# one_chunk=args.one_chunk,
filter_strands=True,
# TODO: test cache_type by passing a function here
cache_type=args.read_cache,
cache_size=args.cache_size,
)
send_message(
read_until_client.connection,
"Read Until is controlling sequencing on this device. You use it at your own risk.",
Severity.WARN,
)
for message, sev in describe_experiment(conditions, mapper):
logger.info(message)
send_message(
read_until_client.connection,
message,
sev,
)
"""
This experiment has N regions on the flowcell.
using reference: /path/to/ref.mmi
Region i:NAME (control=bool) has X targets of which Y are found in the reference.
reads will be unblocked when [u,v], sequenced when [w,x] and polled for more data when [y,z].
"""
# FIXME: currently flowcell size is not included, this should be pulled from
# the read_until_client
analysis_worker = functools.partial(
simple_analysis,
read_until_client,
unblock_duration=args.unblock_duration,
throttle=args.throttle,
batch_size=args.batch_size,
cl=chunk_logger,
pf=paf_logger,
live_toml_path=live_toml,
dry_run=args.dry_run,
run_info=run_info,
conditions=conditions,
mapper=mapper,
caller_kwargs=caller_kwargs,
)
results = run_workflow(
read_until_client,
analysis_worker,
args.workers,
args.run_time,
runner_kwargs={
# "min_chunk_size": args.min_chunk_size,
"first_channel": min(args.channels),
"last_channel": max(args.channels),
},
)
# No results returned
send_message(
read_until_client.connection,
"Read Until is disconnected from this device. Sequencing will proceed normally.",
Severity.WARN,
)
def main():
extra_args = (
(
"--toml",
dict(
metavar="TOML",
required=True,
help="TOML file specifying experimental parameters",
),
),
(
"--chunk-log",
dict(
help="Chunk log",
default="chunk_log.log",
)
),
)
parser, args = get_parser(extra_args=extra_args, file=__file__)
# TODO: Move logging config to separate configuration file
# set up logging to file for DEBUG messages and above
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s %(name)s %(message)s",
filename=args.log_file,
filemode="w",
)
# define a Handler that writes INFO messages or higher to the sys.stderr
console = logging.StreamHandler()
console.setLevel(logging.INFO)
# set a format which is simpler for console use
formatter = logging.Formatter(args.log_format)
console.setFormatter(formatter)
# add the handler to the root logger
logging.getLogger("").addHandler(console)
# Start by logging sys.argv and the parameters used
logger = logging.getLogger("Manager")
logger.info(" ".join(sys.argv))
print_args(args, logger=logger)
read_until_client = read_until.ReadUntilClient(
mk_host=args.host,
mk_port=args.port,
device=args.device,
# one_chunk=args.one_chunk,
filter_strands=True,
# TODO: test cache_type by passing a function here
cache_type=args.read_cache,
cache_size=args.cache_size,
)
# FIXME: currently flowcell size is not included, this should be pulled from
# the read_until_client
analysis_worker = functools.partial(
simple_analysis,
read_until_client,
unblock_duration=args.unblock_duration,
throttle=args.throttle,
batch_size=args.batch_size,
chunk_log=args.chunk_log,
toml_path=args.toml,
dry_run=args.dry_run,
)
results = run_workflow(
read_until_client,
analysis_worker,
args.workers,
args.run_time,
runner_kwargs={
# "min_chunk_size": args.min_chunk_size,
"first_channel": min(args.channels),
"last_channel": max(args.channels),
},
)