def read(cls, fam_path, delimiter='\s+') -> 'Pedigree':
"""Read a PLINK .fam file and return a pedigree object.
**Examples**
>>> ped = hl.Pedigree.read('data/test.fam')
Notes
-------
See `PLINK .fam file <https://www.cog-genomics.org/plink2/formats#fam>`_ for
the required format.
:param str fam_path: path to .fam file.
:param str delimiter: Field delimiter.
:rtype: :class:`.Pedigree`
"""
trios = []
missing_sex_count = 0
missing_sex_values = set()
with Env.fs().open(fam_path) as file:
for line in file:
split_line = re.split(delimiter, line.strip())
num_fields = len(split_line)
if num_fields != 6:
raise FatalError(
"Require 6 fields per line in .fam, but this line has {}: {}"
.format(num_fields, line))
(fam, kid, dad, mom, sex, _) = tuple(split_line)
# 1 is male, 2 is female, 0 is unknown.
is_female = sex == "2" if sex == "1" or sex == "2" else None
if is_female is None:
missing_sex_count += 1
missing_sex_values.add(kid)
trio = Trio(kid, fam if fam != "0" else None,
dad if dad != "0" else None,
mom if mom != "0" else None, is_female)
trios.append(trio)
only_ids = [trio.s for trio in trios]
duplicate_ids = [
id for id, count in Counter(only_ids).items() if count > 1
]
if duplicate_ids:
raise FatalError(
"Invalid pedigree: found duplicate proband IDs\n{}".format(
duplicate_ids))
if missing_sex_count > 0:
warn(
"Found {} samples with missing sex information (not 1 or 2).\n Missing samples: [{}]"
.format(missing_sex_count, missing_sex_values))
return Pedigree(trios)
def __init__(self, sc=None, app_name="Hail", master=None, local='local[*]',
log=None, quiet=False, append=False,
min_block_size=1, branching_factor=50, tmp_dir=None,
default_reference="GRCh37", idempotent=False,
global_seed=6348563392232659379, spark_conf=None,
optimizer_iterations=None, _backend=None):
if Env._hc:
if idempotent:
return
else:
warn('Hail has already been initialized. If this call was intended to change configuration,'
' close the session with hl.stop() first.')
py_version = version()
if log is None:
log = hail.utils.timestamp_path(os.path.join(os.getcwd(), 'hail'),
suffix=f'-{py_version}.log')
self._log = log
tmp_dir = get_env_or_default(tmp_dir, 'TMPDIR', '/tmp')
self.tmp_dir = tmp_dir
optimizer_iterations = get_env_or_default(optimizer_iterations, 'HAIL_OPTIMIZER_ITERATIONS', 3)
if _backend is None:
if os.environ.get('HAIL_APISERVER_URL') is not None:
_backend = ServiceBackend()
else:
_backend = SparkBackend(
idempotent, sc, spark_conf, app_name, master, local, log,
quiet, append, min_block_size, branching_factor, tmp_dir,
optimizer_iterations)
self._backend = _backend
self._warn_cols_order = True
self._warn_entries_order = True
super(HailContext, self).__init__()
# do this at the end in case something errors, so we don't raise the above error without a real HC
Env._hc = self
ReferenceGenome._from_config(_backend.get_reference('GRCh37'), True)
ReferenceGenome._from_config(_backend.get_reference('GRCh38'), True)
ReferenceGenome._from_config(_backend.get_reference('GRCm38'), True)
ReferenceGenome._from_config(_backend.get_reference('CanFam3'), True)
if default_reference in ReferenceGenome._references:
self._default_ref = ReferenceGenome._references[default_reference]
else:
self._default_ref = ReferenceGenome.read(default_reference)
if not quiet:
sys.stderr.write(
'Welcome to\n'
' __ __ <>__\n'
' / /_/ /__ __/ /\n'
' / __ / _ `/ / /\n'
' /_/ /_/\\_,_/_/_/ version {}\n'.format(py_version))
if py_version.startswith('devel'):
sys.stderr.write('NOTE: This is a beta version. Interfaces may change\n'
' during the beta period. We recommend pulling\n'
' the latest changes weekly.\n')
sys.stderr.write(f'LOGGING: writing to {log}\n')
install_exception_handler()
Env.set_seed(global_seed)
def init(sc=None,
app_name='Hail',
master=None,
local='local[*]',
log=None,
quiet=False,
append=False,
min_block_size=0,
branching_factor=50,
tmp_dir='/tmp',
default_reference='GRCh37',
idempotent=False,
global_seed=6348563392232659379,
spark_conf=None,
skip_logging_configuration=False,
local_tmpdir=None,
_optimizer_iterations=None):
"""Initialize Hail and Spark.
Examples
--------
Import and initialize Hail using GRCh38 as the default reference genome:
>>> import hail as hl
>>> hl.init(default_reference='GRCh38') # doctest: +SKIP
Notes
-----
Hail is not only a Python library; most of Hail is written in Java/Scala
and runs together with Apache Spark in the Java Virtual Machine (JVM).
In order to use Hail, a JVM needs to run as well. The :func:`.init`
function is used to initialize Hail and Spark.
This function also sets global configuration parameters used for the Hail
session, like the default reference genome and log file location.
This function will be called automatically (with default parameters) if
any Hail functionality requiring the backend (most of the libary!) is used.
To initialize Hail explicitly with non-default arguments, be sure to do so
directly after importing the module, as in the above example.
Note
----
If a :class:`pyspark.SparkContext` is already running, then Hail must be
initialized with it as an argument:
>>> hl.init(sc=sc) # doctest: +SKIP
See Also
--------
:func:`.stop`
Parameters
----------
sc : pyspark.SparkContext, optional
Spark context. By default, a Spark context will be created.
app_name : :obj:`str`
Spark application name.
master : :obj:`str`, optional
Spark master.
local : :obj:`str`
Local-mode master, used if `master` is not defined here or in the
Spark configuration.
log : :obj:`str`
Local path for Hail log file. Does not currently support distributed
file systems like Google Storage, S3, or HDFS.
quiet : :obj:`bool`
Print fewer log messages.
append : :obj:`bool`
Append to the end of the log file.
min_block_size : :obj:`int`
Minimum file block size in MB.
branching_factor : :obj:`int`
Branching factor for tree aggregation.
tmp_dir : :obj:`str`, optional
Networked temporary directory. Must be a network-visible file
path. Defaults to /tmp in the default scheme.
default_reference : :obj:`str`
Default reference genome. Either ``'GRCh37'``, ``'GRCh38'``,
``'GRCm38'``, or ``'CanFam3'``.
idempotent : :obj:`bool`
If ``True``, calling this function is a no-op if Hail has already been initialized.
global_seed : :obj:`int`, optional
Global random seed.
spark_conf : :obj:`dict[str, str]`, optional
Spark configuration parameters.
skip_logging_configuration : :obj:`bool`
Skip logging configuration.
local_tmpdir : obj:`str`, optional
Local temporary directory. Used on driver and executor nodes.
Must use the file scheme. Defaults to TMPDIR, or /tmp.
"""
from hail.backend.spark_backend import SparkBackend
if Env._hc:
if idempotent:
return
else:
warn(
'Hail has already been initialized. If this call was intended to change configuration,'
' close the session with hl.stop() first.')
log = _get_log(log)
tmpdir = _get_tmpdir(tmp_dir)
local_tmpdir = _get_local_tmpdir(local_tmpdir)
optimizer_iterations = get_env_or_default(_optimizer_iterations,
'HAIL_OPTIMIZER_ITERATIONS', 3)
backend = SparkBackend(idempotent, sc, spark_conf, app_name, master, local,
log, quiet, append, min_block_size,
branching_factor, tmpdir, local_tmpdir,
skip_logging_configuration, optimizer_iterations)
HailContext(log, quiet, append, tmp_dir, local_tmpdir, default_reference,
global_seed, backend)