def analysis(meta_filename: str,
counts_filename: str,
counts_data: str,
project_name: str,
threshold: float,
result_precision: int,
output_path: str,
output_format: str,
means_result_name: str,
significant_means_result_name: str,
deconvoluted_result_name: str,
verbose: bool,
database: Optional[str],
subsampling: bool,
subsampling_log: bool,
subsampling_num_pc: int,
subsampling_num_cells: Optional[int]
):
try:
subsampler = Subsampler(subsampling_log,
subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
LocalMethodLauncher(cpdb_app.create_app(verbose,
database)).cpdb_analysis_local_method_launcher(meta_filename,
counts_filename,
counts_data,
project_name,
threshold,
output_path,
output_format,
means_result_name,
significant_means_result_name,
deconvoluted_result_name,
result_precision,
subsampler,
)
except (ReadFileException, ParseMetaException, ParseCountsException, ThresholdValueException,
AllCountsFilteredException) as e:
app_logger.error(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except EmptyResultException as e:
app_logger.warning(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
def analysis_scanpy(adata,
var_names,
obs_names,
obs_key,
var_key=None,
gene_id_format=None,
project_name='',
threshold=0.1,
result_precision='3',
output_path='',
output_format='csv',
means_result_name='means',
significant_means_result_name='significant_means',
deconvoluted_result_name='deconvoluted',
verbose=True,
database='latest',
subsampling=False,
subsampling_log=True,
subsampling_num_pc=100,
subsampling_num_cells=None,
write=False,
add_to_uns=True):
try:
subsampler = Subsampler(subsampling_log, subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
out = LocalMethodLauncher(cpdb_app.create_app(
verbose, database)).cpdb_analysis_local_method_launcher_scanpy(
adata, var_names, obs_names, obs_key, var_key, gene_id_format,
project_name, threshold, output_path, output_format,
means_result_name, significant_means_result_name,
deconvoluted_result_name, result_precision, subsampler, write,
add_to_uns)
return out
except (ReadFileException, ParseMetaException, ParseCountsException,
ThresholdValueException, AllCountsFilteredException) as e:
app_logger.error(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except EmptyResultException as e:
app_logger.warning(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
def best_path():
saved_file_path = os.path.join(data_dir, 'sources', file_name)
if os.path.exists(download_file_path):
return download_file_path
if os.path.exists(saved_file_path):
return saved_file_path
app_logger.error('Could not find local source for iuphar')
exit(1)
def best_path_for(name: str):
saved_file_path = os.path.join(data_dir, 'sources', name)
if os.path.exists(download_file_path):
return download_file_path
if os.path.exists(saved_file_path):
return saved_file_path
app_logger.error('Could not find local source for {}'.format(name))
exit(1)
def complex_generator(curated_complex: pd.DataFrame,
user_complex: pd.DataFrame,
log_path: str) -> pd.DataFrame:
if (isinstance(user_complex, pd.DataFrame) and user_complex.empty) or \
(not isinstance(user_complex, pd.DataFrame) and not user_complex):
return curated_complex
try:
return _merge_complex(curated_complex, user_complex, log_path)
except MissingRequiredColumns as e:
app_logger.error(e)
def statistical_analysis(meta_filename: str,
counts_filename: str,
project_name: str,
iterations: int,
threshold: float,
result_precision: int,
output_path: str,
means_result_name: str,
pvalues_result_name: str,
significant_mean_result_name: str,
means_pvalues_result_name: str,
deconvoluted_result_name: str,
debug_seed: int,
threads: int,
verbose: bool,
) -> None:
try:
LocalMethodLauncher(cpdb_app.create_app(verbose)). \
cpdb_statistical_analysis_local_method_launcher(meta_filename,
counts_filename,
project_name,
iterations,
threshold,
output_path,
means_result_name,
pvalues_result_name,
significant_mean_result_name,
means_pvalues_result_name,
deconvoluted_result_name,
debug_seed,
threads,
result_precision
)
except (ReadFileException, ParseMetaException, ParseCountsException, ThresholdValueException,
AllCountsFilteredException) as e:
app_logger.error(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except EmptyResultException as e:
app_logger.warning(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except:
app_logger.error('Unexpected error')
if (verbose):
traceback.print_exc(file=sys.stdout)
def _load_meta_counts(counts_filename, meta_filename):
try:
meta_raw = utils.read_data_table_from_file(
os.path.realpath(meta_filename), index_column_first=True)
counts = utils.read_data_table_from_file(
os.path.realpath(counts_filename), index_column_first=True)
except ReadFileException as e:
app_logger.error(e)
exit(1)
meta = pd.DataFrame(index=meta_raw.index)
meta['cell_type'] = meta_raw.iloc[:, 0]
return counts, meta_raw
def download_database(version):
try:
if not version or version == 'latest':
latest_release = _latest_release()
version = latest_release['tag']
zip_to_download = latest_release['url']
else:
releases = _list_releases()
if version not in releases:
app_logger.error('Unavailable version selected')
app_logger.error('Available versions are: {}'.format(', '.join(
releases.keys())))
exit(1)
zip_to_download = releases[version]['url']
print(
'Downloading `{}` release of CellPhoneDB database'.format(version))
output_folder = os.path.expanduser('{}/{}'.format(
cpdb_releases, version))
os.makedirs(output_folder, exist_ok=True)
zip_response = requests.get(zip_to_download)
with zipfile.ZipFile(io.BytesIO(zip_response.content)) as thezip:
root_folder = thezip.namelist()[0]
for name in thezip.namelist():
if name.endswith('/'):
continue
file_folder = os.path.dirname(name)
file_name = os.path.basename(name)
dest_folder = os.path.realpath(
os.path.join(output_folder,
os.path.relpath(file_folder, root_folder)))
dest_file = os.path.join(dest_folder, file_name)
os.makedirs(dest_folder, exist_ok=True)
with thezip.open(name) as zf:
with open(dest_file, 'wb') as fw:
fw.write(zf.read())
except NoReleasesException:
print(
'There are no versions available (or connection could not be made to server to retrieve them)'
)
exit(1)
def analysis(meta_filename: str, counts_filename: str, project_name: str,
threshold: float, output_path: str, means_result_name: str,
deconvoluted_result_name: str, verbose: bool):
try:
LocalMethodLauncher(
cpdb_app.create_app(verbose)).cpdb_analysis_local_method_launcher(
meta_filename,
counts_filename,
project_name,
threshold,
output_path,
means_result_name,
deconvoluted_result_name,
)
except (ReadFileException, ParseMetaException) as e:
app_logger.error(e)
except:
app_logger.error('Unexpected error')
def find_database_for(value: str) -> str:
file_candidate = os.path.expanduser(value)
if os.path.exists(file_candidate):
# todo: warning is perhaps not appropriate, logger doesn't allow info at this point
app_logger.warning(
'User selected database `{}` is available, using it'.format(
file_candidate))
return file_candidate
_ensure_core_version_in_user_dbs()
user_databases_prefix = os.path.expanduser(cpdb_releases)
if not os.path.isdir(user_databases_prefix):
app_logger.error(
'No downloaded databases found, run the `database download` command from the cli first'
)
exit(1)
if value == 'latest' or not value:
available = list_local_versions()
latest_available = available[0]
app_logger.warning(
'Latest local available version is `{}`, using it'.format(
latest_available))
value = latest_available
downloaded_candidate = os.path.join(user_databases_prefix, value,
database_file)
valid_database = os.path.exists(downloaded_candidate)
if valid_database:
# todo: warning is perhaps not appropriate, logger doesn't allow info at this point
app_logger.warning(
'User selected downloaded database `{}` is available, using it'.
format(value))
else:
app_logger.warning(
'User selected database `{}` not available, trying to download it'.
format(value))
download_database(value)
return find_database_for(value)
return downloaded_candidate
def statistical_analysis(meta_filename: str, counts_filename: str,
project_name: str, iterations: str, threshold: float,
output_path: str, means_result_name: str,
pvalues_result_name: str,
significant_mean_result_name: str,
means_pvalues_result_name: str,
deconvoluted_result_name: str, debug_seed: str,
threads: int, verbose: bool):
print(verbose)
try:
LocalMethodLauncher(cpdb_app.create_app(
verbose)).cpdb_statistical_analysis_local_method_launcher(
meta_filename, counts_filename, project_name, iterations,
threshold, output_path, means_result_name, pvalues_result_name,
significant_mean_result_name, means_pvalues_result_name,
deconvoluted_result_name, debug_seed, threads)
except (ReadFileException, ParseMetaException) as e:
app_logger.error(e)
except:
app_logger.error('Unexpected error')
def check_subsampling_params(ctx: Context, argument: Argument, value) -> Any:
subsampling = ctx.params.get('subsampling')
if not subsampling and value is not None:
tpl = 'This parameter ({}) only applies to subsampling, to enable it add `--subsampling` to your command'
app_logger.error(tpl.format(argument.name))
ctx.abort()
if argument.name == 'subsampling_log' and subsampling and value is None:
app_logger.error(
'''In order to perform subsampling you need to specify whether to log1p input counts or not:
to do this specify in your command as --subsampling-log [true|false]'''
)
ctx.abort()
defaults = {'subsampling_num_pc': 100, 'subsampling_num_cells': None}
if subsampling and value is None:
return defaults.get(argument.name, None)
return value
def heatmap_plot(meta_path: str, pvalues_path: str, output_path: str, count_name: str, log_name: str, verbose: bool):
try:
r_plotter.heatmaps_plot(meta_path, pvalues_path, output_path, log_name, count_name)
except MissingR:
print('You cannot perform this plot command unless there is a working R setup according to CellPhoneDB specs')
except RRuntimeException as e:
app_logger.error(str(e))
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
else:
app_logger.error('execute with --verbose to see full stack trace')
try:
s3_access_key = os.environ['S3_ACCESS_KEY']
s3_secret_key = os.environ['S3_SECRET_KEY']
s3_bucket_name = os.environ['S3_BUCKET_NAME']
s3_endpoint = os.environ['S3_ENDPOINT']
rabbit_host = os.environ['RABBIT_HOST']
rabbit_port = os.environ['RABBIT_PORT']
rabbit_user = os.environ['RABBIT_USER']
rabbit_password = os.environ['RABBIT_PASSWORD']
jobs_queue_name = os.environ['RABBIT_JOB_QUEUE']
result_queue_name = os.environ['RABBIT_RESULT_QUEUE']
except KeyError as e:
app_logger.error('ENVIRONMENT VARIABLE {} not defined. Please set it'.format(e))
exit(1)
def create_rabbit_connection():
return pika.BlockingConnection(pika.ConnectionParameters(
host=rabbit_host,
port=rabbit_port,
virtual_host='/',
credentials=credentials
))
app = cpdb_app.create_app()
s3_resource = boto3.resource('s3', aws_access_key_id=s3_access_key,
def statistical_analysis(meta_filename: str,
counts_filename: str,
counts_data='ensembl',
project_name='',
threshold=0.1,
result_precision='3',
output_path='',
output_format='csv',
means_result_name='means',
significant_means_result_name='significant_means',
deconvoluted_result_name='deconvoluted',
verbose=True,
database='latest',
subsampling=False,
subsampling_log=True,
subsampling_num_pc=100,
subsampling_num_cells=None,
debug_seed='-1',
pvalue=0.05,
pvalues_result_name='pvalues',
iterations=1000,
threads=4) -> None:
database = choose_database(None, None, value=database)
try:
subsampler = Subsampler(subsampling_log, subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
LocalMethodLauncher(cpdb_app.create_app(verbose, database)). \
cpdb_statistical_analysis_local_method_launcher(meta_filename,
counts_filename,
counts_data,
project_name,
iterations,
threshold,
output_path,
output_format,
means_result_name,
pvalues_result_name,
significant_means_result_name,
deconvoluted_result_name,
debug_seed,
threads,
result_precision,
pvalue,
subsampler,
)
except (ReadFileException, ParseMetaException, ParseCountsException,
ThresholdValueException, AllCountsFilteredException) as e:
app_logger.error(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except EmptyResultException as e:
app_logger.warning(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
