def analysis(meta_filename: str,
counts_filename: str,
counts_data: str,
project_name: str,
threshold: float,
result_precision: int,
output_path: str,
output_format: str,
means_result_name: str,
significant_means_result_name: str,
deconvoluted_result_name: str,
verbose: bool,
database: Optional[str],
subsampling: bool,
subsampling_log: bool,
subsampling_num_pc: int,
subsampling_num_cells: Optional[int]
):
try:
subsampler = Subsampler(subsampling_log,
subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
LocalMethodLauncher(cpdb_app.create_app(verbose,
database)).cpdb_analysis_local_method_launcher(meta_filename,
counts_filename,
counts_data,
project_name,
threshold,
output_path,
output_format,
means_result_name,
significant_means_result_name,
deconvoluted_result_name,
result_precision,
subsampler,
)
except (ReadFileException, ParseMetaException, ParseCountsException, ThresholdValueException,
AllCountsFilteredException) as e:
app_logger.error(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except EmptyResultException as e:
app_logger.warning(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
def process_method(method, properties, body) -> dict:
metadata = json.loads(body.decode('utf-8'))
job_id = metadata['job_id']
app_logger.info('New Job Queued: {}'.format(job_id))
meta = read_data_from_s3(metadata['file_meta'], s3_bucket_name, index_column_first=False)
counts = read_data_from_s3(metadata['file_counts'], s3_bucket_name, index_column_first=True)
subsampler = Subsampler(bool(metadata['log']),
int(metadata['num_pc']),
int(metadata['num_cells']) if metadata.get('num_cells', False) else None
) if metadata.get('subsampling', False) else None
database_version = metadata.get('database_version', 'latest')
if database_version not in list_local_versions() + ['latest']:
database_version = 'latest'
app = cpdb_app.create_app(verbose=False, database_file=find_database_for(database_version))
if metadata['iterations']:
response = statistical_analysis(app, meta, counts, job_id, metadata, subsampler)
else:
response = non_statistical_analysis(app, meta, counts, job_id, metadata, subsampler)
return response
def analysis_scanpy(adata,
var_names,
obs_names,
obs_key,
var_key=None,
gene_id_format=None,
project_name='',
threshold=0.1,
result_precision='3',
output_path='',
output_format='csv',
means_result_name='means',
significant_means_result_name='significant_means',
deconvoluted_result_name='deconvoluted',
verbose=True,
database='latest',
subsampling=False,
subsampling_log=True,
subsampling_num_pc=100,
subsampling_num_cells=None,
write=False,
add_to_uns=True):
try:
subsampler = Subsampler(subsampling_log, subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
out = LocalMethodLauncher(cpdb_app.create_app(
verbose, database)).cpdb_analysis_local_method_launcher_scanpy(
adata, var_names, obs_names, obs_key, var_key, gene_id_format,
project_name, threshold, output_path, output_format,
means_result_name, significant_means_result_name,
deconvoluted_result_name, result_precision, subsampler, write,
add_to_uns)
return out
except (ReadFileException, ParseMetaException, ParseCountsException,
ThresholdValueException, AllCountsFilteredException) as e:
app_logger.error(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except EmptyResultException as e:
app_logger.warning(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)
def collect_database(database, output_path, **kwargs):
database_file_path = os.path.join(output_path, database)
app = create_app(verbose=True,
database_file=database_file_path,
collecting=True)
app.database_manager.database.drop_everything()
app.database_manager.database.create_all()
LocalCollectorLauncher(database_file_path).all(**kwargs)
def statistical_analysis(meta_filename: str,
counts_filename: str,
project_name: str,
iterations: int,
threshold: float,
result_precision: int,
output_path: str,
means_result_name: str,
pvalues_result_name: str,
significant_mean_result_name: str,
means_pvalues_result_name: str,
deconvoluted_result_name: str,
debug_seed: int,
threads: int,
verbose: bool,
) -> None:
try:
LocalMethodLauncher(cpdb_app.create_app(verbose)). \
cpdb_statistical_analysis_local_method_launcher(meta_filename,
counts_filename,
project_name,
iterations,
threshold,
output_path,
means_result_name,
pvalues_result_name,
significant_mean_result_name,
means_pvalues_result_name,
deconvoluted_result_name,
debug_seed,
threads,
result_precision
)
except (ReadFileException, ParseMetaException, ParseCountsException, ThresholdValueException,
AllCountsFilteredException) as e:
app_logger.error(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except EmptyResultException as e:
app_logger.warning(str(e) +
(':' if (hasattr(e, 'description') and e.description) or (
hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description) if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else '')
)
except:
app_logger.error('Unexpected error')
if (verbose):
traceback.print_exc(file=sys.stdout)
def analysis(meta_filename: str, counts_filename: str, project_name: str,
threshold: float, output_path: str, means_result_name: str,
deconvoluted_result_name: str, verbose: bool):
LocalMethodLauncher(
cpdb_app.create_app(verbose)).cpdb_analysis_local_method_launcher(
meta_filename,
counts_filename,
project_name,
threshold,
output_path,
means_result_name,
deconvoluted_result_name,
)
def statistical_analysis(meta_filename: str, counts_filename: str,
project_name: str, iterations: str, threshold: float,
output_path: str, means_result_name: str,
pvalues_result_name: str,
significant_mean_result_name: str,
means_pvalues_result_name: str,
deconvoluted_result_name: str, debug_seed: str,
threads: int, verbose: bool):
print(verbose)
LocalMethodLauncher(cpdb_app.create_app(
verbose)).cpdb_statistical_analysis_local_method_launcher(
meta_filename, counts_filename, project_name, iterations,
threshold, output_path, means_result_name, pvalues_result_name,
significant_mean_result_name, means_pvalues_result_name,
deconvoluted_result_name, debug_seed, threads)
def wrapper(namefile='', data_path=''):
app_logger.info('Collecting {}'.format(method_name))
if not namefile:
namefile = '{}_input.csv'.format(method_name)
if not data_path:
data_path = data_dir
data = utils.read_data_table_from_file('{}/{}'.format(
data_path, namefile))
if self.database_file:
getattr(
create_app(True, self.database_file, True).collect,
method_name)(data)
else:
getattr(cellphonedb_app.cellphonedb.collect, method_name)(data)
def analysis(meta_filename: str, counts_filename: str, project_name: str,
threshold: float, output_path: str, means_result_name: str,
deconvoluted_result_name: str, verbose: bool):
try:
LocalMethodLauncher(
cpdb_app.create_app(verbose)).cpdb_analysis_local_method_launcher(
meta_filename,
counts_filename,
project_name,
threshold,
output_path,
means_result_name,
deconvoluted_result_name,
)
except (ReadFileException, ParseMetaException) as e:
app_logger.error(e)
except:
app_logger.error('Unexpected error')
def statistical_analysis(meta_filename: str, counts_filename: str,
project_name: str, iterations: str, threshold: float,
output_path: str, means_result_name: str,
pvalues_result_name: str,
significant_mean_result_name: str,
means_pvalues_result_name: str,
deconvoluted_result_name: str, debug_seed: str,
threads: int, verbose: bool):
print(verbose)
try:
LocalMethodLauncher(cpdb_app.create_app(
verbose)).cpdb_statistical_analysis_local_method_launcher(
meta_filename, counts_filename, project_name, iterations,
threshold, output_path, means_result_name, pvalues_result_name,
significant_mean_result_name, means_pvalues_result_name,
deconvoluted_result_name, debug_seed, threads)
except (ReadFileException, ParseMetaException) as e:
app_logger.error(e)
except:
app_logger.error('Unexpected error')
except KeyError as e:
app_logger.error('ENVIRONMENT VARIABLE {} not defined. Please set it'.format(e))
exit(1)
def create_rabbit_connection():
return pika.BlockingConnection(pika.ConnectionParameters(
host=rabbit_host,
port=rabbit_port,
virtual_host='/',
credentials=credentials
))
app = cpdb_app.create_app()
s3_resource = boto3.resource('s3', aws_access_key_id=s3_access_key,
aws_secret_access_key=s3_secret_key,
endpoint_url=s3_endpoint)
s3_client = boto3.client('s3', aws_access_key_id=s3_access_key,
aws_secret_access_key=s3_secret_key,
endpoint_url=s3_endpoint)
def read_data_from_s3(filename: str, s3_bucket_name: str):
s3_object = s3_client.get_object(Bucket=s3_bucket_name, Key=filename)
return utils.read_data_from_s3_object(s3_object, filename, index_column_first=True)
def write_data_in_s3(data: pd.DataFrame, filename: str):
def statistical_analysis(meta_filename: str,
counts_filename: str,
counts_data='ensembl',
project_name='',
threshold=0.1,
result_precision='3',
output_path='',
output_format='csv',
means_result_name='means',
significant_means_result_name='significant_means',
deconvoluted_result_name='deconvoluted',
verbose=True,
database='latest',
subsampling=False,
subsampling_log=True,
subsampling_num_pc=100,
subsampling_num_cells=None,
debug_seed='-1',
pvalue=0.05,
pvalues_result_name='pvalues',
iterations=1000,
threads=4) -> None:
database = choose_database(None, None, value=database)
try:
subsampler = Subsampler(subsampling_log, subsampling_num_pc,
subsampling_num_cells,
verbose) if subsampling else None
LocalMethodLauncher(cpdb_app.create_app(verbose, database)). \
cpdb_statistical_analysis_local_method_launcher(meta_filename,
counts_filename,
counts_data,
project_name,
iterations,
threshold,
output_path,
output_format,
means_result_name,
pvalues_result_name,
significant_means_result_name,
deconvoluted_result_name,
debug_seed,
threads,
result_precision,
pvalue,
subsampler,
)
except (ReadFileException, ParseMetaException, ParseCountsException,
ThresholdValueException, AllCountsFilteredException) as e:
app_logger.error(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except EmptyResultException as e:
app_logger.warning(
str(e) + (':' if (hasattr(e, 'description') and e.description) or
(hasattr(e, 'hint') and e.hint) else '') +
(' {}.'.format(e.description)
if hasattr(e, 'description') and e.description else '') +
(' {}.'.format(e.hint) if hasattr(e, 'hint') and e.hint else ''))
except:
app_logger.error('Unexpected error')
if verbose:
traceback.print_exc(file=sys.stdout)