def process_method(method, properties, body) -> dict: metadata = json.loads(body.decode('utf-8')) job_id = metadata['job_id'] app_logger.info('New Job Queued: {}'.format(job_id)) meta = read_data_from_s3(metadata['file_meta'], s3_bucket_name, index_column_first=False) counts = read_data_from_s3(metadata['file_counts'], s3_bucket_name, index_column_first=True) subsampler = Subsampler(bool(metadata['log']), int(metadata['num_pc']), int(metadata['num_cells']) if metadata.get('num_cells', False) else None ) if metadata.get('subsampling', False) else None database_version = metadata.get('database_version', 'latest') if database_version not in list_local_versions() + ['latest']: database_version = 'latest' app = cpdb_app.create_app(verbose=False, database_file=find_database_for(database_version)) if metadata['iterations']: response = statistical_analysis(app, meta, counts, job_id, metadata, subsampler) else: response = non_statistical_analysis(app, meta, counts, job_id, metadata, subsampler) return response
def process_plot(method, properties, body) -> dict: metadata = json.loads(body.decode('utf-8')) job_id = metadata['job_id'] app_logger.info('New Plot Queued: {}'.format(job_id)) plot_type = metadata.get('type', None) if plot_type == 'dot_plot': return dot_plot_results(metadata.get('file_means'), metadata.get('file_pvalues'), metadata.get('file_rows', None), metadata.get('file_columns', None), job_id ) if plot_type == 'heatmaps_plot': return heatmaps_plot_results(metadata.get('file_meta'), metadata.get('file_pvalues'), job_id ) return { 'job_id': job_id, 'success': False, 'error': { 'id': 'UnknownPlotType', 'message': 'Given plot type does not exist: {}'.format(plot_type) } }
def wrapper(namefile='', data_path=''): app_logger.info('Collecting {}'.format(method_name)) if not namefile: namefile = '{}.csv'.format(method_name) if not data_path: data_path = data_dir data = pd.read_csv('{}/{}'.format(data_path, namefile)) getattr(cellphonedb_app.cellphonedb.collect, method_name)(data)
def process_job(method, properties, body) -> dict: metadata = json.loads(body.decode('utf-8')) job_id = metadata['job_id'] app_logger.info('New Job Queued: {}'.format(job_id)) meta = read_data_from_s3(metadata['file_meta'], s3_bucket_name) counts = read_data_from_s3(metadata['file_counts'], s3_bucket_name) counts = counts.astype(dtype=pd.np.float64, copy=False) if metadata['iterations']: response = statistical_analysis(meta, counts, job_id, metadata) else: response = non_statistical_analysis(meta, counts, job_id, metadata) return response
def wrapper(namefile='', data_path=''): app_logger.info('Collecting {}'.format(method_name)) if not namefile: namefile = '{}_input.csv'.format(method_name) if not data_path: data_path = data_dir data = utils.read_data_table_from_file('{}/{}'.format( data_path, namefile)) if self.database_file: getattr( create_app(True, self.database_file, True).collect, method_name)(data) else: getattr(cellphonedb_app.cellphonedb.collect, method_name)(data)
def process_job(method, properties, body) -> dict: metadata = json.loads(body.decode('utf-8')) job_id = metadata['job_id'] app_logger.info('New Job Queued: {}'.format(job_id)) meta_raw = read_data_from_s3(metadata['file_meta'], s3_bucket_name) try: meta = pd.DataFrame(index=meta_raw.index) meta['cell_type'] = meta_raw.iloc[:, 0] except: raise ParseMetaException counts = read_data_from_s3(metadata['file_counts'], s3_bucket_name) if metadata['iterations']: response = statistical_analysis(meta, counts, job_id, metadata) else: response = non_statistical_analysis(meta, counts, job_id, metadata) return response
jobs_runned = 0 while jobs_runned < 3: job = channel.basic_get(queue=jobs_queue_name, no_ack=True) if all(job): try: job_response = process_job(*job) # TODO: Find more elegant solution connection = create_rabbit_connection() channel = connection.channel() channel.basic_qos(prefetch_count=1) channel.basic_publish(exchange='', routing_key=result_queue_name, body=json.dumps(job_response)) app_logger.info('JOB %s PROCESSED' % job_response['job_id']) except Exception as e: error_response = { 'job_id': json.loads(job[2].decode('utf-8'))['job_id'], 'success': False, 'error': { 'id': 'unknown_error', 'message': '' } } print(traceback.print_exc(file=sys.stdout)) app_logger.error('[-] ERROR DURING PROCESSING JOB %s' % error_response['job_id']) if connection.is_closed: connection = create_rabbit_connection() channel = connection.channel() channel.basic_qos(prefetch_count=1)
def __getattribute__(self, name): method = object.__getattribute__(self, name) if hasattr(method, '__call__'): app_logger.info('Launching Method {}'.format(name)) return method
def signal_handler(sig, frame): app_logger.info( '{} signal received. No more jobs will be consumed.'.format(sig)) global consume_more_jobs consume_more_jobs = False