def setUp(self): """ Set up the database with some orders ready for a credit :return: """ # Build the tests Logger self.log = logging.Logger('Tests') stream = logging.StreamHandler() formatter = logging.Formatter(fmt='%(message)s') stream.setFormatter(formatter) self.log.addHandler(stream) # set us up a bottle application with correct config self.app = bottle.Bottle() config.load(self.app, self.log, join('tests', 'config'), log_output=False) # build the database if it doesn't exist database.build(self.app, self.log, log_output=False) # clear any existing orders in the database conn = database.get_db(self.app) c = conn.cursor() c.execute("DELETE FROM orders") c.execute("DELETE FROM credits") conn.commit() # create test data # 5 test users each with 100 NBT on each exchange/pair/side/rank self.test_data = {} test_order_number = 1 for i in xrange(0, 5): for unit in ['btc', 'ppc']: for side in ['ask', 'bid']: for rank in ['rank_1', 'rank_2']: c.execute("INSERT INTO orders (key,rank,order_id,order_amount," "side,exchange,unit,credited) VALUES " "(%s,%s,%s,%s,%s,%s,%s,%s)", ('TEST_USER_{}'.format(i + 1), rank, test_order_number, 100, side, 'test_exchange', unit, 0)) test_order_number += 1 conn.commit() conn.close() # setup test data for test_get_total_liquidity] # get the orders from the database conn = database.get_db(self.app) c = conn.cursor() c.execute("SELECT * FROM orders") orders = c.fetchall() # get the liquidity as calculated by the main function self.total_liquidity = credit.get_total_liquidity(self.app, orders) # setup data for test_calculate_rewards # target for btc is 2500. total for btc is 2000.0 which is 0.8 of target # so already reward for btc is 0.02 instead of 0.025 # ask and bid are 50:50 so each gets 0.01. rank_1 ratio is 1.0 and rank_2 is 0 for # both. # # target for ppc is 1500. total for ppc is 2000.0 so full reward of 0.0250 # ask is 0.6 * 0.025 = 0.015 # bid is 0.4 * 0.025 = 0.010 # ask rank_1 is 1 # bid rank_1 is 0.8 * 0.010 = 0.008 # bid rank_2 is 0.2 * 0.010 = 0.002 self.rewards = {'test_exchange': {'btc': {'ask': {'rank_1': 0.01, 'rank_2': 0.0}, 'bid': {'rank_1': 0.01, 'rank_2': 0.0} }, 'ppc': {'ask': {'rank_1': 0.015, 'rank_2': 0.0}, 'bid': {'rank_1': 0.008, 'rank_2': 0.002}}}}
app.install(bottle_pgsql.Plugin('dbname={} user={} password={} ' 'host={} port={}'.format(url.path[1:], url.username, url.password, url.hostname, url.port))) else: app.install(bottle_pgsql.Plugin('dbname={} user={} password={} ' 'host={} port={}'.format(app.config['db.name'], app.config['db.user'], app.config['db.pass'], app.config['db.host'], app.config['db.port']))) # Create the database if one doesn't exist database.build(app, log) # Create the Exchange wrapper objects wrappers = {} if 'bittrex' in app.config['exchanges']: wrappers['bittrex'] = src.exchanges.Bittrex() if 'bter' in app.config['exchanges']: wrappers['bter'] = src.exchanges.BTER() if 'ccedk' in app.config['exchanges']: wrappers['ccedk'] = src.exchanges.CCEDK() if 'poloniex' in app.config['exchanges']: wrappers['poloniex'] = src.exchanges.Poloniex() if 'test_exchange' in app.config['exchanges']: wrappers['test_exchange'] = src.exchanges.TestExchange() # save the start time of the server for reporting up-time
def id_spectra( spectra_files: list, database_file: str, verbose: bool = True, min_peptide_len: int = 5, max_peptide_len: int = 20, peak_filter: int = 0, relative_abundance_filter: float = 0.0, ppm_tolerance: int = 20, precursor_tolerance: int = 10, digest: str = '', cores: int = 1, n: int = 5, DEBUG: bool = False, truth_set: str = '', output_dir: str = '' ) -> dict: '''Load in all the spectra and try to create an alignment for every spectrum :param spectra_files: file names of input spectra :type spectra_files: list :param database_file: file name of the fasta database :type database_file: str :param verbose: print progress to the console. (default is True) :type verbose: bool :param min_peptide_len: the minimum length alignment to create (default is 5) :type min_peptide_len: int :param max_peptide_len: the maximum length alignment to create (default is 20) :type max_peptide_len: int :param peak_filter: If set to a number, this metric is used over the relative abundance filter. The most abundanct X peaks to use in the alignment. (default is 0) :type peak_filter: int :param relative_abundance_filter: If peak_filter is set, this parameter is ignored. The relative abundance threshold (in percent as a decimal) a peak must be of the total intensity to be used in the alignment. (default is 0.0) :type relative_abundance_filter: float :param ppm_tolerance: the parts per million error allowed when trying to match masses (default is 20) :type ppm_tolerance: int :param precursor_tolerance: the parts per million error allowed when trying to match a calculated precursor mass to the observed precursor mass (default is 10) :type precurosor_tolerance: int :param digest: the type of digest used in the sample preparation. If left blank, a digest-free search is performed. (default is '') :type digest: str :param cores: the number of cores allowed to use in running the program. If a number provided is greater than the number of cores available, the maximum number of cores is used. (default is 1) :type cores: int :param n: the number of aligments to keep per spectrum. (default is 5) :type n: int :param DEBUG: DEVELOPMENT USE ONLY. Used only for timing of modules. (default is False) :type DEBUG: bool :param truth_set: the path to a json file of the desired alignments to make for each spectrum. The format of the file is {spectrum_id: {'sequence': str, 'hybrid': bool, 'parent': str}}. If left an empty string, the program proceeds as normal. Otherwise results of the analysis will be saved in the file 'fall_off.json' saved in the output directory specified. (default is '') :type truth_set: str :param output_dir: the full path to the output directory to save all output files. (default is '') :type output_dir: str :returns: alignments for all spectra save in the form {spectrum.id: Alignments} :rtype: dict ''' DEV = False truth = None # for dev use only. If a truth set is passed in, we can check where results # drop off. if is_json(truth_set) and is_file(truth_set): DEV = True print( ''' DEV set to True. Tracking when correct answer falls off. Results are stored in a json named 'fall_off.json' in the specified output directory File will be of the form { spectrum_id: { hybrid: bool, truth_sequence: str, fall_off_operation: str, } } ''' ) # load in the truth set truth = json.load(open(truth_set, 'r')) fall_off = None # build/load the database verbose and print('Loading database...') db = database.build(database_file) verbose and print('Done') # load all of the spectra verbose and print('Loading spectra...') spectra, boundaries, mz_mapping = preprocessing_utils.load_spectra( spectra_files, ppm_tolerance, peak_filter=peak_filter, relative_abundance_filter=relative_abundance_filter ) verbose and print('Done') # get the boundary -> kmer mappings for b and y ions matched_masses_b, matched_masses_y, db = merge_search.match_masses(boundaries, db, max_peptide_len) # keep track of the alingment made for every spectrum results = {} if DEV: fall_off = {} fall_off = mp.Manager().dict() truth = mp.Manager().dict(truth) # if we only get 1 core, don't do the multiporcessing bit if cores == 1: # go through and id all spectra for i, spectrum in enumerate(spectra): print(f'Creating alignment for spectrum {i+1}/{len(spectra)} [{to_percent(i+1, len(spectra))}%]', end='\r') # get b and y hits b_hits, y_hits = [], [] for mz in spectrum.spectrum: # get the correct boundary mapped = mz_mapping[mz] b = boundaries[mapped] b = hashable_boundaries(b) if b in matched_masses_b: b_hits += matched_masses_b[b] if b in matched_masses_y: y_hits += matched_masses_y[b] is_last = DEBUG and i == len(spectra) - 1 # pass it into id_spectrum results[spectrum.id] = id_spectrum( spectrum, db, b_hits, y_hits, ppm_tolerance, precursor_tolerance, n, digest_type=digest, truth=truth, fall_off=fall_off, is_last=is_last ) else: print('Initializing other processors...') results = mp.Manager().dict() if DEV: fall_off = mp.Manager().dict() truth = mp.Manager().dict(truth) # start up processes and queue for parallelizing things q = mp.Manager().Queue() num_processes = cores ps = [ mp.Process( target=mp_id_spectrum, args=(q, copy.deepcopy(db), results, fall_off, truth) ) for _ in range(num_processes) ] # start each of the process for p in ps: p.start() print('Done.') # go through and id all spectra for i, spectrum in enumerate(spectra): # get b and y hits b_hits, y_hits = [], [] for mz in spectrum.spectrum: # get the correct boundary mapped = mz_mapping[mz] b = boundaries[mapped] b = hashable_boundaries(b) if b in matched_masses_b: b_hits += matched_masses_b[b] if b in matched_masses_y: y_hits += matched_masses_y[b] # create a named tuple to put in the database o = MPSpectrumID( b_hits, y_hits, spectrum, ppm_tolerance, precursor_tolerance, n, digest ) q.put(o) while len(results) < len(spectra): print(f'\rCreating an alignment for {len(results)}/{len(spectra)} [{to_percent(len(results), len(spectra))}%]', end='') time.sleep(1) # now send 'exit' message to all our processes [q.put('exit') for _ in range(num_processes)] # join them for p in ps: p.join() # if we have set DEV, we need to dump this to a json if DEV: output_dir = output_dir + '/' if output_dir[-1] != '/' else output_dir safe_write_fall_off = {} # we need to convert all our DEVFallOffEntries to dicts for k, v in fall_off.items(): safe_write_fall_off[k] = v._asdict() JSON.save_dict(output_dir + 'fall_off.json', safe_write_fall_off) return results