def tii_symbols(): client = c() symbols = client.list_stock_tickers() e = get_exchange('') for symbol in symbols: try: meta = client.get_ticker_metadata(symbol['ticker']) except HTTPError as err: print(colored.red(err)) except Exception as err: print(colored.red(err)) if symbol['assetType'] == 'Stock': market_type = 'cs' else: print(symbol['assetType']) try: Market.create(symbol=symbol['ticker'], name=meta['name'], exchange=e, market_type=market_type, description=meta['description']) print(colored.green(symbol['ticker'])) except IntegrityError: DB.rollback() except Exception as err: print(colored.red(err)) DB.rollback()
def run_snp_variations(group_info, user): group_name = group_info.keys() groupA = group_info[group_name[0]] groupB = group_info[group_name[1]] create_group_info(groupA, groupB, filename='vs'.join(group_name)) cmd = "python {script} -i {input} -o {output} -g {group} -d {depth}".format( script=os.path.join(SNP_SCRIPT_DIR, SCRIPT_FILE), input=INPUT_TABLE, output=os.path.join(basedir, 'app', 'static', 'variation_results', 'vs'.join(group_name) + '_table'), group=os.path.join(SNP_SCRIPT_DIR, 'vs'.join(group_name)), depth='5') subprocess.call(cmd, shell=True) os.chdir(os.path.join(basedir, 'app', 'static', 'variation_results')) zip_cmd = 'zip {0} {1}'.format('vs'.join(group_name) + '_table.zip', 'vs'.join(group_name) + '_table') subprocess.call(zip_cmd, shell=True) ''' rm_cmd = 'rm -rf {0}'.format( 'vs'.join(group_name) + '_table' ) subprocess.call(rm_cmd, shell=True) ''' db = DB() results = db.execute( "select email from user where username='******'".format(user)) if results[0][0]: to = results[0][0] send_mail(to, 'Snp Variation Results', 'mail/variation_results', user=user, filename='vs'.join(group_name) + '_table') return 'done'
def run_blast_result(genename): # before run blast rm last search results anyway if os.listdir(BLAST_OUT_PATH): rm_cmd = 'rm {}'.format(os.path.join(BLAST_OUT_PATH, '*')) subprocess.call(rm_cmd, shell=True) # get search trans genes db = DB() results = db.execute( "select GENE_TRANS from geneTrans_map where GENE='{gene}'".format( gene=genename)) results = [result[0] for result in results] if results: blast_cmd = "blastdbcmd -entry {genename} -db '{db}' -line_length 100 -out {out}" for gene_trans in results: run_cds_cmd = blast_cmd.format( genename=gene_trans, db=os.path.join(BASE_DB_DIR, BLAST_CDS_DB), out=os.path.join(BLAST_OUT_PATH, 'gene.cds.' + gene_trans)) run_protein_cmd = blast_cmd.format( genename=gene_trans, db=os.path.join(BASE_DB_DIR, BLAST_PROTEIN_DB), out=os.path.join(BLAST_OUT_PATH, 'gene.protein.' + gene_trans)) subprocess.call(run_cds_cmd, shell=True) subprocess.call(run_protein_cmd, shell=True) blast_results = get_blast_result(results) return blast_results return {}
def get_locus_result(genename, blast_results): cds_seq_dict = blast_results.get('cds_seq', 'NA') pro_seq_dict = blast_results.get('pro_seq', 'NA') db = DB() locus_result = {} cmd = """select l.*, f.BLAST_Hit_Accession, f.Description, f.Pfam_ID, f.Interpro_ID, f.GO_ID from locus l left join func f on l.GENE_ID=f.GENE_ID where l.GENE_ID='{0}'; """.format(genename) result = db.execute(cmd, get_all=False) if result: gene_id, chr, pos_start, pos_end = result[1:5] blast_hit, description, pfam_id, interpro_id, go_id = result[5:] locus_result['gene_identification'] = { 'Gene Product Name': description, 'Locus Name': genename } locus_result['gene_attributes'] = { 'Chromosome': chr, "Gene Postion": '{start} - {end}'.format(start=pos_start, end=pos_end) } header = [ 'Accession', 'Description', 'Pfam_ID', 'Interpro_ID', 'GO_ID' ] locus_result['gene_annotation'] = {} locus_result['gene_annotation']['header'] = header locus_result['gene_annotation']['body'] = [ blast_hit, description, pfam_id, interpro_id, go_id ] locus_result['gene_cds_seq'] = cds_seq_dict locus_result['gene_pro_seq'] = pro_seq_dict return locus_result
def __init__(self): self.__config = ConfigParser() self.__config.read('app/config') self.__db = DB() self.__retries = 0 self.__max_retries = int( self.__config.get('komornik.pl', 'max_retries'))
def fetch_blast_result(genename): MAX_ROW_LEN = 125 db = DB() command = "select GENE_ID,VAL from {table} where GENE_ID like '{gene}%'" pep_results = db.execute(command.format(table='pep_tb', gene=wildcard_gene(genename))) cds_results = db.execute(command.format(table='cds_tb', gene=wildcard_gene(genename))) if len(pep_results) == 0 and len(cds_results) == 0: return {} pro_seq = {k:v for k,v in pep_results} cds_seq = {k:v for k,v in cds_results} # print it pretty for k,v in pro_seq.items(): if len(v) > MAX_ROW_LEN: i = 0 over_len = math.ceil(len(v) / MAX_ROW_LEN) * MAX_ROW_LEN tmp_str = "" while i < over_len: tmp_str += v[i:i+MAX_ROW_LEN] + '\n' i += MAX_ROW_LEN pro_seq[k] = tmp_str for k,v in cds_seq.items(): if len(v) > MAX_ROW_LEN: i = 0 over_len = math.ceil(len(v) / MAX_ROW_LEN) * MAX_ROW_LEN tmp_str = "" while i < over_len: tmp_str += v[i:i+MAX_ROW_LEN] + '\n' i += MAX_ROW_LEN cds_seq[k] = tmp_str return {'pro_seq': pro_seq, 'cds_seq': cds_seq}
def batch_query_gene(genes, max_input=1000): ''' get a gene string by search locus database ''' if ',' in genes: gene_list = [] genes = [each.split(',') for each in genes.split()] for gene_part in genes: gene_list += gene_part else: gene_list = genes.split() if len(gene_list) > max_input: return [] if len(gene_list) == 1: _search = "('{0}')".format(gene_list[0]) else: _search = tuple([str(each) for each in gene_list]) db = DB() cmd = """select l.*, f.Description, f.Pfam_Description, f.Interpro_Description, f.GO_Description from locus l left join func f on l.GENE_ID=f.GENE_ID where l.GENE_ID in {0}; """.format(_search) result = db.execute(cmd) if result: result = [(each[1],) + each[5:] for each in result] df1 = pd.DataFrame(result) df2 = pd.DataFrame(gene_list) df3 = pd.merge(df2, df1, how='left').fillna("") return [list(df3.iloc[i,:]) for i in range(len(df3))] return []
def setUp(self): self.redis.flushall() self.vpn = DB.Vpn('abc') self.vpn.update( veth = "fakeveth", veth_state = "down", chal = DB.Challenge('chalfoo') ) self.vpn.chal.files.extend(["test-compose.yml"]) self.user = DB.User('auserid') self.user.update( vlan = 4000, cn = 'ausername', status = "active" ) self.addr = Address('127.0.0.1', 42) self.connection = DB.Connection(self.addr) self.connection.update( addr = self.addr, alive = True, user = self.user, vpn = self.vpn ) self.cluster = None self.user.connections.add(self.connection) self.vpn.links[4000] = 'bridged'
def fetch_expression_data(gene_id, samples, table="iwgsc_refseq"): sample_str = ','.join(samples) cmd = "select {samples} from {table} where gene='{gene}'".format( samples=sample_str, table=table, gene=gene_id) db = DB() results = db.execute(cmd) if len(results) == 0: return [] return list(results[0])
def create_app(): """ Creates the flask app :return: Delayed jobs flask app """ base_path = RUN_CONFIG.get('base_path', '') outputs_base_path = RUN_CONFIG.get('outputs_base_path', 'outputs') flask_app = Flask(__name__, static_url_path=f'{base_path}/{outputs_base_path}', static_folder=job_submission_service.JOBS_OUTPUT_DIR) # flask_app.config['SERVER_NAME'] = RUN_CONFIG.get('server_public_host') flask_app.config['SQLALCHEMY_DATABASE_URI'] = RUN_CONFIG.get( 'sql_alchemy').get('database_uri') flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = RUN_CONFIG.get( 'sql_alchemy').get('track_modifications') flask_app.config['SECRET_KEY'] = RUN_CONFIG.get('server_secret_key') enable_cors = RUN_CONFIG.get('enable_cors', False) if enable_cors: CORS(flask_app) with flask_app.app_context(): DB.init_app(flask_app) CACHE.init_app(flask_app) RATE_LIMITER.init_app(flask_app) for handler in flask_app.logger.handlers: RATE_LIMITER.logger.addHandler(handler) create_tables = RUN_CONFIG.get('sql_alchemy').get( 'create_tables', False) if create_tables: DB.create_all() generate_default_config = RUN_CONFIG.get('generate_default_config', False) if generate_default_config: delayed_job_models.generate_default_job_configs() flask_app.register_blueprint(SWAGGER_BLUEPRINT, url_prefix=f'{base_path}/swagger') flask_app.register_blueprint(SUBMISSION_BLUEPRINT, url_prefix=f'{base_path}/submit') flask_app.register_blueprint(JOB_STATUS_BLUEPRINT, url_prefix=f'{base_path}/status') flask_app.register_blueprint(ADMIN_AUTH_BLUEPRINT, url_prefix=f'{base_path}/admin') flask_app.register_blueprint(ADMIN_TASKS_BLUEPRINT, url_prefix=f'{base_path}/admin') flask_app.register_blueprint( CUSTOM_STATISTICS_BLUEPRINT, url_prefix=f'{base_path}/custom_statistics') return flask_app
def insert_market(exchange, name): for market in exchange.load_markets(): print(market) try: CryptoMarket.create(market=market, exchange=CryptoExchange.get(title=name).id) except IntegrityError: DB.rollback() except Exception as err: print(colored.red(err))
def put_exchanges(): for e in ccxt.exchaanges: print(e) try: if e is not None: CryptoExchange.create(title=e) except IntegrityError: DB.rollback() except Exception as err: print(colored.red(err))
def save_register(form_data): db = DB() user = form_data['user'] passwd = form_data['password'] email = form_data['email'] db.insert('users',{'username':user, 'password':passwd, 'email':email, 'create_at':datetime.now().strftime("%y-%m-%d %H:%M")}) return True, 'ok'
class CustomJobConfig(DB.Model): """ Class that represents a custom key-value instance for the configuration of jobs, that will be passed to the job's run params """ job_type = DB.Column(DB.String(length=60), DB.ForeignKey('default_job_config.job_type'), nullable=False, primary_key=True) key = DB.Column(DB.Text, primary_key=True) value = DB.Column(DB.Text)
class User(DB.Model): __tablename__ = "users" id = DB.Column(DB.Integer, primary_key=True) phone_number = DB.Column(DB.Text, nullable=False, unique=True) confirmed = DB.Column(DB.Boolean, nullable=False, default=False) last_active = DB.Column(DB.DateTime) @validates("phone_number") def validate_phone_number(self, _, phone_number): return normalize_number(phone_number)
def test_db(): db = DB() with testing.postgresql.Postgresql() as postgresql: db.conn = psycopg2.connect(**postgresql.dsn()) db.cur = db.conn.cursor() db.cur.execute( "CREATE TABLE django_session (session_key varchar(255),session_data varchar(255),expire_date varchar(255))" ) db.cur.execute( "INSERT INTO django_session (session_key, session_data, expire_date) VALUES (%s, %s, %s)", ('123', "321", datetime.datetime.now())) assert db.get_session_data('123') == '321'
def check_login(form_data): db = DB() user = unicode(form_data['user']) result = db.execute(get_user_cmd.format(user.encode('utf-8')), get_all=False) if result: passwd = form_data['password'] password, is_active = db.execute(get_passwd_cmd.format(user.encode('utf-8')), get_all=False) if password == passwd and is_active == 'Y': return True, 'ok' elif password != passwd: return False, 'password error' return False, 'user not active' return False, 'user not found'
def iex_symbols(): markets = get_available_symbols(output_format='pandas') e = get_exchange('IEX') for m in markets: try: Market.create(symbol=m['symbol'], name=m['name'], market_type=m['type'], exchange=e, enabled=m['isEnabled']) print(colored.green(m)) except IntegrityError: DB.rollback() except Exception as err: print(colored.red(err)) DB.rollback()
def setup(): clean_db_cache('test_users') db = DB() db.insert( 'test_users', { 'username': '******', 'password': '******', 'email': '*****@*****.**', 'create_at': datetime.datetime.now().strftime("%y-%m-%d %H:%M"), 'is_active': 'Y', 'snp_table': 'snp_mRNA_table', 'expr_table': 'expr_gene_pos', 'desc_table': 'locus_gene_mlocus' })
def test_cluster_worker_ensure_cluster_up(self): with Shim(sut) as manager: with Shim(sut.ClusterWorker) as Worker: class FakeCmd: UP = object() @assertCalled(self, 1) def __init__(this, action, project, files): pass @assertCalled(self, 1) def run(self): pass @assertCalled(self, 1) def bridge_link_if_ready(this, user, vpn, cluster): self.assertEqual(self.vpn.id, vpn.id) self.assertEqual(self.user.id, user.id) self.assertEqual(DB.Cluster(user, vpn).id, cluster.id) Worker.bridge_link_if_ready = bridge_link_if_ready manager.ComposeCmd = FakeCmd cluster = DB.Cluster(self.user, self.vpn) cluster.delete() worker = manager.ClusterWorker('foo', 'bar') worker.ensure_cluster_up(self.user, self.vpn, cluster, self.connection) worker.ensure_cluster_up(self.user, self.vpn, cluster, self.connection) self.assertEqual(cluster.status, 'up')
def get_expr_table(table, gene_ids, groupA, groupB, map_groupA, map_groupB): db = DB() select_columns = ['GENE_ID', 'CHR', 'POS_START', 'POS_END' ] + groupA + groupB select_columns_str = ','.join(select_columns) results = [] for gene in gene_ids: cmd = "select {columns} from {table} where GENE_ID='{gene_id}';".format( columns=select_columns_str, table=table, gene_id=gene) result = db.execute(cmd, get_all=False) if not result: return (gene, '') results.append(list(result)) return_select_columns = ['GENE_ID', 'CHR', 'POS_START', 'POS_END' ] + map_groupA + map_groupB return return_select_columns, results
def test_cluster_worker_ensure_cluster_stopped(self): with Shim(sut) as manager: class FakeCmd: STOP = object() @assertCalled(self, 1) def __init__(this, action, project, files): pass @assertCalled(self, 1) def run(self): pass manager.ComposeCmd = FakeCmd cluster = DB.Cluster(self.user, self.vpn) cluster.delete() worker = manager.ClusterWorker('foo', 'bar') worker.ensure_cluster_stopped(self.user, self.vpn, cluster) cluster.status = 'up' worker.ensure_cluster_stopped(self.user, self.vpn, cluster) self.assertEqual(cluster.status, 'stopped') worker.ensure_cluster_stopped(self.user, self.vpn, cluster) self.assertEqual(cluster.status, 'stopped')
async def ws_handler(self, websocket, path): self.socket = websocket try: session = parse_cookie( websocket.request_headers['cookie'])['sessionid'] self.id = int( session_decoder.decode( DB().get_session_data(session), constants.DJANGO_SECRET_KEY)['_auth_user_id']) except BaseException as e: print('auth failed {}'.format(e)) self.id = -1 if self.id != -1: print('Successfully authorized {}'.format(self.id)) client = ClientSession(self.id, websocket) self.distributor.process_ws_connection(client) try: while True: pack = await websocket.recv() print('WS {} received {}'.format(self.id, pack)) self.distributor.process_ws_packet(pack, client) except websockets.ConnectionClosed as exc: self.distributor.process_ws_disconnection(client) print("exc", exc) print("User {} disconnected".format(self.id)) await websocket.close()
def test_vlan_worker_bridge_cluster(self): with Shim(sut) as manager: class FakeCmd: ADDIF = object() @assertCalled(self, 1) def __init__(this, action, bridge_id, vlan_if): pass @assertCalled(self, 1) def run(self): pass manager.BrctlCmd = FakeCmd manager.get_bridge_id = lambda cluster_id: "bogus" # The real function requires docker cluster = DB.Cluster(self.user, self.vpn) self.vpn.links[self.user.vlan] = 'up' worker = manager.VlanWorker('foo', 'bar') worker.bridge_cluster(self.vpn, self.user) self.assertNotEqual(self.vpn.links[self.user.vlan], 'bridged') cluster.status = "up" worker.bridge_cluster(self.vpn, self.user) self.assertEqual(self.vpn.links[self.user.vlan], 'bridged')
def __init__(self): super().__init__(AppConfig(), DConfigSettings(), DValueSettings()) self.db: DB = DB(self.database) # services self.data_service = DataService(self.base_params) # scheduler self.scheduler.add_job(self.data_service.generate_data, 6000)
async def register_product(name, password): if not Product.pattern_of_name_of_product.match(name): return { "failure": True, "message": "Wrong name" } elif not Product.pattern_of_passwords.match(password): return { "failure": True, "message": "Wrong password" } date_base = DB.DataBase("localhost", 27017) products = await date_base.get_products_collection() if await products.count_documents({"name": name}) > 0: return { "failure": True, "message": "No old_secret and secret params" } result = await products.insert_one({"name": name, "password": bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')}) id = result.inserted_id.binary.hex() return {"result": id}
def eod_symbols(e='US'): df = get_exchange_symbols(exchange_code=e) for i in range(len(df)): try: symbol = df.ix[i].name name = df.ix[i]['Name'] exchange = df.ix[i]['Exchange'] if exchange != 'NaN': e = ge(exchange) else: e = ge('') Market.create(symbol=symbol, name=name, exchange=e) print(colored.green(symbol)) except IntegrityError: DB.rollback() except Exception as err: print(colored.red(err)) DB.rollback()
def insert_vote(cls, candidate_id): session = DB.get_session() new_vote = Vote(reference=candidate_id, up_date=datetime.now()) session.add(new_vote) session.flush() session.refresh(new_vote) id_to_return = new_vote.id session.commit() return id_to_return
class InputFile(DB.Model): """ Class that represents an input file to the job """ id = DB.Column(DB.Integer, primary_key=True) input_key = DB.Column(DB.String(length=120)) internal_path = DB.Column(DB.Text, nullable=False) public_url = DB.Column(DB.Text) job_id = DB.Column(DB.String(length=120), DB.ForeignKey('delayed_job.id'), nullable=False)
def get_locus_result(genename, blast_results): cds_seq_dict = blast_results.get('cds_seq', 'NA') pro_seq_dict = blast_results.get('pro_seq', 'NA') db = DB() locus_result = {} cmd = """select l.*, f.Description, f.Pfam_Description, f.Interpro_Description, f.GO_Description from locus l left join func f on l.GENE_ID=f.GENE_ID where l.GENE_ID='{0}'; """.format(genename) result = db.execute(cmd, get_all=False) if result: locus_result['orthologous_gene'] = {} ortho_header = ['Arabidopsis_thaliana', 'Hordeum_vulgare', 'Oryza_sativa', 'Triticum_aestivum', 'Zea_mays'] locus_result['orthologous_gene']['header'] = ortho_header locus_result['orthologous_gene']['body'] = [] cmd = "select l.GENE_ID, o.* from locus l left join ortho o on l.GENE_ID=o.GENE_ID where l.GENE_ID='{0}';".format(genename) ortho_result = db.execute(cmd, get_all=False) if ortho_result: ortho_result_list = ortho_result[3:] ortho_result_list = [printPretty(each) for each in ortho_result_list if each is not None] locus_result['orthologous_gene']['body'] = ortho_result_list gene_id, chr, pos_start, pos_end = result[1:5] description, pfam_desc, interpro_desc, go_desc = result[5:] locus_result['gene_identification'] = {'Gene Product Name': description, 'Locus Name': genename} locus_result['gene_attributes'] = {'Chromosome': chr, "Gene Postion":'{start} - {end}'.format(start=pos_start, end=pos_end)} header = ['Description', 'Pfam_Description', 'Interpro_Description', 'GO_Description'] locus_result['gene_annotation'] = {} locus_result['gene_annotation']['header'] = header locus_result['gene_annotation']['body'] = [description, pfam_desc, interpro_desc, go_desc] # match 01G and 02G TraesCS1A02G000100 #result = db.execute("select * from tissue_expression where Gene_id='{0}'".format(genename)) result = db.execute("select * from tissue_expression where Gene_id like '{0}'".format(wildcard_gene(genename))) if result: row = [float(each) for each in result[0][2:]] else: row = [] locus_result['tissue_expression'] = row locus_result['gene_cds_seq'] = cds_seq_dict locus_result['gene_pro_seq'] = pro_seq_dict return locus_result