def tii_symbols():
client = c()
symbols = client.list_stock_tickers()
e = get_exchange('')
for symbol in symbols:
try:
meta = client.get_ticker_metadata(symbol['ticker'])
except HTTPError as err:
print(colored.red(err))
except Exception as err:
print(colored.red(err))
if symbol['assetType'] == 'Stock':
market_type = 'cs'
else:
print(symbol['assetType'])
try:
Market.create(symbol=symbol['ticker'],
name=meta['name'],
exchange=e,
market_type=market_type,
description=meta['description'])
print(colored.green(symbol['ticker']))
except IntegrityError:
DB.rollback()
except Exception as err:
print(colored.red(err))
DB.rollback()
def run_snp_variations(group_info, user):
group_name = group_info.keys()
groupA = group_info[group_name[0]]
groupB = group_info[group_name[1]]
create_group_info(groupA, groupB, filename='vs'.join(group_name))
cmd = "python {script} -i {input} -o {output} -g {group} -d {depth}".format(
script=os.path.join(SNP_SCRIPT_DIR, SCRIPT_FILE),
input=INPUT_TABLE,
output=os.path.join(basedir, 'app', 'static', 'variation_results',
'vs'.join(group_name) + '_table'),
group=os.path.join(SNP_SCRIPT_DIR, 'vs'.join(group_name)),
depth='5')
subprocess.call(cmd, shell=True)
os.chdir(os.path.join(basedir, 'app', 'static', 'variation_results'))
zip_cmd = 'zip {0} {1}'.format('vs'.join(group_name) + '_table.zip',
'vs'.join(group_name) + '_table')
subprocess.call(zip_cmd, shell=True)
'''
rm_cmd = 'rm -rf {0}'.format(
'vs'.join(group_name) + '_table'
)
subprocess.call(rm_cmd, shell=True)
'''
db = DB()
results = db.execute(
"select email from user where username='******'".format(user))
if results[0][0]:
to = results[0][0]
send_mail(to,
'Snp Variation Results',
'mail/variation_results',
user=user,
filename='vs'.join(group_name) + '_table')
return 'done'
def run_blast_result(genename):
# before run blast rm last search results anyway
if os.listdir(BLAST_OUT_PATH):
rm_cmd = 'rm {}'.format(os.path.join(BLAST_OUT_PATH, '*'))
subprocess.call(rm_cmd, shell=True)
# get search trans genes
db = DB()
results = db.execute(
"select GENE_TRANS from geneTrans_map where GENE='{gene}'".format(
gene=genename))
results = [result[0] for result in results]
if results:
blast_cmd = "blastdbcmd -entry {genename} -db '{db}' -line_length 100 -out {out}"
for gene_trans in results:
run_cds_cmd = blast_cmd.format(
genename=gene_trans,
db=os.path.join(BASE_DB_DIR, BLAST_CDS_DB),
out=os.path.join(BLAST_OUT_PATH, 'gene.cds.' + gene_trans))
run_protein_cmd = blast_cmd.format(
genename=gene_trans,
db=os.path.join(BASE_DB_DIR, BLAST_PROTEIN_DB),
out=os.path.join(BLAST_OUT_PATH, 'gene.protein.' + gene_trans))
subprocess.call(run_cds_cmd, shell=True)
subprocess.call(run_protein_cmd, shell=True)
blast_results = get_blast_result(results)
return blast_results
return {}
def get_locus_result(genename, blast_results):
cds_seq_dict = blast_results.get('cds_seq', 'NA')
pro_seq_dict = blast_results.get('pro_seq', 'NA')
db = DB()
locus_result = {}
cmd = """select l.*, f.BLAST_Hit_Accession, f.Description, f.Pfam_ID,
f.Interpro_ID, f.GO_ID from locus l left join func f
on l.GENE_ID=f.GENE_ID where l.GENE_ID='{0}';
""".format(genename)
result = db.execute(cmd, get_all=False)
if result:
gene_id, chr, pos_start, pos_end = result[1:5]
blast_hit, description, pfam_id, interpro_id, go_id = result[5:]
locus_result['gene_identification'] = {
'Gene Product Name': description,
'Locus Name': genename
}
locus_result['gene_attributes'] = {
'Chromosome': chr,
"Gene Postion": '{start} - {end}'.format(start=pos_start,
end=pos_end)
}
header = [
'Accession', 'Description', 'Pfam_ID', 'Interpro_ID', 'GO_ID'
]
locus_result['gene_annotation'] = {}
locus_result['gene_annotation']['header'] = header
locus_result['gene_annotation']['body'] = [
blast_hit, description, pfam_id, interpro_id, go_id
]
locus_result['gene_cds_seq'] = cds_seq_dict
locus_result['gene_pro_seq'] = pro_seq_dict
return locus_result
def __init__(self):
self.__config = ConfigParser()
self.__config.read('app/config')
self.__db = DB()
self.__retries = 0
self.__max_retries = int(
self.__config.get('komornik.pl', 'max_retries'))
def fetch_blast_result(genename):
MAX_ROW_LEN = 125
db = DB()
command = "select GENE_ID,VAL from {table} where GENE_ID like '{gene}%'"
pep_results = db.execute(command.format(table='pep_tb', gene=wildcard_gene(genename)))
cds_results = db.execute(command.format(table='cds_tb', gene=wildcard_gene(genename)))
if len(pep_results) == 0 and len(cds_results) == 0:
return {}
pro_seq = {k:v for k,v in pep_results}
cds_seq = {k:v for k,v in cds_results}
# print it pretty
for k,v in pro_seq.items():
if len(v) > MAX_ROW_LEN:
i = 0
over_len = math.ceil(len(v) / MAX_ROW_LEN) * MAX_ROW_LEN
tmp_str = ""
while i < over_len:
tmp_str += v[i:i+MAX_ROW_LEN] + '\n'
i += MAX_ROW_LEN
pro_seq[k] = tmp_str
for k,v in cds_seq.items():
if len(v) > MAX_ROW_LEN:
i = 0
over_len = math.ceil(len(v) / MAX_ROW_LEN) * MAX_ROW_LEN
tmp_str = ""
while i < over_len:
tmp_str += v[i:i+MAX_ROW_LEN] + '\n'
i += MAX_ROW_LEN
cds_seq[k] = tmp_str
return {'pro_seq': pro_seq,
'cds_seq': cds_seq}
def batch_query_gene(genes, max_input=1000):
'''
get a gene string by search locus database
'''
if ',' in genes:
gene_list = []
genes = [each.split(',') for each in genes.split()]
for gene_part in genes:
gene_list += gene_part
else:
gene_list = genes.split()
if len(gene_list) > max_input:
return []
if len(gene_list) == 1:
_search = "('{0}')".format(gene_list[0])
else:
_search = tuple([str(each) for each in gene_list])
db = DB()
cmd = """select l.*, f.Description, f.Pfam_Description,
f.Interpro_Description, f.GO_Description from locus l left join func f
on l.GENE_ID=f.GENE_ID where l.GENE_ID in {0};
""".format(_search)
result = db.execute(cmd)
if result:
result = [(each[1],) + each[5:] for each in result]
df1 = pd.DataFrame(result)
df2 = pd.DataFrame(gene_list)
df3 = pd.merge(df2, df1, how='left').fillna("")
return [list(df3.iloc[i,:]) for i in range(len(df3))]
return []
def setUp(self):
self.redis.flushall()
self.vpn = DB.Vpn('abc')
self.vpn.update(
veth = "fakeveth",
veth_state = "down",
chal = DB.Challenge('chalfoo')
)
self.vpn.chal.files.extend(["test-compose.yml"])
self.user = DB.User('auserid')
self.user.update(
vlan = 4000,
cn = 'ausername',
status = "active"
)
self.addr = Address('127.0.0.1', 42)
self.connection = DB.Connection(self.addr)
self.connection.update(
addr = self.addr,
alive = True,
user = self.user,
vpn = self.vpn
)
self.cluster = None
self.user.connections.add(self.connection)
self.vpn.links[4000] = 'bridged'
def fetch_expression_data(gene_id, samples, table="iwgsc_refseq"):
sample_str = ','.join(samples)
cmd = "select {samples} from {table} where gene='{gene}'".format(
samples=sample_str, table=table, gene=gene_id)
db = DB()
results = db.execute(cmd)
if len(results) == 0:
return []
return list(results[0])
def create_app():
"""
Creates the flask app
:return: Delayed jobs flask app
"""
base_path = RUN_CONFIG.get('base_path', '')
outputs_base_path = RUN_CONFIG.get('outputs_base_path', 'outputs')
flask_app = Flask(__name__,
static_url_path=f'{base_path}/{outputs_base_path}',
static_folder=job_submission_service.JOBS_OUTPUT_DIR)
# flask_app.config['SERVER_NAME'] = RUN_CONFIG.get('server_public_host')
flask_app.config['SQLALCHEMY_DATABASE_URI'] = RUN_CONFIG.get(
'sql_alchemy').get('database_uri')
flask_app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = RUN_CONFIG.get(
'sql_alchemy').get('track_modifications')
flask_app.config['SECRET_KEY'] = RUN_CONFIG.get('server_secret_key')
enable_cors = RUN_CONFIG.get('enable_cors', False)
if enable_cors:
CORS(flask_app)
with flask_app.app_context():
DB.init_app(flask_app)
CACHE.init_app(flask_app)
RATE_LIMITER.init_app(flask_app)
for handler in flask_app.logger.handlers:
RATE_LIMITER.logger.addHandler(handler)
create_tables = RUN_CONFIG.get('sql_alchemy').get(
'create_tables', False)
if create_tables:
DB.create_all()
generate_default_config = RUN_CONFIG.get('generate_default_config',
False)
if generate_default_config:
delayed_job_models.generate_default_job_configs()
flask_app.register_blueprint(SWAGGER_BLUEPRINT,
url_prefix=f'{base_path}/swagger')
flask_app.register_blueprint(SUBMISSION_BLUEPRINT,
url_prefix=f'{base_path}/submit')
flask_app.register_blueprint(JOB_STATUS_BLUEPRINT,
url_prefix=f'{base_path}/status')
flask_app.register_blueprint(ADMIN_AUTH_BLUEPRINT,
url_prefix=f'{base_path}/admin')
flask_app.register_blueprint(ADMIN_TASKS_BLUEPRINT,
url_prefix=f'{base_path}/admin')
flask_app.register_blueprint(
CUSTOM_STATISTICS_BLUEPRINT,
url_prefix=f'{base_path}/custom_statistics')
return flask_app
def insert_market(exchange, name):
for market in exchange.load_markets():
print(market)
try:
CryptoMarket.create(market=market, exchange=CryptoExchange.get(title=name).id)
except IntegrityError:
DB.rollback()
except Exception as err:
print(colored.red(err))
def put_exchanges():
for e in ccxt.exchaanges:
print(e)
try:
if e is not None:
CryptoExchange.create(title=e)
except IntegrityError:
DB.rollback()
except Exception as err:
print(colored.red(err))
def save_register(form_data):
db = DB()
user = form_data['user']
passwd = form_data['password']
email = form_data['email']
db.insert('users',{'username':user,
'password':passwd,
'email':email,
'create_at':datetime.now().strftime("%y-%m-%d %H:%M")})
return True, 'ok'
class CustomJobConfig(DB.Model):
"""
Class that represents a custom key-value instance for the configuration of jobs, that will be passed to the job's
run params
"""
job_type = DB.Column(DB.String(length=60),
DB.ForeignKey('default_job_config.job_type'),
nullable=False,
primary_key=True)
key = DB.Column(DB.Text, primary_key=True)
value = DB.Column(DB.Text)
class User(DB.Model):
__tablename__ = "users"
id = DB.Column(DB.Integer, primary_key=True)
phone_number = DB.Column(DB.Text, nullable=False, unique=True)
confirmed = DB.Column(DB.Boolean, nullable=False, default=False)
last_active = DB.Column(DB.DateTime)
@validates("phone_number")
def validate_phone_number(self, _, phone_number):
return normalize_number(phone_number)
def test_db():
db = DB()
with testing.postgresql.Postgresql() as postgresql:
db.conn = psycopg2.connect(**postgresql.dsn())
db.cur = db.conn.cursor()
db.cur.execute(
"CREATE TABLE django_session (session_key varchar(255),session_data varchar(255),expire_date varchar(255))"
)
db.cur.execute(
"INSERT INTO django_session (session_key, session_data, expire_date) VALUES (%s, %s, %s)",
('123', "321", datetime.datetime.now()))
assert db.get_session_data('123') == '321'
def check_login(form_data):
db = DB()
user = unicode(form_data['user'])
result = db.execute(get_user_cmd.format(user.encode('utf-8')), get_all=False)
if result:
passwd = form_data['password']
password, is_active = db.execute(get_passwd_cmd.format(user.encode('utf-8')), get_all=False)
if password == passwd and is_active == 'Y':
return True, 'ok'
elif password != passwd:
return False, 'password error'
return False, 'user not active'
return False, 'user not found'
def iex_symbols():
markets = get_available_symbols(output_format='pandas')
e = get_exchange('IEX')
for m in markets:
try:
Market.create(symbol=m['symbol'], name=m['name'],
market_type=m['type'],
exchange=e, enabled=m['isEnabled'])
print(colored.green(m))
except IntegrityError:
DB.rollback()
except Exception as err:
print(colored.red(err))
DB.rollback()
def setup():
clean_db_cache('test_users')
db = DB()
db.insert(
'test_users', {
'username': '******',
'password': '******',
'email': '*****@*****.**',
'create_at': datetime.datetime.now().strftime("%y-%m-%d %H:%M"),
'is_active': 'Y',
'snp_table': 'snp_mRNA_table',
'expr_table': 'expr_gene_pos',
'desc_table': 'locus_gene_mlocus'
})
def test_cluster_worker_ensure_cluster_up(self):
with Shim(sut) as manager:
with Shim(sut.ClusterWorker) as Worker:
class FakeCmd:
UP = object()
@assertCalled(self, 1)
def __init__(this, action, project, files):
pass
@assertCalled(self, 1)
def run(self):
pass
@assertCalled(self, 1)
def bridge_link_if_ready(this, user, vpn, cluster):
self.assertEqual(self.vpn.id, vpn.id)
self.assertEqual(self.user.id, user.id)
self.assertEqual(DB.Cluster(user, vpn).id, cluster.id)
Worker.bridge_link_if_ready = bridge_link_if_ready
manager.ComposeCmd = FakeCmd
cluster = DB.Cluster(self.user, self.vpn)
cluster.delete()
worker = manager.ClusterWorker('foo', 'bar')
worker.ensure_cluster_up(self.user, self.vpn, cluster,
self.connection)
worker.ensure_cluster_up(self.user, self.vpn, cluster,
self.connection)
self.assertEqual(cluster.status, 'up')
def get_expr_table(table, gene_ids, groupA, groupB, map_groupA, map_groupB):
db = DB()
select_columns = ['GENE_ID', 'CHR', 'POS_START', 'POS_END'
] + groupA + groupB
select_columns_str = ','.join(select_columns)
results = []
for gene in gene_ids:
cmd = "select {columns} from {table} where GENE_ID='{gene_id}';".format(
columns=select_columns_str, table=table, gene_id=gene)
result = db.execute(cmd, get_all=False)
if not result:
return (gene, '')
results.append(list(result))
return_select_columns = ['GENE_ID', 'CHR', 'POS_START', 'POS_END'
] + map_groupA + map_groupB
return return_select_columns, results
def test_cluster_worker_ensure_cluster_stopped(self):
with Shim(sut) as manager:
class FakeCmd:
STOP = object()
@assertCalled(self, 1)
def __init__(this, action, project, files):
pass
@assertCalled(self, 1)
def run(self):
pass
manager.ComposeCmd = FakeCmd
cluster = DB.Cluster(self.user, self.vpn)
cluster.delete()
worker = manager.ClusterWorker('foo', 'bar')
worker.ensure_cluster_stopped(self.user, self.vpn, cluster)
cluster.status = 'up'
worker.ensure_cluster_stopped(self.user, self.vpn, cluster)
self.assertEqual(cluster.status, 'stopped')
worker.ensure_cluster_stopped(self.user, self.vpn, cluster)
self.assertEqual(cluster.status, 'stopped')
async def ws_handler(self, websocket, path):
self.socket = websocket
try:
session = parse_cookie(
websocket.request_headers['cookie'])['sessionid']
self.id = int(
session_decoder.decode(
DB().get_session_data(session),
constants.DJANGO_SECRET_KEY)['_auth_user_id'])
except BaseException as e:
print('auth failed {}'.format(e))
self.id = -1
if self.id != -1:
print('Successfully authorized {}'.format(self.id))
client = ClientSession(self.id, websocket)
self.distributor.process_ws_connection(client)
try:
while True:
pack = await websocket.recv()
print('WS {} received {}'.format(self.id, pack))
self.distributor.process_ws_packet(pack, client)
except websockets.ConnectionClosed as exc:
self.distributor.process_ws_disconnection(client)
print("exc", exc)
print("User {} disconnected".format(self.id))
await websocket.close()
def test_vlan_worker_bridge_cluster(self):
with Shim(sut) as manager:
class FakeCmd:
ADDIF = object()
@assertCalled(self, 1)
def __init__(this, action, bridge_id, vlan_if):
pass
@assertCalled(self, 1)
def run(self):
pass
manager.BrctlCmd = FakeCmd
manager.get_bridge_id = lambda cluster_id: "bogus" # The real function requires docker
cluster = DB.Cluster(self.user, self.vpn)
self.vpn.links[self.user.vlan] = 'up'
worker = manager.VlanWorker('foo', 'bar')
worker.bridge_cluster(self.vpn, self.user)
self.assertNotEqual(self.vpn.links[self.user.vlan], 'bridged')
cluster.status = "up"
worker.bridge_cluster(self.vpn, self.user)
self.assertEqual(self.vpn.links[self.user.vlan], 'bridged')
def __init__(self):
super().__init__(AppConfig(), DConfigSettings(), DValueSettings())
self.db: DB = DB(self.database)
# services
self.data_service = DataService(self.base_params)
# scheduler
self.scheduler.add_job(self.data_service.generate_data, 6000)
async def register_product(name, password):
if not Product.pattern_of_name_of_product.match(name):
return {
"failure": True,
"message": "Wrong name"
}
elif not Product.pattern_of_passwords.match(password):
return {
"failure": True,
"message": "Wrong password"
}
date_base = DB.DataBase("localhost", 27017)
products = await date_base.get_products_collection()
if await products.count_documents({"name": name}) > 0:
return {
"failure": True,
"message": "No old_secret and secret params"
}
result = await products.insert_one({"name": name, "password": bcrypt.hashpw(password.encode('utf-8'), bcrypt.gensalt()).decode('utf-8')})
id = result.inserted_id.binary.hex()
return {"result": id}
def eod_symbols(e='US'):
df = get_exchange_symbols(exchange_code=e)
for i in range(len(df)):
try:
symbol = df.ix[i].name
name = df.ix[i]['Name']
exchange = df.ix[i]['Exchange']
if exchange != 'NaN':
e = ge(exchange)
else:
e = ge('')
Market.create(symbol=symbol, name=name, exchange=e)
print(colored.green(symbol))
except IntegrityError:
DB.rollback()
except Exception as err:
print(colored.red(err))
DB.rollback()
def insert_vote(cls, candidate_id):
session = DB.get_session()
new_vote = Vote(reference=candidate_id, up_date=datetime.now())
session.add(new_vote)
session.flush()
session.refresh(new_vote)
id_to_return = new_vote.id
session.commit()
return id_to_return
class InputFile(DB.Model):
"""
Class that represents an input file to the job
"""
id = DB.Column(DB.Integer, primary_key=True)
input_key = DB.Column(DB.String(length=120))
internal_path = DB.Column(DB.Text, nullable=False)
public_url = DB.Column(DB.Text)
job_id = DB.Column(DB.String(length=120),
DB.ForeignKey('delayed_job.id'),
nullable=False)
def get_locus_result(genename, blast_results):
cds_seq_dict = blast_results.get('cds_seq', 'NA')
pro_seq_dict = blast_results.get('pro_seq', 'NA')
db = DB()
locus_result = {}
cmd = """select l.*, f.Description, f.Pfam_Description,
f.Interpro_Description, f.GO_Description from locus l left join func f
on l.GENE_ID=f.GENE_ID where l.GENE_ID='{0}';
""".format(genename)
result = db.execute(cmd, get_all=False)
if result:
locus_result['orthologous_gene'] = {}
ortho_header = ['Arabidopsis_thaliana', 'Hordeum_vulgare', 'Oryza_sativa', 'Triticum_aestivum', 'Zea_mays']
locus_result['orthologous_gene']['header'] = ortho_header
locus_result['orthologous_gene']['body'] = []
cmd = "select l.GENE_ID, o.* from locus l left join ortho o on l.GENE_ID=o.GENE_ID where l.GENE_ID='{0}';".format(genename)
ortho_result = db.execute(cmd, get_all=False)
if ortho_result:
ortho_result_list = ortho_result[3:]
ortho_result_list = [printPretty(each) for each in ortho_result_list if each is not None]
locus_result['orthologous_gene']['body'] = ortho_result_list
gene_id, chr, pos_start, pos_end = result[1:5]
description, pfam_desc, interpro_desc, go_desc = result[5:]
locus_result['gene_identification'] = {'Gene Product Name': description,
'Locus Name': genename}
locus_result['gene_attributes'] = {'Chromosome': chr,
"Gene Postion":'{start} - {end}'.format(start=pos_start, end=pos_end)}
header = ['Description', 'Pfam_Description', 'Interpro_Description', 'GO_Description']
locus_result['gene_annotation'] = {}
locus_result['gene_annotation']['header'] = header
locus_result['gene_annotation']['body'] = [description, pfam_desc, interpro_desc, go_desc]
# match 01G and 02G TraesCS1A02G000100
#result = db.execute("select * from tissue_expression where Gene_id='{0}'".format(genename))
result = db.execute("select * from tissue_expression where Gene_id like '{0}'".format(wildcard_gene(genename)))
if result:
row = [float(each) for each in result[0][2:]]
else:
row = []
locus_result['tissue_expression'] = row
locus_result['gene_cds_seq'] = cds_seq_dict
locus_result['gene_pro_seq'] = pro_seq_dict
return locus_result
