def __init__(self):
self.__headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.90 Safari/537.36'
}
self.__urlString = Config.get_setting('GLOBAL', 'QRESP_SERVER_URL')
self.__schemaString = Config.get_setting('GLOBAL', 'SCHEMA_URL')
self.__httpUrlString = Config.get_setting('GLOBAL', 'HTTP_SERVER_URL')
def _get_wanted_info():
db_service = DBService(host=Config.get('db.url'),
user=Config.get('db.username'),
password=Config.get('db.password'),
db=Config.get('db.name'),
charset='utf8')
return db_service.select_one(QueryMapper.select_recruitment_site_query,
'Wanted')
def process():
config = Config()
server = JIRA(server=config.url, basic_auth=(config.login, config.password))
tab = tt.Texttable()
tab.header(['name','devd','time','fin','closed','ratio','sko','',''])
members = json.loads(config.team_members())
for member in members:
user = server.search_users(member)[0]
user_stat = get_full_stat(server, user)
tab.add_row(user_stat.values())
print(tab.draw())
def publish():
"""
Published the created metadata
"""
form = PublishForm(request.form)
serverslist = Servers()
try:
form.server.choices = [(qrespserver['qresp_server_url'],
qrespserver['qresp_server_url'])
for qrespserver in serverslist.getServersList()]
except Exception as e:
print(e)
raise InvalidUsage('Could not fetch serverlist, \n ' + str(e),
status_code=410)
if request.method == 'POST' and form.validate():
fileServerPath = session.get(CURATOR_FIELD.PROJECT,
{}).get("fileServerPath", "")
serverpathList = fileServerPath.rsplit("/", 2)
previewFolder = serverpathList[len(serverpathList) -
2] + "_" + serverpathList[
len(serverpathList) - 1]
error = []
try:
with open("papers/" + previewFolder + "/data.json",
"r") as jsonData:
error = serverslist.validateSchema(json.load(jsonData))
except:
error.append(
"No server found. Place files in a publicly accessible server."
)
if len(error) > 0:
return jsonify(error=error), 200
else:
session['publishserver'] = form.server.data
session['emailAddress'] = form.emailId.data
for item in serverslist.getServersList():
if form.server.data in item['qresp_server_url']:
session['maintaineraddresses'] = item[
'qresp_maintainer_emails']
googleauth = GoogleAuth(
Config.get_setting(app.config['env'], 'GOOGLE_CLIENT_ID'),
Config.get_setting(app.config['env'], 'REDIRECT_URI'),
Config.get_setting('GOOGLE_API', 'SCOPE'))
google = googleauth.getGoogleAuth()
auth_url, state = google.authorization_url(Config.get_setting(
'GOOGLE_API', 'AUTH_URI'),
access_type='offline')
session['oauth_state'] = state
return jsonify(data=auth_url), 200
return jsonify(error=form.errors), 400
def auth_token(
config: Config,
logger: Logger,
) -> Generator[Dict[str, str], None, None]:
retailer_repository = RetailerRepository(config, logger)
auth_service = AuthService(config, logger, retailer_repository)
reatailer_model = RetailerModel(
full_name="Authenticated User",
document="99899899899",
email=EmailStr("*****@*****.**"),
password="******",
)
retailer_repository.insert_retailer(reatailer_model)
retailer = retailer_repository.get_retailer_by_email("*****@*****.**")
auth_token = auth_service.generate_access_token(
{"sub": "*****@*****.**"},
expires_delta=timedelta(
minutes=int(config.get_config("ACCESS_TOKEN_EXPIRE_MINUTES"))
),
)
yield {"Authorization": f"Bearer {auth_token}"}
if retailer:
retailer_repository.delete_retailer(retailer.id)
def search():
"""
This method helps in filtering paper content
"""
try:
selected_servers = urllib.parse.unquote(
request.args.get('servers', type=str, default=''))
fetchdata = FetchDataFromAPI(
selected_servers,
str(request.host_url).strip("/") if Config.get_setting(
'DEV', 'MONGODB_HOST') else None)
allpaperslist = fetchdata.fetchOutput('/api/search')
except Exception as e:
app.logger.error(e)
return jsonify({"error": e}), 500
try:
collectionlist = fetchdata.fetchOutput('/api/collections')
authorslist = fetchdata.fetchOutput('/api/authors')
publicationlist = fetchdata.fetchOutput('/api/publications')
allPapersSize = len(allpaperslist)
except Exception as e:
app.logger.error(e)
return jsonify({"allpaperslist": allpaperslist, "error": str(e)}), 500
# return render_template('search.html', allpaperslist=allpaperslist)
return jsonify({
"allpaperslist": allpaperslist,
"collectionlist": collectionlist,
"authorslist": authorslist,
"publicationlist": publicationlist,
"allPapersSize": allPapersSize,
"error": None
}), 200
def searchWord():
"""
Filtering paper content
"""
allpaperslist = []
try:
searchWord = ""
paperTitle = request.args.get('paperTitle', type=str)
doi = request.args.get('doi', type=str)
tags = json.loads(request.args.get('tags'))
collectionList = json.loads(request.args.get('collectionList'))
authorsList = json.loads(request.args.get('authorsList'))
publicationList = json.loads(request.args.get('publicationList'))
selected_servers = urllib.parse.unquote(
request.args.get('servers', type=str, default=''))
fetchdata = FetchDataFromAPI(
selected_servers,
str(request.host_url).strip("/") if Config.get_setting(
app.config['env'], 'MONGODB_HOST') else None)
url = '/api/search'+"?searchWord="+searchWord+"&paperTitle="+paperTitle+"&doi="+doi+"&tags="+",".join(tags)+"&collectionList="+",".join(collectionList) + \
"&authorsList="+",".join(authorsList) + \
"&publicationList="+",".join(publicationList)
allpaperslist = fetchdata.fetchOutput(url)
return jsonify(allpaperslist=allpaperslist), 200
except Exception as e:
print(e, file=sys.stderr)
return jsonify(allpaperslist=allpaperslist), 400
def main():
args = get_args()
cfg = Config()
keys = [
["vectorizer", "batch_size"],
["vectorizer", "samples"],
["vectorizer", "vector_size"],
["vectorizer", "epochs"],
]
cfg.setup(keys)
v = Vectorize(
indir=args.indir,
batch_size=cfg.batch_size,
n_samples=cfg.samples,
epochs=cfg.epochs,
vector_size=cfg.vector_size,
)
v.run()
def verifypasscode():
"""
This method verifies with input passcode of flask connection.
"""
form = PassCodeForm(request.form)
confirmpasscode = Config.get_setting(app.config['env'],
'QRESP_DB_SECRET_KEY')
if request.method == 'POST' and form.validate():
if confirmpasscode == form.passcode.data:
return jsonify(msg="success"), 200
return jsonify(msg="failed"), 400
def get_config():
cfg = Config()
keys = [
["seq2vec", "gpu"],
["seq2vec", "load"],
["seq2vec", "epochs"],
["seq2vec", "lr"],
["seq2vec", "weight_decay"],
["seq2vec", "hidden_size"],
["seq2vec", "n_layers"],
["seq2vec", "bidirectional"],
["seq2vec", "batch_size"],
["seq2vec", "max_seqlen"],
["seq2vec", "train_samples"],
["seq2vec", "model_dir"],
["seq2vec", "predict_intervals"],
["seq2vec", "predict_samples"],
["visdom", "server"],
["visdom", "port"],
]
cfg.setup(keys)
return cfg
def store_to_db(recruit_notices, site_seq):
db_service = DBService(host=Config.get('db.url'),
user=Config.get('db.username'),
password=Config.get('db.password'),
db=Config.get('db.name'),
charset='utf8')
cursor = db_service.get_cursor()
for recruit_notice in recruit_notices:
cursor.execute(QueryMapper.insert_raw_collection_query,
(site_seq, recruit_notice['company'],
recruit_notice['condition_type'].value))
raw_collection_id = db_service.get_last_id()
keywords = recruit_notice['keywords']
for keyword in keywords:
cursor.execute(QueryMapper.insert_raw_word_query,
(keyword, raw_collection_id))
db_service.commit_and_close()
def paperdetails(paperid):
"""
Fetching papers details content
"""
paperdetail = []
workflowdetail = []
try:
selected_servers = urllib.parse.unquote(
request.args.get('servers', type=str, default=''))
fetchdata = FetchDataFromAPI(
selected_servers,
str(request.host_url).strip("/") if Config.get_setting(
app.config['env'], 'MONGODB_HOST') else None)
url = '/api/paper/' + paperid
paperdetail = fetchdata.fetchOutput(url)
url = '/api/workflow/' + paperid
workflowdetail = fetchdata.fetchOutput(url)
return jsonify(paperdetail=paperdetail,
workflowdetail=workflowdetail), 200
except Exception as e:
print(e, file=sys.stderr)
return jsonify({'error': True}), 500
def chartworkflow():
"""
Fetching chart workflow content
"""
chartworkflowdetail = []
try:
paperid = request.args.get('paperid', 0, type=str)
paperid = str(paperid).strip()
chartid = request.args.get('chartid', 0, type=str)
chartid = str(chartid).strip()
selected_servers = urllib.parse.unquote(
request.args.get('servers', type=str, default=''))
fetchdata = FetchDataFromAPI(
selected_servers,
str(request.host_url).strip("/") if Config.get_setting(
app.config['env'], 'MONGODB_HOST') else None)
url = '/api/paper/' + paperid + '/chart/' + chartid
chartworkflowdetail = fetchdata.fetchOutput(url)
return jsonify(chartworkflowdetail=chartworkflowdetail), 200
except Exception as e:
print(e)
flash('Error in chartdetails. ' + str(e))
return jsonify(chartworkflowdetail=chartworkflowdetail), 400
def process_for_user(name='g.frolov'):
config = Config()
server = JIRA(server=config.url, basic_auth=(config.login, config.password))
tab = tt.Texttable()
tab.header(
['name', 'devd', 'time', 'fin', 'closed', 'ratio', 'sko', '', ''])
member = name
user = server.search_users(member)[0]
periods = 8
period_len_days = 30
for period in xrange(periods):
user_stat = get_full_stat(server, user, during=period_len_days, since_days_back = period * period_len_days)
tab.add_row(user_stat.values())
print(tab.draw())
def setUp(self):
self.app = self.create_app()
self.client = app.test_client()
self.ctx = self.app.test_request_context()
self.ctx.push()
Config.initialize('project/config_mock.ini')
self.pagenames = [
'/', 'qrespcurator', 'startfromscratch', 'getTreeInfo',
'addToWorkflow', 'mint', 'preview/files_paper',
'previewchartworkflow?paperid=files_paper&chartid=c0',
'qrespexplorer',
'search?servers=https%3A%2F%2Fpaperstack.uchicago.edu',
'searchWord?searchWord=&paperTitle=&tags=&doi=&collectionList=[]&authorsList=[]&publicationList=[]',
'paperdetails/5941869f1bd40fd44db0024a',
'chartworkflow?paperid=5941869f1bd40fd44db0024a&chartid=c0',
'insertDOI?paperId=5941869f1bd40fd44db0024a&doi=25678',
'oauth2callback'
]
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
with open(os.path.join(__location__, 'data.json')) as f:
paperdata = json.load(f)
paperdata = ConvertField.convertToString(
["files", "properties", "URLs", "patches"], [
CURATOR_FIELD.CHARTS, CURATOR_FIELD.TOOLS,
CURATOR_FIELD.DATASETS, CURATOR_FIELD.SCRIPTS,
CURATOR_FIELD.HEADS
], paperdata)
paperform = PaperForm(**paperdata)
detailsform = DetailsForm(**paperform.info.insertedBy.data)
serverform = ServerForm(
**{'hostUrl': 'https://notebook.rcc.uchicago.edu/files'})
chartform = {
'id': 'c0',
'caption': 'chart 1',
'number': '1',
'files':
'charts/figure1/figure1.csv, charts/figure1/figure1.ipynb, charts/figure1/figure1.jpg',
'imageFile': 'charts/figure1/figure1.jpg',
'notebookFile': 'charts/figure1/figure1.ipynb',
'properties': 'potential energy, band gap'
}
toolform = {
'id': 't0',
'kind': 'software',
'packageName': 'West',
'URLs': '',
'version': '3.0.0',
'programName': 'wstat.x',
'patches': 'tools/modded_qbox.diff',
'description': 'Modified west code',
'facilityName': '',
'measurement': ''
}
datasetform = {
'id': 'd0',
'files': 'datasets/datasetA.dat, datasets/datasetB.dat',
'readme': 'DAT files',
'URLs': ''
}
scriptform = {
'id': 'd0',
'files': 'datasets/datasetA.dat, datasets/datasetB.dat',
'readme': 'DAT files',
'URLs': ''
}
referenceform = {
'kind': 'journal',
'DOI': '10.1021/jacs.6b00225',
'title':
'Photoelectron Spectra of Aqueous Solutions from First Principles',
'page': '6912-6915',
'publishedAbstract':
'We present a combined computational and experimental study of the photoelectron spectrum of a simple aqueous solution of NaCl. Measurements were conducted on microjets, and first-principles calculations were performed using hybrid functionals and many-body perturbation theory at the G0W0 level, starting with wave functions computed in ab initio molecular dynamics simulations. We show excellent agreement between theory and experiments for the positions of both the solute and solvent excitation energies on an absolute energy scale and for peak intensities. The best comparison was obtained using wave functions obtained with dielectric-dependent self-consistent and range-separated hybrid functionals. Our computational protocol opens the way to accurate, predictive calculations of the electronic properties of electrolytes, of interest to a variety of energy problems.',
'volume': '138',
'year': 2016,
'URLs': 'http://dx.doi.org/10.1021/jacs.6b00225',
'school': ''
}
documentationform = DocumentationForm(**paperform.documentation.data)
adminform = {
'hostname': 'mongomock://localhost',
'port': 27017,
'username': None,
'password': None,
'dbname': 'mongoenginetest',
'collection': 'paper'
}
self.postPageForms = {
'uploadFile': paperform,
'details': detailsform,
'server': serverform,
'charts': chartform,
'charts/delete': chartform,
'tools': toolform,
'tools/delete': toolform,
'datasets': datasetform,
'datasets/delete': datasetform,
'scripts': scriptform,
'scripts/delete': scriptform,
'reference': referenceform,
'fetchReferenceDOI': {
'doi': '10.1021/nl903868w'
},
'documentation': documentationform,
'addToWorkflow': {},
'saveNodesAndEdges': [['t0', 'd0'], [{
'id': 't0'
}, {
'id': 'd0'
}]],
'publish': {},
'download': {},
'verifyPasscode': {
'passcode': '123456'
},
'admin': adminform,
'getDescriptor': {
"metadata": paperform.data
}
}
from datetime import timedelta
from fastapi import APIRouter, HTTPException
from starlette import status
from project.config import Config
from project.dal.retailer import RetailerRepository
from project.dtos.auth import Token, AuthData
from project.logger import Logger
from project.services.auth.auth_service import AuthService
config = Config()
logger = Logger()
router = APIRouter()
retailer_repository = RetailerRepository(config, logger)
auth_service = AuthService(config, logger, retailer_repository)
@router.post("/auth/token/", response_model=Token)
def login_for_access_token(auth_data: AuthData) -> Token:
retailer = auth_service.authenticate_retailer(email=auth_data.email,
password=auth_data.password)
if not retailer:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Incorrect username or password",
headers={"WWW-Authenticate": "Bearer"},
)
def environment( # type: ignore
self, mock_env_var, config: Config) -> Environment:
return config.get_environment()
def test_get_environment( # type: ignore
self, mock_env_var, config: Config):
with pytest.raises(InvalidEnvironmentError):
config.get_environment()
def expected_config_value(self, config: Config, config_key: str) -> Any:
return config.get_config(config_key)
def expected_config_or_default_value(self, config: Config, config_key: str,
default_value: Any) -> Any:
return config.get_config_or_default(config_key, default_value)
def config(self):
return Config("tests/config/project.cfg", "tests/config/project.spc")
def config(self) -> Config:
return Config()
def _load_invalid():
cfg = Config("tests/config/invalid.cfg",
"tests/config/project.spc")
def config() -> Config:
return Config()
def authorized():
"""
Callback for authorization
"""
form = PublishForm()
serverslist = Servers()
form.server.choices = [(qrespserver['qresp_server_url'],
qrespserver['qresp_server_url'])
for qrespserver in serverslist.getServersList()]
if 'error' in request.args:
if request.args.get('error') == 'access_denied':
print('denied access.')
return redirect(url_for('qrespcurator'))
if 'code' not in request.args and 'state' not in request.args:
print('denied access.')
return redirect(url_for('qrespcurator'))
googleauth = GoogleAuth(
Config.get_setting(app.config['env'], 'GOOGLE_CLIENT_ID'),
Config.get_setting(app.config['env'], 'REDIRECT_URI'))
google = googleauth.getGoogleAuth(state=session.get('oauth_state'))
try:
token = google.fetch_token(Config.get_setting('GOOGLE_API',
'TOKEN_URI'),
client_secret=Config.get_setting(
app.config['env'],
'GOOGLE_CLIENT_SECRET'),
authorization_response=request.url)
except Exception as e:
print(e)
return 'HTTP Error occurred.'
try:
google = googleauth.getGoogleAuth(token=token)
resp = google.get(Config.get_setting('GOOGLE_API', 'USER_INFO'))
if resp.status_code == 200:
user_data = resp.json()
emailAddress = session.get('emailAddress')
server = session.get("publishserver")
if user_data['email'] in emailAddress:
try:
fileServerPath = session.get(CURATOR_FIELD.PROJECT,
{}).get("fileServerPath", "")
serverpathList = fileServerPath.rsplit("/", 2)
previewFolder = serverpathList[len(serverpathList) -
2] + "_" + serverpathList[
len(serverpathList) - 1]
with open("papers/" + previewFolder + "/data.json",
"r") as jsonData:
senddescriptor = SendDescriptor(server)
jsondata = json.load(jsonData)
response = senddescriptor.sendDescriptorToServer(
jsondata)
if response.status_code == 400:
flash("Paper already exists")
return redirect(url_for('qrespcurator'))
else:
try:
paperdata = response.json()
maintaineraddresses = session.get(
'maintaineraddresses')
body = 'The user ' + str(jsondata['info']['insertedBy']['firstName']) + ' with email address ' + \
emailAddress + ' has inserted paper with paper id ' + \
str(paperdata["paperid"])
fromx = Config.get_setting(
'GLOBAL', 'MAIL_ADDR')
to = maintaineraddresses
msg = MIMEMultipart()
msg['Subject'] = 'New paper inserted into Qresp ecosystem'
msg['From'] = fromx
msg['To'] = ", ".join(to)
msg.attach(MIMEText(body, 'plain'))
mailserver = smtplib.SMTP(
'smtp.gmail.com', 587)
mailserver.starttls()
mailserver.login(
Config.get_setting('GLOBAL', 'MAIL_ADDR'),
Config.get_setting('SECRETS', 'MAIL_PWD'))
mailserver.sendmail(fromx, to, msg.as_string())
mailserver.close()
return redirect(server + "/paperdetails/" +
paperdata["paperid"])
except Exception as e:
print(traceback.format_exc())
flash('Could not email your administrator' +
str(e))
flash('Published')
return redirect(url_for('qrespcurator'))
except Exception as e:
print(e)
flash('No paper found')
return redirect(url_for('qrespcurator'))
else:
flash(
'Unauthorized access. Could not verify your email address')
return redirect(url_for('qrespcurator'))
except Exception as e:
print(e)
flash('Unauthorized access. Could not verify your email address')
return redirect(url_for('qrespcurator'))
flash('Published')
return redirect(url_for('qrespcurator'))
from project.config import Config
from project.functionality import Example
# Random function to demonstrate we can pass _anything_ to 'make_config' inside 'Config'.
def uppercase(words):
return words.upper()
# We create our custom configuration without saving it.
Config(arg="hello world", func=uppercase)
# We initialize our Example object without passing the 'Config' object to it.
example = Example()
print(example.arg)
# >>> "HELLO WORLD"
def expires_delta(self, config: Config) -> timedelta:
return timedelta(minutes=int(config.get_config("ACCESS_TOKEN_EXPIRE_MINUTES")))
import connexion
import os
from flask_session import Session
from flask_sitemap import Sitemap
from project.config import Config
from flask_mongoengine import MongoEngine
from flask_cors import CORS
Config.initialize()
# Create the application instance
connexionapp = connexion.FlaskApp(__name__)
# Read the swagger.yml file to configure the endpoints
swagger_file = (os.path.join(os.getcwd(), 'project/swagger.yml'))
connexionapp.add_api(swagger_file)
app = connexionapp.app
# Create protection and session variables
app.secret_key = Config.get_setting('SECRETS', 'FLASK_SECRET_KEY')
SESSION_TYPE = 'filesystem'
app.config.from_object(__name__)
app.config['env'] = 'DEV'
os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
Session(app)
ext = Sitemap(app)
CORS(app)
#initialize db
if Config.get_setting(app.config['env'], 'MONGODB_HOST'):
db = MongoEngine()
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
from project.config import Config
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
app_config = Config()
host = app_config.get_config("DB_HOST")
port = app_config.get_config("DB_PORT")
user = app_config.get_config("DB_USER")
password = app_config.get_config("DB_PASSWORD")
database = app_config.get_config("DB_DATABASE")
database_url = f"mysql+mysqldb://{user}:{password}@{host}:{port}/{database}"
config.set_main_option("sqlalchemy.url", database_url)
# Interpret the config file for Python logging.
# This line sets up loggers basically.
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata