def authenticated_api(host_uri, token=None):
"""
Note: if this helper is called, it's implied that an actual API connection
is needed, so it does try to connect and verify credentials.
"""
conf = fatcat_openapi_client.Configuration()
conf.host = host_uri
if not token:
token = os.environ['FATCAT_API_AUTH_TOKEN']
if not token:
sys.stderr.write(
'This client requires a fatcat API token (eg, in env var FATCAT_API_AUTH_TOKEN)\n'
)
sys.exit(-1)
conf.api_key["Authorization"] = token
conf.api_key_prefix["Authorization"] = "Bearer"
api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(conf))
# verify up front that auth is working
api.auth_check()
return api
def auth_api(token):
conf = fatcat_openapi_client.Configuration()
conf.api_key["Authorization"] = token
conf.api_key_prefix["Authorization"] = "Bearer"
conf.host = Config.FATCAT_API_HOST
return fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(conf))
def public_api(host_uri):
"""
Note: unlike the authenticated variant, this helper might get called even
if the API isn't going to be used, so it's important that it doesn't try to
actually connect to the API host or something.
"""
conf = fatcat_openapi_client.Configuration()
conf.host = host_uri
return fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(conf))
def process_query(query: FulltextQuery) -> FulltextHits:
if not query.q:
return do_fulltext_search(query)
# try handling raw identifier queries
if len(query.q.strip().split()) == 1 and not '"' in query.q:
doi = clean_doi(query.q)
if doi:
return do_lookup_query(f'doi:"{doi}"')
pmcid = clean_pmcid(query.q)
if pmcid:
return do_lookup_query(f'pmcid:"{pmcid}"')
if query.q.strip().startswith("key:"):
return do_lookup_query(query.q)
# if this is a citation string, do a fuzzy lookup
if settings.ENABLE_CITATION_QUERY and sniff_citation_query(query.q):
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = settings.FATCAT_API_HOST
api_client = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(api_conf))
fatcat_es_client = elasticsearch.Elasticsearch(
"https://search.fatcat.wiki")
key: Optional[str] = None
try:
key = try_fuzzy_match(
query.q,
grobid_host=settings.GROBID_HOST,
es_client=fatcat_es_client,
fatcat_api_client=api_client,
)
except elasticsearch.exceptions.RequestError as e:
logging.warn(f"citation fuzzy failure: {e}")
pass
except Exception as e:
# TODO: sentry log?
logging.warn(f"citation fuzzy failure: {e}")
raise e
if key:
result = do_lookup_query(f"key:{key}")
if result:
result.query_type = "citation"
return result
# fall through to regular query, with pre-parsing
query = copy.copy(query)
if query.q:
query.q = pre_parse_query(query.q)
return do_fulltext_search(query)
def test_issue_db_basics() -> None:
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = settings.FATCAT_API_HOST
api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(api_conf))
es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_FATCAT_BASE)
issue_db = IssueDB(settings.SCHOLAR_ISSUEDB_PATH)
issue_db.init_db()
with open("tests/files/sim_collections.json", "r") as f:
issue_db.load_pubs(f.readlines(), api)
with open("tests/files/sim_items.json", "r") as f:
issue_db.load_issues(f.readlines(), es_client)
def __init__(
self,
work_pipeline: WorkPipeline,
sim_pipeline: SimPipeline,
produce_docs_topic: str,
**kwargs: Any,
):
super().__init__(**kwargs)
self.sim_pipeline = sim_pipeline
self.work_pipeline = work_pipeline
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = kwargs.get("fatcat_api_host", "https://api.fatcat.wiki/v0")
self.fatcat_api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(api_conf)
)
self.producer = self.create_kafka_producer(self.kafka_brokers)
self.produce_topic = produce_docs_topic
print(f"Will produce bundles to: {self.produce_topic}", file=sys.stderr)
print("| _| (_| | || (_| (_| | |_|_|")
print("|_| \__,_|\__\___\__,_|\__(_)")
print()
admin_id = "aaaaaaaaaaaabkvkaaaaaaaaae"
#fatcat_openapi_client.configuration.api_key['Authorization'] = 'YOUR_API_KEY'
#fatcat_openapi_client.configuration.api_key_prefix['Authorization'] = 'Bearer'
local_conf = fatcat_openapi_client.Configuration()
local_conf.api_key[
"Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAhYyMDE5MDEwMS1kZXYtZHVtbXkta2V5AAImZWRpdG9yX2lkID0gYWFhYWFhYWFhYWFhYmt2a2FhYWFhYWFhYWkAAht0aW1lID4gMjAxOS0wMS0wOVQwMDo1Nzo1MloAAAYgnroNha1hSftChtxHGTnLEmM/pY8MeQS/jBSV0UNvXug="
local_conf.api_key_prefix["Authorization"] = "Bearer"
local_conf.host = 'http://localhost:9411/v0'
local_conf.debug = True
local_api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(local_conf))
#prod_conf = fatcat_openapi_client.Configuration()
#prod_conf.api_key["Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw"
#prod_conf.api_key_prefix["Authorization"] = "Bearer"
#prod_conf.host = 'https://api.fatcat.wiki/v0'
#prod_api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(prod_conf))
qa_conf = fatcat_openapi_client.Configuration()
qa_conf.api_key[
"Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw"
qa_conf.api_key_prefix["Authorization"] = "Bearer"
qa_conf.host = 'https://api.qa.fatcat.wiki/v0'
qa_api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(qa_conf))
def main() -> None:
"""
Run this command like:
python -m fatcat_scholar.issue_db
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers()
parser.add_argument(
"--db-file",
help="sqlite3 database file to open",
default="data/issue_db.sqlite",
type=str,
)
sub = subparsers.add_parser("init_db",
help="create sqlite3 output file and tables")
sub.set_defaults(func="init_db")
sub = subparsers.add_parser(
"load_pubs", help="update container-level stats from JSON file")
sub.set_defaults(func="load_pubs")
sub.add_argument(
"json_file",
help="collection-level metadata, as JSON-lines",
nargs="?",
default=sys.stdin,
type=argparse.FileType("r"),
)
sub = subparsers.add_parser("load_issues",
help="update item-level stats from JSON file")
sub.set_defaults(func="load_issues")
sub.add_argument(
"json_file",
help="item-level metadata, as JSON-lines",
nargs="?",
default=sys.stdin,
type=argparse.FileType("r"),
)
sub = subparsers.add_parser(
"load_counts",
help="update volume-level stats from elasticsearch endpoint")
sub.set_defaults(func="load_counts")
args = parser.parse_args()
if not args.__dict__.get("func"):
print("tell me what to do! (try --help)")
sys.exit(-1)
idb = IssueDB(args.db_file)
api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient())
es_client = elasticsearch.Elasticsearch("https://search.fatcat.wiki")
if args.func == "load_pubs":
idb.load_pubs(args.json_file, api)
elif args.func == "load_issues":
idb.load_issues(args.json_file, es_client)
elif args.func == "load_counts":
idb.load_counts(es_client)
else:
func = getattr(idb, args.func)
func()
no_intra_emphasis=True,
strikethrough=True,
escape=True,
)
login_manager = LoginManager()
login_manager.init_app(app)
login_manager.login_view = "/auth/login"
oauth = OAuth(app)
# Grabs sentry config from SENTRY_DSN environment variable
sentry = Sentry(app)
conf = fatcat_openapi_client.Configuration()
conf.host = Config.FATCAT_API_HOST
api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(conf))
# remove most jinja2 template whitespace
app.jinja_env.trim_blocks = True
app.jinja_env.lstrip_blocks = True
def auth_api(token):
conf = fatcat_openapi_client.Configuration()
conf.api_key["Authorization"] = token
conf.api_key_prefix["Authorization"] = "Bearer"
conf.host = Config.FATCAT_API_HOST
return fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(conf))
if Config.FATCAT_API_AUTH_TOKEN:
print("Found and using privileged token (eg, for account signup)")
priv_api = auth_api(Config.FATCAT_API_AUTH_TOKEN)
def main() -> None:
"""
Run this command like:
python -m fatcat_scholar.issue_db
"""
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
subparsers = parser.add_subparsers()
parser.add_argument(
"--db-file",
help="sqlite3 database file to open",
default=settings.SCHOLAR_ISSUEDB_PATH,
type=str,
)
sub = subparsers.add_parser("init_db",
help="create sqlite3 output file and tables")
sub.set_defaults(func="init_db")
sub = subparsers.add_parser(
"load_pubs", help="update container-level stats from JSON file")
sub.set_defaults(func="load_pubs")
sub.add_argument(
"json_file",
help="collection-level metadata, as JSON-lines",
nargs="?",
default=sys.stdin,
type=argparse.FileType("r"),
)
sub = subparsers.add_parser("load_issues",
help="update item-level stats from JSON file")
sub.set_defaults(func="load_issues")
sub.add_argument(
"json_file",
help="item-level metadata, as JSON-lines",
nargs="?",
default=sys.stdin,
type=argparse.FileType("r"),
)
sub = subparsers.add_parser(
"load_counts",
help="update volume-level stats from elasticsearch endpoint")
sub.set_defaults(func="load_counts")
args = parser.parse_args()
if not args.__dict__.get("func"):
parser.print_help(file=sys.stderr)
sys.exit(-1)
idb = IssueDB(args.db_file)
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = settings.FATCAT_API_HOST
api = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(api_conf))
es_client = elasticsearch.Elasticsearch(settings.ELASTICSEARCH_FATCAT_BASE)
if args.func == "load_pubs":
idb.load_pubs(args.json_file, api)
elif args.func == "load_issues":
idb.load_issues(args.json_file, es_client)
elif args.func == "load_counts":
idb.load_counts(es_client)
else:
func = getattr(idb, args.func)
func()
import elasticsearch
import fatcat_openapi_client
citation = sys.argv[1]
print("Sending to GROBID...")
resp = grobid_process_citation(citation)
print(resp)
if not resp:
sys.exit(0)
ref = parse_citation_xml(resp)
print(ref)
if not ref:
sys.exit(0)
release = ref_to_release(ref)
print(release)
es_backend = os.environ.get(
"ELASTICSEARCH_FATCAT_BASE", "https://search.fatcat.wiki"
)
es_client = elasticsearch.Elasticsearch(es_backend)
api_conf = fatcat_openapi_client.Configuration()
api_conf.host = os.environ.get("FATCAT_API_HOST", "https://api.fatcat.wiki/v0")
api_client = fatcat_openapi_client.DefaultApi(
fatcat_openapi_client.ApiClient(api_conf)
)
matches = fuzzy_match(release, es_client=es_client, api_client=api_client)
print(matches)
if not matches or matches[0] not in ("EXACT", "STRONG", "WEAK"):
sys.exit(0)
print(matches[2].work_id)
print("| _| (_| | || (_| (_| | |_|_|")
print("|_| \__,_|\__\___\__,_|\__(_)")
print()
admin_id = "aaaaaaaaaaaabkvkaaaaaaaaae"
# fatcat_openapi_client.configuration.api_key['Authorization'] = 'YOUR_API_KEY'
# fatcat_openapi_client.configuration.api_key_prefix['Authorization'] = 'Bearer'
local_conf = fatcat_openapi_client.Configuration()
local_conf.api_key[
"Authorization"
] = "AgEPZGV2LmZhdGNhdC53aWtpAhYyMDE5MDEwMS1kZXYtZHVtbXkta2V5AAImZWRpdG9yX2lkID0gYWFhYWFhYWFhYWFhYmt2a2FhYWFhYWFhYWkAAht0aW1lID4gMjAxOS0wMS0wOVQwMDo1Nzo1MloAAAYgnroNha1hSftChtxHGTnLEmM/pY8MeQS/jBSV0UNvXug="
local_conf.api_key_prefix["Authorization"] = "Bearer"
local_conf.host = "http://localhost:9411/v0"
local_conf.debug = True
local_api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(local_conf))
# prod_conf = fatcat_openapi_client.Configuration()
# prod_conf.api_key["Authorization"] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw"
# prod_conf.api_key_prefix["Authorization"] = "Bearer"
# prod_conf.host = 'https://api.fatcat.wiki/v0'
# prod_api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(prod_conf))
qa_conf = fatcat_openapi_client.Configuration()
qa_conf.api_key[
"Authorization"
] = "AgEPZGV2LmZhdGNhdC53aWtpAg4yMDE4LTEyLTMxLWRldgACJmVkaXRvcl9pZCA9IGFhYWFhYWFhYWFhYWJrdmthYWFhYWFhYWFlAAIeY3JlYXRlZCA9IDIwMTgtMTItMzFUMjE6MTU6NDdaAAAGIMWFZeZ54pH4OzNl5+U5X3p1H1rMioSuIldihuiM5XAw"
qa_conf.api_key_prefix["Authorization"] = "Bearer"
qa_conf.host = "https://api.qa.fatcat.wiki/v0"
qa_api = fatcat_openapi_client.DefaultApi(fatcat_openapi_client.ApiClient(qa_conf))
