def gb_parliament():
counter = 0
filename = 'gb_parliament.csv'
# setup logger
logging = logger.logger_config(with_file=False)
logging.info('\n\nProgram starts..')
# setup db connections and clean from previous data
appES = databases.ES()
appNeo = databases.Neo()
appNeo.cleanNeo4j()
# appES.cleanES()
with open(filename, 'r') as file:
logging.info("Reading the file {a}".format(a=filename))
for row in csv.DictReader(file):
person = Person(row['id'], row['name'], row['sort_name'],
row['email'], 'GB') # creating objects
organization = Organization(row['group_id'], row['group'])
membership = Membership(row['id'], row['group_id'])
appNeo.insertPersonNeo4j(person)
appNeo.insertOrganizationNeo4j(organization)
appNeo.insertMembershipNeo4j(membership)
appES.insertPersonES(person)
appES.insertOrganizationES(organization)
appES.insertMembershipES(membership)
counter += 1
# close connections
appES.close()
appNeo.close()
logging.info(f"Records read from a file: {counter}")
logger.info('Updating citations and references')
min_days_to_update = age_days * 86400
papers = list(db_papers.find())
logger.info(f'Fetching {len(papers)} documents')
for idx, p in enumerate(papers):
if idx % 500 == 0:
logger.info(f'Updating batch {idx}')
cur_sem_sch = sem_sch_papers.find_one({'_id': p['_id']})
if not cur_sem_sch or (datetime.utcnow() -
cur_sem_sch['last_rec_update']
).total_seconds() > min_days_to_update:
res = fetch_paper_data(p)
if res:
sem_sch_papers.update({'_id': res['_id']}, {'$set': res}, True)
for a in res['authors']:
sem_sch_authors.update({'_id': a['name']}, {}, True)
else:
logger.error('Failed to fetch paper data - {}'.format(
p['_id']))
sleep(5)
else:
logger.debug('Paper is already in DB')
logger.info('Finished updating refs')
if __name__ == '__main__':
from logger import logger_config
logger_config(info_filename='citations_fetcher.log')
update_all_papers()
import sys
import threading
import json
import os
import numpy as np
import tensorflow as tf
from flask import Flask, request, redirect, jsonify, render_template, url_for
from foodnonfood import read_tensor_from_image_file
from logger import logger_config
UPLOAD_DIR = 'static/uploaded_images' # Directory to upload image files
EXTENSIONS = ['.jpeg', '.jpg',
'.png'] # file extensions which can use for inference
LOGGER = logger_config('tfs_cli')
tf.app.flags.DEFINE_string('version', None, 'spacify model version')
tf.app.flags.DEFINE_boolean('debug', 'True', 'debug option')
tf.app.flags.DEFINE_string('hostport', 'localhost:8501',
'host:port of predict server')
FLAGS = tf.app.flags.FLAGS
app = Flask(__name__)
app.config['UPLOAD_DIR'] = UPLOAD_DIR
def _predict(hostport, image_file):
host, port = hostport.split(':')
with tf.Session() as sess:
image = read_tensor_from_image_file(sess, image_file)
import logging
import schedule
import time
from fetch_papers import fetch_papers_main
from twitter_daemon import main_twitter_fetcher
from fetch_citations_and_references import update_all_papers
from logger import logger_config
if __name__ == '__main__':
logger_config(info_filename='background_tasks.log')
logger = logging.getLogger(__name__)
logger.info('Start background tasks')
schedule.every(30).minutes.do(main_twitter_fetcher)
schedule.every(2).hours.do(fetch_papers_main)
schedule.every().saturday.at("00:10").do(update_all_papers)
while True:
schedule.run_pending()
time.sleep(1)
}).sort("time_published", pymongo.DESCENDING).limit(5))
papers = [{
'name': p['title'],
'type': 'paper',
'authors': p['authors']
} for p in papers]
return jsonify(authors + papers)
# -----------------------------------------------------------------------------
# int main
# -----------------------------------------------------------------------------
if __name__ == "__main__":
logger_config()
logger = logging.getLogger(__name__)
parser = argparse.ArgumentParser()
parser.add_argument('-p',
'--prod',
dest='prod',
action='store_true',
help='run in prod?')
parser.add_argument('-r',
'--num_results',
dest='num_results',
type=int,
default=200,
help='number of results to return per query')
parser.add_argument('--port',
@catch_exceptions(logger=logger)
def main_twitter_fetcher():
tweets = fetch_tweets()
papers_to_update = process_tweets(tweets)
summarize_tweets(papers_to_update)
@catch_exceptions(logger=logger)
def recalculate():
all_papers = list(db_papers.find({}, {'_id': 1}))
summarize_tweets([p['_id'] for p in all_papers])
# -----------------------------------------------------------------------------
# authenticate to twitter API
api = get_api_connector()
# connect to mongodb instance
client = pymongo.MongoClient()
mdb = client.arxiv
db_tweets = mdb.tweets # the "tweets" collection in "arxiv" database
db_papers = mdb.papers
# main loop
if __name__ == '__main__':
from logger import logger_config
logger_config(info_filename='twitter_daemon.log')
main_twitter_fetcher()
'extraction_conf': row.get('high_conf', '') == 'True',
'conferences': row.get('proceeding', ''),
'stars': int(row.get('stars', 0)),
'framework': row.get('framework'),
'datasets': row.get('datasets', '').split('|'),
'tasks': row.get('tasks', '').split('|'),
'paperswithcode_link': row.get('url', '')
}
try:
papers.update(cur_id, {'$set': {'code': obj}})
except Exception as e:
logger.error('Failed to update paper {} - {}'.format(
cur_id['_id'], e))
else:
logger.info('Paper not found - {}'.format(cur_id['_id']))
@catch_exceptions(logger=logger)
def fetch_code_data():
logger.info('Fetching data from papers with code')
data = fetch_data()
logger.info('Updating DB with data from papers with code')
update_db(data)
logger.info('Finished updating data from papers with code')
if __name__ == "__main__":
from logger import logger_config
logger_config(info_filename='arxiv_fetcher.log')
fetch_code_data()
CommandHandler('cancel', cancel),
MessageHandler(Filters.regex(ButtonPattern.CANCEL.value), cancel),
MessageHandler(Filters.text, change_topic)
]
},
fallbacks=[
CommandHandler('cancel', cancel),
MessageHandler(Filters.regex(ButtonPattern.CANCEL.value), cancel),
])
updater.dispatcher.add_handler(CommandHandler('hello', hello))
updater.dispatcher.add_handler(CommandHandler('start', hello))
updater.dispatcher.add_handler(CommandHandler('current_topic', current_topic))
updater.dispatcher.add_handler(
MessageHandler(Filters.regex(ButtonPattern.HELLO.value), hello))
updater.dispatcher.add_handler(
MessageHandler(Filters.regex(ButtonPattern.TOPIC.value), current_topic))
updater.dispatcher.add_handler(conv_handler)
updater.dispatcher.add_handler(MessageHandler(Filters.text, get_bot_response))
if __name__ == '__main__':
_download_nltk_resources()
logger.logger_config(level='DEBUG', root_level='WARNING')
log.info('Bot has been started... \n Press ctrl + C to STOP the bot')
updater.start_polling()
updater.idle()
log.info('Bot has been started')
```
"""
import os
import sys
from flask import Flask, request, redirect, jsonify, render_template, url_for
import tensorflow as tf
from foodnonfood import FoodNonfood
from logger import logger_config
UPLOAD_DIR = 'static/uploaded_images' # Directory to upload image files
EXTENSIONS = ['.jpeg', '.jpg', '.png'] # file extensions which can use for inference
FOODNONFOOD = FoodNonfood() # Initialization of FoodNonfood classifier
LOGGER = logger_config('flask_app')
tf.app.flags.DEFINE_boolean('debug', 'True', 'debug option')
tf.app.flags.DEFINE_string('model_path', None, 'file path of custom model')
FLAGS = tf.app.flags.FLAGS
app = Flask(__name__)
app.config['UPLOAD_DIR'] = UPLOAD_DIR
def _make_upload_dir():
if not os.path.isdir(UPLOAD_DIR):
os.makedirs(UPLOAD_DIR)
def _check_extension(filename):
if body['entity'] == 'organization':
neo.removeRecordNeo4j('organization', body['group_id'])
if body['entity'] == 'membership':
neo.removeRecordNeo4j('membership', body['id'])
neo.close()
return "Deleted successfully"
except Exception as e:
logging.error(f'Error while deleting a record. Details: {e}', exc_info=True)
neo.close()
return f"Fail {e}"
# Delete by id or group_id depending on the collection
@app.route('/parliament_records_es', methods=['DELETE'])
def delete_es():
try:
es = ES()
body = request.json
print(body['entity'])
es.removeRecordES(index='person', doc=body["queries"])
es.close()
except Exception as e:
logging.error(f'Error while deleting a record. Details: {e}', exc_info=True)
es.close()
return f"Fail {e}"
if __name__ == "__main__":
logging = logger.logger_config(with_file=True)
config = load_configs()
app.run(debug=False, host=config['kazdream']['url'], port=config['kazdream']['port'])
